From 52339188ca5416cdb551f20eceb98cd477c140fe Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:06:08 +0800 Subject: [PATCH 1/7] feat: add list-screen scene generator bdon_vision.scenes lays native card tiles out with the list widget geometry (1920x1080 reference canvas, centred six-member or four-snap rows, top padding and scroll offset) over blurred backgrounds, with a header bar, light UI distractors, out-of-gallery tiles recorded as foreign_cards and card-free screens. Truth files stay bdon-synthetic/2 compatible; added keys are listed in schema_extension. Generation is chunked over worker processes, resumable and deterministic per scene index; verify and stats subcommands check hashes and summarise a set. A separate final acquisition profile uses screen sizes and quality steps that appear in no other profile. List tiles of the BD rarity are rendered, and cropped UI sprites are reused across renderer processes. --- bdon_vision/assets.py | 15 + bdon_vision/scenarios.py | 4 + bdon_vision/scenes.py | 966 +++++++++++++++++++++++++++++++++++++++ docs/TRAINING.md | 12 + tests/test_scenes.py | 66 +++ 5 files changed, 1063 insertions(+) create mode 100644 bdon_vision/scenes.py create mode 100644 tests/test_scenes.py diff --git a/bdon_vision/assets.py b/bdon_vision/assets.py index 9c6babd..b2645a4 100644 --- a/bdon_vision/assets.py +++ b/bdon_vision/assets.py @@ -44,6 +44,9 @@ def native_module(data): old_line="text_values[ref('_statusValueText')]=str({'total':sum(state.power),'performance':state.power[0],'technic':state.power[1],'visual':state.power[2]}.get(state.param,''))" new_line=old_line+"\n if variant in ('snap','formation_snap') and state.param in ('performance','technic','visual'):\n text_values[ref('_statusValueText')]=format({'performance':state.power[0],'technic':state.power[1],'visual':state.power[2]}[state.param]/100.,'.2f')+'%'" if new_line not in code:code=code.replace(old_line,new_line) + # The list-row frame gradient catalogs carry key 20 (BD rarity) next to 2/3/4/10. + code=code.replace("if self.rarity not in (2,3,4,10): raise ValueError('Unknown rarity')", + "if self.rarity not in (2,3,4,10,20): raise ValueError('Unknown rarity')") if code!=original: temporary=source.with_name(source.name+f'.{os.getpid()}.tmp') temporary.write_text(code,encoding='utf-8');os.replace(temporary,source) @@ -51,6 +54,18 @@ def native_module(data): module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) + crop=module.crop_sprites + def crop_sprites(): + # Reuse the cropped sprite set when it matches the sprite table, so that + # many renderer processes can start without rewriting shared files. + index=module.UI/'sprite-index.json' + if index.exists(): + sprites=read(index) + if sprites.keys()==read(module.UI/'sprites.json').keys() and all( + (module.UI/s.get('file','')).is_file() for s in sprites.values()): + return sprites + return crop() + module.crop_sprites=crop_sprites from .font import NativeFont module.GameText = lambda: NativeFont(data) base=module.CardRenderer diff --git a/bdon_vision/scenarios.py b/bdon_vision/scenarios.py index 3911897..ee76090 100644 --- a/bdon_vision/scenarios.py +++ b/bdon_vision/scenarios.py @@ -11,6 +11,10 @@ ui_scale=(.88,1.04),resize=(.62,.94),quality=[57,69,81,91],blur=(.1,.65),rounds=[2,3]), 'stress':dict(resolutions=[(960,540),(2280,1080),(2532,1170),(2800,1752),(3840,2160)], ui_scale=(.65,1.28),resize=(.30,.78),quality=[30,42,51,78],blur=(.2,1.15),rounds=[2,3,4]), + # Held-back acquisition for a single evaluation after all models are frozen: + # its screen sizes and quality steps appear in no other profile. + 'final':dict(resolutions=[(1334,750),(1792,828),(2224,1668),(2400,1080),(2388,1668)], + ui_scale=(.86,1.06),resize=(.50,.92),quality=[47,64,77,88],blur=(.05,.8),rounds=[1,2,3]), } diff --git a/bdon_vision/scenes.py b/bdon_vision/scenes.py new file mode 100644 index 0000000..8675938 --- /dev/null +++ b/bdon_vision/scenes.py @@ -0,0 +1,966 @@ +"""List-screen scene generator (v2) with a parallel, resumable dataset writer. + +Native card tiles from ``assets.native_module`` are placed with the list widget +geometry: a 1920x1080 reference canvas scaled by ``min(W/1920, H/1080)``, +rows of six member or four snap cells centred horizontally, 180 reference +units of top padding, and a vertical scroll offset. Scenes add blurred +game-art backgrounds with haze, a header (title, sort and view buttons), +light UI distractors, out-of-gallery tiles (``foreign_cards``) and card-free +screens. The output is a ``bdon-synthetic/2`` dataset: the ``cards`` and +``ignored_cards`` lists keep their meaning and every new key is additive. +""" +import argparse +import colorsys +import hashlib +import io +import json +import math +import multiprocessing +import os +import random +import shutil +import time +from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor +from functools import lru_cache +from pathlib import Path + +import numpy as np +from PIL import Image, ImageDraw, ImageEnhance, ImageFilter, ImageFont, ImageOps + +from .assets import read, write, native_module +from .scenarios import PROFILES, VERSION, acquire + +GENERATOR = 'scenes-v2.1' +LOCALE_COLUMNS = {'ja': '_japanese', 'en': '_english', 'zh-Hant': '_traditionalChinese', + 'zh-Hans': '_simplifiedChinese', 'ko': '_korean'} +MODES = ('level', 'training', 'total', 'performance', 'technic', 'visual', 'hide') +MEMBER_ONLY_MODES = ('training', 'total') +RARITIES = {'member': (2, 3, 4, 10, 20), 'snap': (2, 3, 4, 10)} +# Reference-canvas geometry of the list widgets and their row prefabs. +GRID = {'member': dict(tile=(224, 294), columns=6, spacing=27), + 'snap': dict(tile=(326, 184), columns=4, spacing=36)} +REFERENCE = (1920, 1080) +LIST_PADDING_TOP, LIST_PADDING_BOTTOM = 180, 60 +RANK_CENTER = {'member': (194.5, 267.2), 'snap': (306.1, 153.8)} +TITLE_TEXT = {'member': 'ui_title_member_card_list', 'snap': 'ui_title_support_card_list'} +SORT_KEYS = {'member': ['AcquiredAt', 'Release', 'TotalPower', 'Rarity', 'Level', 'Character', 'Awakening', + 'MemberCardRank', 'Performance', 'Technique', 'Visual'], + 'snap': ['AcquiredAt', 'Release', 'TotalPower', 'Rarity', 'Level', 'SupportCardRank', + 'Performance', 'Technique', 'Visual']} +FOREIGN_METHODS = ('cross_kind', 'flip_recolor', 'mix', 'procedural') +DISTRACTOR_ICONS = ['IconSetting', 'IconInfo', 'IconHelp', 'IconRanking', 'IconSearchOutLine', 'IconFriend', + 'IconGood', 'IconReward', 'IconLog', 'IconFavorite', 'IconPlay', 'IconTag', 'IconDiary'] +SCHEMA_EXTENSION = { + 'screenshot': ['kind', 'display_mode', 'negative', 'screen_resolution', 'layout', 'background', + 'ui_elements', 'foreign_cards', 'ignored_foreign_cards'], + 'card': ['rarity', 'card_type', 'rank_visible', 'selected', 'event_bonus', 'badge', 'occluded_fraction'], + 'ignored_card': ['bbox', 'visible_fraction'], + 'foreign_card': 'card fields without id, plus foreign_method and foreign_sources', +} + + +def scene_backgrounds(data, profile): + """Optional scene backdrops; every fifth one is kept out of the training profile.""" + def number(path): + return int(''.join(ch for ch in path.stem if ch.isdigit()) or 0) + paths = sorted((Path(data)/'backgrounds/adv-stage').glob('*.webp')) + return [p for p in paths if profile != 'train' or number(p) % 5 != 0] + + +def heldout(card): + """Identities kept out of encoder training (members and snaps separately).""" + return card['id'] % 7 == 0 + + +def list_layout(kind, width, height, ui_scale=1.): + """Native list geometry in output pixels for scroll offset 0.""" + base = min(width/REFERENCE[0], height/REFERENCE[1]) + s = base*ui_scale + grid = GRID[kind] + tw, th = grid['tile'] + row = grid['columns']*tw+(grid['columns']-1)*grid['spacing'] + return dict(scale=s, base_scale=base, columns=grid['columns'], + origin=(width/2-row*s/2, LIST_PADDING_TOP*s), + pitch=((tw+grid['spacing'])*s, (th+grid['spacing'])*s), tile=(tw*s, th*s)) + + +def _box_fraction(mask, box, width, height): + """Covered fraction of an in-screen box; out-of-screen parts count as covered.""" + x, y, w, h = box + if w <= 0 or h <= 0: + return 1. + x0, y0, x1, y1 = max(0, math.floor(x)), max(0, math.floor(y)), min(width, math.ceil(x+w)), min(height, math.ceil(y+h)) + inside = max(0, x1-x0)*max(0, y1-y0) + covered = int(mask[y0:y1, x0:x1].sum()) if inside else 0 + return 1.-(inside-covered)/max(1., w*h) + + +def _composite(base, image, x, y): + """Alpha-composite with clipping at every edge of the destination.""" + left, top = max(0, -x), max(0, -y) + right, bottom = min(image.width, base.width-x), min(image.height, base.height-y) + if right > left and bottom > top: + base.alpha_composite(image, (x+left, y+top), (left, top, right, bottom)) + + +def _hue_shift(image, degrees, saturation=1.): + hsv = np.array(image.convert('RGB').convert('HSV'), dtype=np.int16) + hsv[..., 0] = (hsv[..., 0]+round(degrees/360*256)) % 256 + hsv[..., 1] = np.clip(hsv[..., 1]*saturation, 0, 255) + return Image.fromarray(hsv.astype(np.uint8), 'HSV').convert('RGB') + + +def _color(rng, light=None): + h = rng.random() + s = rng.uniform(.25, .85) + v = rng.uniform(.75, 1.) if light else rng.uniform(.25, .7) if light is False else rng.uniform(.35, 1.) + return tuple(round(c*255) for c in colorsys.hsv_to_rgb(h, s, v)) + + +def _crop_aspect(image, aspect, rng, zoom=1.): + """Random crop with the requested aspect, optionally zoomed in.""" + w, h = image.size + cw, ch = (w, w/aspect) if w/h < aspect else (h*aspect, h) + cw, ch = cw/zoom, ch/zoom + x, y = rng.uniform(0, w-cw), rng.uniform(0, h-ch) + return image.crop((round(x), round(y), round(x+cw), round(y+ch))) + + +def _figure(size, rng): + """Bust of a stylised character on a transparent layer.""" + w, h = size + im = Image.new('RGBA', size) + d = ImageDraw.Draw(im, 'RGBA') + r = rng.uniform(.15, .3)*min(w, h) + cx, cy = rng.uniform(.25, .75)*w, rng.uniform(.28, .55)*h + hair, cloth, accent, eye = _color(rng), _color(rng), _color(rng, True), _color(rng) + skin = (rng.randint(244, 255), rng.randint(212, 238), rng.randint(196, 224)) + line = tuple(max(0, c-110) for c in hair) + lw = max(1, round(r*.035)) + length = rng.uniform(.5, 3.2) + if rng.random() < .3: + for sgn in (-1, 1): + d.polygon([(cx+sgn*.45*r, cy-1.1*r), (cx+sgn*1.15*r, cy-1.9*r), (cx+sgn*1.05*r, cy-.6*r)], fill=hair, outline=line) + d.ellipse((cx-1.2*r, cy-1.3*r, cx+1.2*r, cy+1.0*r), fill=hair, outline=line, width=lw) + d.polygon([(cx-1.2*r, cy-.1*r), (cx-(1.3+rng.uniform(0, .35))*r, cy+length*r), (cx-.5*r, cy+(length+.25)*r), + (cx+.5*r, cy+(length+.25)*r), (cx+(1.3+rng.uniform(0, .35))*r, cy+length*r), (cx+1.2*r, cy-.1*r)], fill=hair) + d.rectangle((cx-.28*r, cy+.7*r, cx+.28*r, cy+1.5*r), fill=tuple(round(c*.9) for c in skin)) + d.polygon([(cx-2.1*r, h+5), (cx-1.55*r, cy+1.6*r), (cx-.32*r, cy+1.35*r), (cx+.32*r, cy+1.35*r), (cx+1.55*r, cy+1.6*r), + (cx+2.1*r, h+5)], fill=cloth, outline=line, width=lw) + d.polygon([(cx-.5*r, cy+1.35*r), (cx, cy+rng.uniform(1.8, 2.3)*r), (cx+.5*r, cy+1.35*r)], fill=accent, outline=line, width=lw) + d.ellipse((cx-.85*r, cy-.85*r, cx+.85*r, cy+.75*r), fill=skin) + d.polygon([(cx-.83*r, cy+.05*r), (cx-.35*r, cy+.85*r), (cx, cy+1.0*r), (cx+.35*r, cy+.85*r), (cx+.83*r, cy+.05*r)], fill=skin) + d.chord((cx-1.15*r, cy-1.35*r, cx+1.15*r, cy+.1*r), 180, 360, fill=hair) + edges = np.linspace(cx-1.05*r, cx+1.05*r, rng.randint(4, 8)+1) + for left, right in zip(edges[:-1], edges[1:]): + tip = (left+right)/2+rng.uniform(-.12, .12)*r + d.polygon([(left, cy-.65*r), (right, cy-.65*r), (tip, cy+rng.uniform(-.3, .15)*r)], fill=hair) + for sgn in (-1, 1): + d.polygon([(cx+sgn*.72*r, cy-.55*r), (cx+sgn*.98*r, cy-.45*r), (cx+sgn*(.92+rng.uniform(0, .25))*r, cy+rng.uniform(.6, 1.5)*r), + (cx+sgn*.7*r, cy+.35*r)], fill=hair) + ey, ew, eh, gap = cy+rng.uniform(.05, .2)*r, rng.uniform(.17, .25)*r, rng.uniform(.22, .33)*r, rng.uniform(.33, .42)*r + dark = tuple(max(0, c-80) for c in eye) + for sgn in (-1, 1): + ex = cx+sgn*gap + d.ellipse((ex-ew, ey-eh, ex+ew, ey+eh), fill=(255, 255, 255)) + d.ellipse((ex-ew*.85, ey-eh*.8, ex+ew*.85, ey+eh), fill=eye) + d.ellipse((ex-ew*.45, ey-eh*.15, ex+ew*.45, ey+eh*.65), fill=dark) + d.ellipse((ex-ew*.6, ey-eh*.65, ex-ew*.05, ey-eh*.1), fill=(255, 255, 255, 235)) + d.arc((ex-ew*1.2, ey-eh*1.15, ex+ew*1.2, ey+eh*1.05), 195, 345, fill=line, width=max(1, round(r*.07))) + d.line([(ex-ew*.9, ey-eh*1.6), (ex+ew*.8, ey-eh*1.75+sgn*.02*r)], fill=line, width=max(1, round(r*.035))) + d.ellipse((ex-ew, ey+eh*1.15, ex+ew, ey+eh*1.55), fill=(255, 130, 150, 80)) + my, mw = cy+.62*r, rng.uniform(.07, .2)*r + if rng.random() < .5: + d.chord((cx-mw, my-mw*.5, cx+mw, my+mw), 0, 180, fill=(196, 70, 92), outline=line, width=lw) + else: + d.arc((cx-mw, my-mw, cx+mw, my+mw*.5), 20, 160, fill=line, width=lw) + if rng.random() < .45: + sgn = rng.choice((-1, 1)) + bx, by, br = cx+sgn*.8*r, cy-.95*r, rng.uniform(.18, .32)*r + d.polygon([(bx, by), (bx-2*br, by-br), (bx-2*br, by+br)], fill=accent, outline=line) + d.polygon([(bx, by), (bx+2*br, by-br), (bx+2*br, by+br)], fill=accent, outline=line) + d.ellipse((bx-br*.5, by-br*.5, bx+br*.5, by+br*.5), fill=accent, outline=line) + return im.rotate(rng.uniform(-18, 18), resample=Image.Resampling.BICUBIC, center=(cx, cy)) + + +def _procedural_art(size, rng): + """Illustration-like tile: painted backdrop, light effects and stylised figures.""" + w, h = size + k = 2 + yy, xx = np.mgrid[0:h*k, 0:w*k].astype(np.float32) + angle = rng.uniform(0, 2*math.pi) + t = np.clip((xx/(w*k)-.5)*math.cos(angle)+(yy/(h*k)-.5)*math.sin(angle)+.5, 0, 1)[..., None] + a, b = np.array(_color(rng)), np.array(_color(rng)) + im = Image.fromarray(np.uint8(a*(1-t)+b*t)).convert('RGBA') + d = ImageDraw.Draw(im, 'RGBA') + if rng.random() < .3: + base = h*k*rng.uniform(.55, .85) + x = -10 + while x < w*k: + bw = rng.uniform(.06, .18)*w*k + top = base-rng.uniform(.05, .45)*h*k + d.rectangle((x, top, x+bw, h*k), fill=(*_color(rng, False), 255)) + for _ in range(rng.randint(0, 10)): + wx, wy = rng.uniform(x, x+bw), rng.uniform(top, h*k) + d.rectangle((wx, wy, wx+4*k, wy+5*k), fill=(255, 240, 180, 200)) + x += bw+rng.uniform(0, 8)*k + for _ in range(rng.randint(0, 4)): + points = [(rng.uniform(-.2, 1.2)*w*k, rng.uniform(-.2, 1.2)*h*k) for _ in range(rng.randint(3, 5))] + d.polygon(points, fill=(*_color(rng), rng.randint(40, 160))) + if rng.random() < .4: + for _ in range(rng.randint(2, 6)): + x0 = rng.uniform(0, w*k) + spread = rng.uniform(.05, .25)*w*k + d.polygon([(x0, -5), (x0+rng.uniform(-.6, .6)*w*k-spread, h*k), (x0+rng.uniform(-.6, .6)*w*k+spread, h*k)], + fill=(255, 255, rng.randint(200, 255), rng.randint(25, 80))) + for _ in range(rng.randint(3, 16)): + r = rng.uniform(.02, .16)*min(w, h)*k + cx, cy = rng.uniform(0, w*k), rng.uniform(0, h*k) + d.ellipse((cx-r, cy-r, cx+r, cy+r), fill=(*_color(rng, True), rng.randint(30, 120))) + figures = rng.choices([0, 1, 2], weights=(.2, .65, .15))[0] + for _ in range(figures): + im.alpha_composite(_figure(im.size, rng)) + d = ImageDraw.Draw(im, 'RGBA') + for _ in range(rng.randint(0, 12)): + cx, cy, r = rng.uniform(0, w*k), rng.uniform(0, h*k), rng.uniform(2, 9)*k + d.polygon([(cx, cy-r), (cx+r*.25, cy-r*.25), (cx+r, cy), (cx+r*.25, cy+r*.25), (cx, cy+r), + (cx-r*.25, cy+r*.25), (cx-r, cy), (cx-r*.25, cy-r*.25)], fill=(255, 255, 255, rng.randint(120, 230))) + im = im.convert('RGB').resize((w, h), Image.Resampling.LANCZOS) + if rng.random() < .6: + im = im.filter(ImageFilter.GaussianBlur(rng.uniform(.3, 1.2))) + return ImageEnhance.Sharpness(im).enhance(rng.uniform(1., 2.2)) + + +class SceneContext: + """Per-process resources: renderers, catalogue, fonts, sprites and art.""" + + def __init__(self, data, locales): + self.data = Path(data) + self.native = native_module(self.data) + self.renderers = {locale: self.native.CardRenderer(LOCALE_COLUMNS[locale]) for locale in locales} + self.catalog = read(self.data/'catalog.json')['cards'] + self.support_ranks = read(self.data/'master/MasterSupportCardRank.json')['_allData'] + self.member_limits = read(self.data/'master/MasterMemberCardLevelLimit.json')['_allData'] + self.texts = {row['_id']: row for row in read(self.data/'master/MasterText.json')['_allData']} + self.ui = self.data/'native/game-ui' + def short(value): + value = str(value or '') + return 2 <= len(value) <= 16 and '\n' not in value and '{' not in value and sum(c.isalnum() for c in value) >= 2 + self.short_texts = sorted(key for key, row in self.texts.items() if key.startswith('ui_') and all( + short(row.get(column)) for column in LOCALE_COLUMNS.values())) + self.sources = [c for c in self.catalog if not heldout(c)] + self.stage_pools = {} + self.art_cache = {} + self.overrides = {} + original = self.native.picture + + def picture(path): + image = self.overrides.get(Path(path).name) + return image if image is not None else original(path) + self.native.picture = picture + + def text(self, key, locale): + row = self.texts.get(key, {}) + return str(row.get(LOCALE_COLUMNS[locale]) or row.get('_english') or key) + + @lru_cache(maxsize=64) + def font(self, locale, size): + name = 'Pretendard-SemiBold.font' if locale == 'ko' else 'FZLTH_GB18030L2_R.font' + return ImageFont.truetype(str(self.ui/'fonts'/name), max(4, round(size))) + + @lru_cache(maxsize=64) + def sprite(self, name): + return Image.open(self.ui/'sprites'/f'{name}.png').convert('RGBA') + + def art(self, card, variant=None): + """Gallery artwork as RGB: snap landscape, member square or portrait.""" + if card['kind'] == 'snap': + name = f"snap-{card['asset_id']}.webp" + else: + name = f"member-{card['asset_id']}{'' if variant == 'portrait' else '-square'}.webp" + if name not in self.art_cache: + with Image.open(self.data/'native/assets'/name) as image: + self.art_cache[name] = image.convert('RGB') + return self.art_cache[name] + + # Card states --------------------------------------------------------------------------- + def card_state(self, kind, rarity, rank_group, rng): + rank, awake = rng.randint(1, 5), rng.randint(1, 5) + if kind == 'snap': + choices = [r for r in self.support_ranks if r['_group'] == rank_group and r['_rank'] == rank] + if not choices: + choices = [r for r in self.support_ranks if r['_group'] == rank_group] + rank = choices[0]['_rank'] + limit = choices[0]['_limitLevel'] + else: + limits = [r['_limitLevel'] for r in self.member_limits if r['_rarity'] == rarity and r['_awakeCount'] == awake] + limit = limits[0] if limits else 50 + level = rng.randint(1, limit) + power = tuple(rng.randint(500, 18000) if kind == 'member' else rng.randint(10, 1500) for _ in range(3)) + return dict(level=level, rank=rank, awake=awake, power=power) + + # Foreign artwork ----------------------------------------------------------------------- + def foreign_art(self, kind, rng): + """Out-of-gallery artwork built only from training-visible identities.""" + method = rng.choices(FOREIGN_METHODS, weights=(.32, .26, .2, .22))[0] + size = (384, 384) if kind == 'member' else (512, 288) + aspect = size[0]/size[1] + other = 'snap' if kind == 'member' else 'member' + sources = [] + if method == 'cross_kind': + card = rng.choice([c for c in self.sources if c['kind'] == other]) + source = self.art(card, 'portrait' if other == 'member' and rng.random() < .5 else None) + image = _crop_aspect(source, aspect, rng, rng.uniform(1., 1.3)).resize(size, Image.Resampling.LANCZOS) + sources.append(card) + elif method == 'flip_recolor': + card = rng.choice([c for c in self.sources if c['kind'] == kind]) + image = _crop_aspect(ImageOps.mirror(self.art(card)), aspect, rng, rng.uniform(1.2, 1.7)) + image = _hue_shift(image.resize(size, Image.Resampling.LANCZOS), rng.uniform(60, 300), rng.uniform(.6, 1.4)) + if rng.random() < .2: + image = Image.merge('RGB', [image.getchannel(i) for i in rng.sample(range(3), 3)]) + sources.append(card) + elif method == 'mix': + cards = rng.sample(self.sources, 2) + a, b = [_crop_aspect(self.art(c), aspect, rng, rng.uniform(1., 1.5)).resize(size, Image.Resampling.LANCZOS) + for c in cards] + style = rng.choice(['split', 'blend', 'strips']) + if style == 'blend': + image = Image.blend(a, b, rng.uniform(.35, .65)) + else: + yy, xx = np.mgrid[0:size[1], 0:size[0]].astype(np.float32) + if style == 'split': + angle = rng.uniform(0, math.pi) + t = (xx-size[0]/2)*math.cos(angle)+(yy-size[1]/2)*math.sin(angle) + mask = np.clip(t/rng.uniform(4, 40)+.5, 0, 1) + else: + mask = (np.floor(xx/size[0]*rng.randint(2, 4)) % 2).astype(np.float32) + image = Image.composite(b, a, Image.fromarray(np.uint8(mask*255))) + if rng.random() < .5: + image = _hue_shift(image, rng.uniform(-40, 40), rng.uniform(.8, 1.2)) + sources.extend(cards) + else: + image = _procedural_art(size, rng) + return image.convert('RGBA'), method, [{'kind': c['kind'], 'id': c['id']} for c in sources] + + # Backgrounds --------------------------------------------------------------------------- + @lru_cache(maxsize=16) + def stage(self, path): + with Image.open(path) as image: + return image.convert('RGB') + + def stage_pool(self, profile): + if profile not in self.stage_pools: + self.stage_pools[profile] = scene_backgrounds(self.data, profile) + return self.stage_pools[profile] + + @staticmethod + def _haze(image, rng): + """Washed-out list backdrop: desaturate, lift mid-tones, add a white haze gradient.""" + image = ImageEnhance.Color(image).enhance(rng.uniform(.5, .85)) + gamma = rng.uniform(.55, .85) + lut = [round(255*(i/255)**gamma) for i in range(256)]*3 + image = image.point(lut) + top, bottom = rng.uniform(.08, .28), rng.uniform(.18, .48) + fog = np.linspace(top, bottom, image.height, dtype=np.float32)[:, None, None] + tint = np.array(rng.choice([(255, 255, 255), (250, 248, 255), (255, 250, 244)]), np.float32) + return np.asarray(image, np.float32)*(1-fog)+tint*fog, {'gamma': round(gamma, 3), 'haze': [round(top, 3), round(bottom, 3)]} + + def background(self, width, height, rng, nrng, profile): + stages = self.stage_pool(profile) + roll = rng.random() + small = (max(32, width//8), max(18, height//8)) + info = {} + if stages and roll < .55: + path = rng.choice(stages) + crop = _crop_aspect(self.stage(path), width/height, rng, rng.uniform(1., 1.8)) + if rng.random() < .5: + crop = ImageOps.mirror(crop) + sigma = rng.uniform(.8, 2.) + low = crop.resize(small, Image.Resampling.BICUBIC).filter(ImageFilter.GaussianBlur(sigma)) + arr, tone = self._haze(low.resize((width, height), Image.Resampling.BICUBIC), rng) + info = {'type': 'stage', 'source': path.stem, 'blur': round(sigma*8, 1), **tone} + elif roll < (.75 if stages else .72): + snap = rng.random() < .85 + card = rng.choice([c for c in self.sources if c['kind'] == ('snap' if snap else 'member')]) + source = self.art(card) + crop = _crop_aspect(source, width/height, rng, rng.uniform(1., 2.4) if snap else rng.uniform(1.6, 3.)) + if rng.random() < .5: + crop = ImageOps.mirror(crop) + sigma = rng.uniform(1.3, 3.2) if snap else rng.uniform(2.4, 4.) + low = crop.resize(small, Image.Resampling.BICUBIC).filter(ImageFilter.GaussianBlur(sigma)) + arr, tone = self._haze(low.resize((width, height), Image.Resampling.BICUBIC), rng) + info = {'type': 'blurred_art', 'source': {'kind': card['kind'], 'id': card['id']}, 'blur': round(sigma*8, 1), **tone} + elif roll < (.88 if stages else .9): + noise = nrng.integers(100, 235, (max(1, height//16), max(1, width//16), 3), dtype=np.uint8) + low = Image.fromarray(noise).resize(small, Image.Resampling.BICUBIC).filter(ImageFilter.GaussianBlur(1.5)) + arr = np.asarray(low.resize((width, height), Image.Resampling.BICUBIC), np.float32) + info = {'type': 'noise'} + else: + yy = np.linspace(0, 1, height, dtype=np.float32)[:, None, None] + a, b = [np.array(colorsys.hsv_to_rgb(rng.random(), rng.uniform(.05, .4), rng.uniform(.75, 1.)), np.float32)*255 + for _ in range(2)] + arr = np.broadcast_to(a*(1-yy)+b*yy, (height, width, 3)).copy() + pattern = self.sprite('Pattern1_tiling') + tile = pattern.resize((max(8, round(88*height/1080)),)*2) + layer = Image.new('RGBA', (width, height)) + for y in range(0, height, tile.height): + for x in range(0, width, tile.width): + layer.alpha_composite(tile, (x, y)) + alpha = np.asarray(layer.getchannel('A'), np.float32)[..., None]/255*rng.uniform(.08, .25) + arr = arr*(1-alpha)+255*alpha + info = {'type': 'gradient_pattern'} + if rng.random() < .5: + grain = rng.uniform(1., 4.) + arr = arr+nrng.normal(0, grain, (height, width, 1)).astype(np.float32) + info['grain'] = round(grain, 2) + return Image.fromarray(np.uint8(np.clip(arr, 0, 255))).convert('RGBA'), info + + # Header and UI --------------------------------------------------------------------------- + def header(self, size, s, kind, locale, rng, inset): + """Title capsule plus sort/view buttons, drawn 2x supersampled.""" + width, height = size + k = s*2 + band = min(height, math.ceil(118*s)+2) + layer = Image.new('RGBA', (width*2, band*2)) + shadow = Image.new('RGBA', layer.size) + d, ds = ImageDraw.Draw(layer), ImageDraw.Draw(shadow) + + def p(*values): + return [v*k for v in values] + + def jitter(color, amount=10): + return tuple(max(0, min(255, c+rng.randint(-amount, amount))) for c in color)+((255,) if len(color) == 3 else ()) + blue, edge, plum = jitter((72, 86, 148)), jitter((38, 42, 88), 6), jitter((56, 45, 61)) + light = jitter((126, 134, 200)) + left = inset+rng.uniform(-3, 3) + right = width/s-inset+rng.uniform(-3, 3) + dy = rng.uniform(-3, 3) + elements = [] + + def add(kind_name, box): + elements.append({'type': kind_name, 'bbox': [box[0]*s, box[1]*s, (box[2]-box[0])*s, (box[3]-box[1])*s]}) + + def icon(name, cx, cy, size_): + image = self.sprite(name) + image = ImageOps.contain(image, (max(1, round(size_*k)),)*2, Image.Resampling.LANCZOS) + _composite(layer, image, round(cx*k-image.width/2), round(cy*k-image.height/2)) + + def circle(cx, cy, r, name): + ds.ellipse(p(cx-r, cy-r+5, cx+r, cy+r+5), fill=(0, 0, 0, 90)) + frame = self.sprite('FrameNormalButton_H80').resize((round(2*r*k),)*2, Image.Resampling.LANCZOS) + _composite(layer, frame, round((cx-r)*k), round((cy-r)*k)) + icon(name, cx, cy, r*.95) + title = self.text(TITLE_TEXT[kind], locale) + font = self.font(locale, 32*k) + y1, y2 = 21+dy, 97+dy + x1 = left+10.7 + x2 = max(left+553, left+346+font.getlength(title)/k+60) + ds.rounded_rectangle(p(x1, y1+5, x2, y2+5), radius=(y2-y1)/2*k, fill=(0, 0, 0, 80)) + d.rounded_rectangle(p(x1, y1, x2, y2), radius=(y2-y1)/2*k, fill=plum, outline=jitter((104, 92, 122)), + width=max(1, round(2*k))) + d.rounded_rectangle(p(x1+1, y1+1, left+156, y2-1), radius=30*k, fill=blue, outline=light, + width=max(1, round(1.5*k)), corners=(True, False, False, True)) + d.rounded_rectangle(p(left+177, y1+1, left+317, y2-1), radius=5*k, fill=blue, outline=light, width=max(1, round(1.5*k))) + icon('IconBack2', left+83, (y1+y2)/2, 46) + icon('IconHome', left+247, (y1+y2)/2, 50) + d.text(((left+346)*k, (y1+y2)/2*k), title, font=font, fill=(255, 255, 255, 255), anchor='lm', + stroke_width=max(0, round(.7*k)), stroke_fill=(255, 255, 255, 255)) + add('title', (x1, y1, x2, y2)) + cy = 59+dy + if rng.random() < .93: + circle(right-682, cy, 38.6, 'IconFilter') + add('button', (right-682-38.6, cy-38.6, right-682+38.6, cy+38.6)) + if rng.random() < .93: + label = self.text('ui_sort_key_'+rng.choice(SORT_KEYS[kind]), locale) + box = (right-608, cy-37, right-348, cy+37) + ds.rounded_rectangle(p(box[0], box[1]+5, box[2], box[3]+5), radius=37*k, fill=(0, 0, 0, 90)) + d.rounded_rectangle(p(*box), radius=37*k, fill=blue, outline=edge, width=max(1, round(3*k))) + d.rounded_rectangle(p(box[0]+5, box[1]+4, box[2]-5, cy), radius=30*k, fill=(255, 255, 255, 18)) + sort_font = self.font(locale, 28*k) + while sort_font.getlength(label) > (box[2]-box[0]-30)*k and sort_font.size > 8: + sort_font = self.font(locale, sort_font.size*.9) + d.text(((box[0]+box[2])/2*k, cy*k), label, font=sort_font, fill=(255, 255, 255, 255), anchor='mm', + stroke_width=max(0, round(.5*k)), stroke_fill=(255, 255, 255, 255)) + add('sort_button', box) + if rng.random() < .93: + circle(right-284, cy, 38.6, rng.choice(['IconDesc_Mask', 'IconAsc_Mask'])) + add('button', (right-284-38.6, cy-38.6, right-284+38.6, cy+38.6)) + if rng.random() < .93: + circle(right-182, cy, 38.6, 'IconSwitch') + add('button', (right-182-38.6, cy-38.6, right-182+38.6, cy+38.6)) + if rng.random() < .93: + box = (right-119, 12+dy, right-21, 107+dy) + ds.rounded_rectangle(p(box[0]+2, box[1]+6, box[2]+2, box[3]+6), radius=6*k, fill=(0, 0, 0, 100)) + d.rounded_rectangle(p(*box), radius=6*k, fill=jitter((66, 78, 150)), outline=edge, width=max(1, round(3*k))) + icon('IconMenu', (box[0]+box[2])/2, (box[1]+box[3])/2, 52) + add('button', box) + shadow = shadow.filter(ImageFilter.GaussianBlur(4*k)) + shadow.alpha_composite(layer) + out = shadow.convert('RGBa').resize((width, band), Image.Resampling.LANCZOS).convert('RGBA') + solid = layer.convert('RGBa').resize((width, band), Image.Resampling.LANCZOS).convert('RGBA') + return out, np.asarray(solid.getchannel('A')) >= 128, elements + + def distractors(self, overlay, size, s, rng, avoid, locale): + """Small UI items around the grid; recorded as UI elements.""" + width, height = size + d = ImageDraw.Draw(overlay) + elements = [] + gx0, gx1 = avoid + for _ in range(rng.choice([0, 0, 1, 1, 2, 3]) if rng.random() < .35 else 0): + size_ = rng.uniform(40, 90)*s + sides = [r for r in [(0, gx0), (gx1, width)] if r[1]-r[0] > 1.6*size_] + if not sides or height-size_*1.3 <= 130*s: + break + a, b = rng.choice(sides) + x, y = rng.uniform(a+.25*size_, b-1.25*size_), rng.uniform(130*s, height-1.3*size_) + image = ImageOps.contain(self.sprite(rng.choice(DISTRACTOR_ICONS)), (round(size_),)*2, Image.Resampling.LANCZOS) + if rng.random() < .5: + frame = self.sprite('FrameNormalButton_H80').resize((round(size_*1.4),)*2, Image.Resampling.LANCZOS) + _composite(overlay, frame, round(x-size_*.2), round(y-size_*.2)) + elements.append({'type': 'icon_button', 'bbox': [x-size_*.2, y-size_*.2, size_*1.4, size_*1.4]}) + else: + elements.append({'type': 'icon', 'bbox': [x, y, image.width, image.height]}) + _composite(overlay, image, round(x), round(y)) + if rng.random() < .06: + text = self.text(rng.choice(self.short_texts), locale) + font = self.font(locale, 30*s) + w, h = font.getlength(text)+80*s, 76*s + x, y = width/2-w/2, rng.choice([height*.82, height*.5, 140*s]) + d.rounded_rectangle((x, y, x+w, y+h), radius=h/2, fill=(24, 22, 40, rng.randint(170, 225))) + d.text((width/2, y+h/2), text, font=font, fill=(255, 255, 255, 255), anchor='mm') + elements.append({'type': 'toast', 'bbox': [x, y, w, h]}) + if rng.random() < .08: + font = self.font('en', 22*s) + clock = f'{rng.randint(0, 23)}:{rng.randint(0, 59):02d}' + d.text((28*s, 6*s), clock, font=font, fill=(255, 255, 255, 235)) + bx = width-90*s + d.rounded_rectangle((bx, 9*s, bx+46*s, 29*s), radius=5*s, outline=(255, 255, 255, 235), width=max(1, round(2*s))) + d.rectangle((bx+4*s, 13*s, bx+4*s+38*s*rng.uniform(.2, 1), 25*s), fill=(255, 255, 255, 235)) + elements.append({'type': 'status_bar', 'bbox': [0, 0, width, 34*s]}) + if rng.random() < .1: + r = 44*s + x, y = width-rng.uniform(70, 160)*s, height-rng.uniform(70, 160)*s + frame = self.sprite('FrameNormalButton_H80').resize((round(2*r),)*2, Image.Resampling.LANCZOS) + _composite(overlay, frame, round(x-r), round(y-r)) + icon = ImageOps.contain(self.sprite(rng.choice(DISTRACTOR_ICONS)), (round(r),)*2, Image.Resampling.LANCZOS) + _composite(overlay, icon, round(x-icon.width/2), round(y-icon.height/2)) + elements.append({'type': 'floating_button', 'bbox': [x-r, y-r, 2*r, 2*r]}) + return elements + + # Scene --------------------------------------------------------------------------------- + def scene(self, index, config): + profile, seed = config['profile'], config['seed'] + rng = random.Random(f'{GENERATOR}:{profile}:{seed}:{index}') + nrng = np.random.default_rng([seed, index, 2]) + self.overrides.clear() + kind = 'member' if index % 2 == 0 else 'snap' + locale = rng.choice(config['locales']) + modes = [m for m in config['modes'] if kind == 'member' or m not in MEMBER_ONLY_MODES] + mode = rng.choice(modes) + renderer = self.renderers[locale] + width, height = rng.choice(PROFILES[profile]['resolutions']) + ui_scale = rng.uniform(*PROFILES[profile]['ui_scale']) + geometry = list_layout(kind, width, height, ui_scale) + s = geometry['scale'] + tw, th = GRID[kind]['tile'] + cols, spacing = GRID[kind]['columns'], GRID[kind]['spacing'] + pitch_x = (tw+spacing+rng.uniform(-.8, .8))*s + pitch_y = (th+spacing+rng.uniform(-.8, .8))*s + x0 = geometry['origin'][0]+rng.uniform(-10, 10)*s + top = (LIST_PADDING_TOP+rng.uniform(-6, 6))*s + inset = rng.uniform(0, 90) if width/height > 2.05 and rng.random() < .7 else 0. + scene, background = self.background(width, height, rng, nrng, profile) + negative = rng.random() < config['negative_rate'] + pool = [c for c in self.catalog if c['kind'] == kind and (profile != 'train' or not heldout(c))] + rng.shuffle(pool) + rows_visible = max(1, math.ceil((height-top)/pitch_y)) + capacity = cols*rows_visible + roll = rng.random() + if roll < .45: + amount = rng.randint(min(capacity, len(pool)), len(pool)) + elif roll < .85: + amount = rng.randint(1, min(capacity, len(pool))) + else: + amount = rng.randint(1, min(6, len(pool))) + foreign_rate = rng.uniform(.1, .32) if rng.random() < config['foreign_screen_fraction'] else 0. + box = [] + real = iter(pool) + while len(box) < amount and not negative: + if rng.random() < foreign_rate: + box.append(None) + else: + card = next(real, None) + if card is None: + break + box.append(card) + rows = math.ceil(len(box)/cols) + content = LIST_PADDING_TOP+rows*th+max(0, rows-1)*spacing+LIST_PADDING_BOTTOM + max_scroll = max(0., content-height/s) + scroll = 0. if max_scroll <= 0 or rng.random() < .4 else rng.uniform(0, max_scroll) + resample = rng.choice([Image.Resampling.BILINEAR, Image.Resampling.BICUBIC, Image.Resampling.LANCZOS]) + render_scale = 1. if s <= 1.15 else min(2., round(s*4)/4) + selection = rng.random() + event = rng.random() < .1 + badges = rng.random() < .1 + type_bonus = rng.random() < .08 + placed = [] + cw, ch = round(tw*s), round(th*s) + box_index = index//4 + for slot, card in enumerate(box): + x = round(x0+(slot % cols)*pitch_x) + y = round(top-scroll*s+(slot//cols)*pitch_y) + if y+ch+40*s < 0 or y-40*s > height or x+cw < 0 or x > width: + continue + if card is None: + art, method, sources = self.foreign_art(kind, rng) + asset = 900000+slot + for name in ([f'member-{asset}.webp', f'member-{asset}-square.webp'] if kind == 'member' else [f'snap-{asset}.webp']): + self.overrides[name] = art + rarity = rng.choice(RARITIES[kind]) + card_type = rng.randint(1, 5) + template = rng.choice([c for c in self.catalog if c['kind'] == kind]) + state = self.card_state(kind, rarity, template['rank_group'], rng) + identity = {'foreign_method': method, 'foreign_sources': sources} + else: + asset, rarity, card_type = card['asset_id'], card['rarity'], card['card_type'] + # Four neighbouring scenes share one player box: states stay consistent. + state_rng = random.Random(f'state:{profile}:{seed}:{box_index}:{kind}:{card["id"]}') + state = self.card_state(kind, rarity, card['rank_group'], state_rng) + identity = {'id': card['id']} + placed.append(dict(identity=identity, x=x, y=y, rarity=rarity, card_type=card_type, state=state, asset=asset, + event_bonus=rng.choice([5, 10, 15, 20, 30, 50]) if event and rng.random() < .5 else 0, + badge=badges and rng.random() < .15, type_bonus=type_bonus and rng.random() < .4)) + on_screen = [i for i, p in enumerate(placed) if 0 <= p['y'] and p['y']+ch <= height] + selected = set() + if selection < .07 and placed: + selected = {rng.choice(on_screen or list(range(len(placed))))} + elif selection < .13: + selected = {i for i in range(len(placed)) if rng.random() < .35} + for i, p in enumerate(placed): + st = p['state'] + state = self.native.CardState(asset_id=p['asset'], rarity=p['rarity'], card_type=p['card_type'], + level=st['level'], rank=st['rank'], awake_count=st['awake'], param=mode, + power=st['power'], selected=i in selected, event_bonus=p['event_bonus'], + badge=p['badge'], type_bonus=p['type_bonus']) + tile = renderer.render(state, kind, scale=render_scale) + tile = tile.resize((round((tw+64)*s), round((th+64)*s)), resample) + _composite(scene, tile, p['x']-round(32*s), p['y']-round(32*s)) + p['selected'] = i in selected + overlay = Image.new('RGBA', (width, height)) + ui_elements = [] + scrollable = max_scroll > 0 and (kind == 'member' or rng.random() < .3) + if scrollable and rng.random() < .9: + draw = ImageDraw.Draw(overlay) + right = width/s-inset-50 + y_top, y_bottom = 222*s, (height/s-142)*s + if y_bottom > y_top+20*s: + draw.rounded_rectangle(((right-6)*s, y_top, (right-2)*s, y_bottom), radius=2*s, fill=(255, 255, 255, 70)) + length = max(.08, min(1., (height/s)/content))*(y_bottom-y_top) + start = y_top+(scroll/max_scroll)*(y_bottom-y_top-length) + draw.rounded_rectangle(((right-6)*s, start, (right-2)*s, start+length), radius=2*s, fill=(255, 255, 255, 235)) + ui_elements.append({'type': 'scrollbar', 'bbox': [(right-6)*s, y_top, 4*s, y_bottom-y_top]}) + grid_left = x0-40*s + grid_right = x0+(cols*tw+(cols-1)*spacing)*s+40*s + ui_elements += self.distractors(overlay, (width, height), s, rng, (grid_left, grid_right), locale) + if negative and rng.random() < .55: + draw = ImageDraw.Draw(overlay) + font = self.font(locale, 44*s) + text = self.text('ui_not_exist', locale) + draw.text((width/2+2*s, height/2+3*s), text, font=font, fill=(40, 36, 70, 140), anchor='mm') + draw.text((width/2, height/2), text, font=font, fill=(255, 255, 255, 255), anchor='mm') + box_ = draw.textbbox((width/2, height/2), text, font=font, anchor='mm') + ui_elements.append({'type': 'empty_message', 'bbox': [box_[0], box_[1], box_[2]-box_[0], box_[3]-box_[1]]}) + occluder = np.asarray(overlay.getchannel('A')) >= 128 + scene.alpha_composite(overlay) + header = rng.random() < .93 + if header: + layer, solid, elements = self.header((width, height), s, kind, locale, rng, inset) + scene.alpha_composite(layer, (0, 0)) + occluder = occluder.copy() + occluder[:solid.shape[0]] |= solid + ui_elements += elements + if rng.random() < .04: + veil = Image.new('RGBA', (width, height), rng.choice([(0, 0, 0), (255, 255, 255)])+(rng.randint(30, 70),)) + scene.alpha_composite(veil) + background['veil'] = True + cards, ignored, foreign, ignored_foreign = [], [], [], [] + rank_cx, rank_cy = RANK_CENTER[kind] + for p in placed: + x, y = p['x'], p['y'] + inside = max(0, min(height, y+ch)-max(0, y))*max(0, min(width, x+cw)-max(0, x)) + covered = _box_fraction(occluder, (x, y, cw, ch), width, height) + visible = max(0., 1.-covered) + occluded = max(0., covered-(1-inside/(cw*ch))) + st = p['state'] + field_box = (x-6*s, y+ch-56*s, 144*s, 56*s) + field_visible = (y+ch <= height and y+ch-56*s >= 0 and x-6*s >= 0 and x+138*s <= width + and _box_fraction(occluder, field_box, width, height) < .02) + rank_box = (x+(rank_cx-32)*s, y+(rank_cy-32*95/102)*s, 64*s, 64*95/102*s) + rank_visible = (x+(rank_cx-32)*s >= 2 and x+(rank_cx+32)*s <= width-2 and y+(rank_cy-32*95/102)*s >= 2 + and y+(rank_cy+32*95/102)*s <= height-2 and _box_fraction(occluder, rank_box, width, height) < .02) + entry = {'kind': kind, **({'id': p['identity']['id']} if 'id' in p['identity'] else {}), + 'level': st['level'] if field_visible and mode == 'level' else None, + 'awake_count': st['awake'] if field_visible and mode == 'training' else None, + 'card_rank': st['rank'] if rank_visible and mode != 'hide' else None, + 'bbox': [x, y, cw, ch], 'display_mode': mode, 'field_visible': field_visible, + 'visible_fraction': visible, + 'latent_state': {'level': st['level'], 'card_rank': st['rank'], 'awake_count': st['awake']}, + 'rarity': p['rarity'], 'card_type': p['card_type'], 'rank_visible': rank_visible, + 'selected': p['selected'], 'event_bonus': p['event_bonus'], 'badge': p['badge'], + 'occluded_fraction': round(occluded, 4)} + if 'id' not in p['identity']: + entry.update(p['identity']) + if visible >= .5: + (cards if 'id' in entry else foreign).append(entry) + elif visible > 0: + small = {'kind': kind, **({'id': entry['id']} if 'id' in entry else {}), + 'reason': 'less_than_half_visible', 'bbox': [x, y, cw, ch], 'visible_fraction': visible} + if 'id' in entry: + ignored.append(small) + else: + small.update(p['identity']) + ignored_foreign.append(small) + scene, acquisition = acquire(scene, rng, profile) + sx, sy = acquisition['scale_x'], acquisition['scale_y'] + + def scale_box(box): + return [v*(sx if i % 2 == 0 else sy) for i, v in enumerate(box)] + for entry in cards+ignored+foreign+ignored_foreign+ui_elements: + entry['bbox'] = scale_box(entry['bbox']) + output = Path(config['output']) + name = f'{index:06d}.' + ('webp' if config['image_format'] == 'webp' else 'png') + buffer = io.BytesIO() + if config['image_format'] == 'webp': + scene.save(buffer, format='WEBP', lossless=True, quality=config['webp_effort'], method=config['webp_method']) + else: + scene.save(buffer, format='PNG', compress_level=config['png_level']) + body = buffer.getvalue() + temporary = output/(name+f'.{os.getpid()}.tmp') + temporary.write_bytes(body) + os.replace(temporary, output/name) + self.overrides.clear() + return {'file': name, 'cards': cards, 'ignored_cards': ignored, + 'quality': min(stage['quality'] for stage in acquisition['compression']), + 'sha256': hashlib.sha256(body).hexdigest(), 'box_id': f'{profile}-{box_index}', 'locale': locale, + 'acquisition': acquisition, 'kind': kind, 'display_mode': mode, 'negative': negative, + 'screen_resolution': [width, height], + 'layout': {'canvas_scale': geometry['base_scale'], 'ui_scale': ui_scale, 'scale': s, 'columns': cols, + 'origin': [x0*sx, (top-scroll*s)*sy], 'pitch': [pitch_x*sx, pitch_y*sy], + 'tile': [cw*sx, ch*sy], 'scroll': scroll, 'max_scroll': max_scroll, + 'box_size': len(box), 'safe_inset': inset, 'header': header}, + 'background': background, 'ui_elements': ui_elements, + 'foreign_cards': foreign, 'ignored_foreign_cards': ignored_foreign} + + +# Parallel dataset writer ------------------------------------------------------------------- +_CONTEXT = None + + +def _initialize(data, locales): + global _CONTEXT + os.environ['OMP_NUM_THREADS'] = '1' + _CONTEXT = SceneContext(data, locales) + + +def _part_valid(output, part, start, end, check_files=True): + try: + document = read(part) + except (OSError, ValueError): + return None + records = document.get('records', []) + if document.get('start') != start or len(records) != end-start: + return None + for record in records if check_files else []: + path = output/record['file'] + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != record['sha256']: + return None + return records + + +def _chunk(job): + start, end, config = job + output = Path(config['output']) + part = output/'.parts'/f'{start:07d}.json' + if part.exists() and _part_valid(output, part, start, end) is not None: + return start, end-start, True + records = [_CONTEXT.scene(i, config) for i in range(start, end)] + write(part, {'start': start, 'end': end, 'records': records}) + return start, end-start, False + + +def generate(data, output, count, seed, profile, modes=MODES, locales=tuple(LOCALE_COLUMNS), workers=1, chunk=16, + image_format='webp', negative_rate=.05, foreign_screen_fraction=.6, webp_effort=60, webp_method=4, + png_level=6): + data, output = Path(data), Path(output) + if profile not in PROFILES: + raise ValueError(f'Unknown profile: {profile}') + output.mkdir(parents=True, exist_ok=True) + (output/'.parts').mkdir(exist_ok=True) + native_module(data) # applies the renderer patch layer once, before workers start + config = dict(output=str(output), seed=seed, profile=profile, modes=list(modes), locales=list(locales), + image_format=image_format, negative_rate=negative_rate, + foreign_screen_fraction=foreign_screen_fraction, webp_effort=webp_effort, + webp_method=webp_method, png_level=png_level) + jobs = [(start, min(count, start+chunk), config) for start in range(0, count, chunk)] + started, done = time.time(), 0 + + def progress(final=False): + write(output/'progress.json', {'generated': done, 'total': count, 'elapsed_s': round(time.time()-started, 1), + 'complete': final}) + if workers <= 1: + _initialize(data, locales) + results = map(_chunk, jobs) + for _, amount, _ in results: + done += amount + progress() + else: + method = 'fork' if 'fork' in multiprocessing.get_all_start_methods() else 'spawn' + with multiprocessing.get_context(method).Pool(workers, _initialize, (str(data), tuple(locales))) as pool: + for _, amount, _ in pool.imap_unordered(_chunk, jobs): + done += amount + progress() + print(json.dumps({'generated': done, 'total': count}), flush=True) + records = [] + for start, end, _ in jobs: + part_records = _part_valid(output, output/'.parts'/f'{start:07d}.json', start, end, check_files=False) + if part_records is None: + raise RuntimeError(f'Chunk {start}-{end} is missing or inconsistent') + records.extend(part_records) + font_manifest = data/'native/card-fonts/manifest.json' + write(output/'truth.json', { + 'schema': 'bdon-synthetic/2', 'generator': GENERATOR, 'seed': seed, 'profile': profile, + 'acquisition_version': VERSION, 'composition': 'list-layout-v2', 'screenshots': records, 'complete': True, + 'count': len(records), 'display_modes': list(modes), 'locales': list(locales), 'image_format': image_format, + 'identity_pool': 'heldout-excluded' if profile == 'train' else 'all', + 'background_pool': {'scene_backgrounds': len(scene_backgrounds(data, profile)), + 'rule': 'training profile skips scene backgrounds whose number % 5 == 0'}, + 'heldout_rule': 'id % 7 == 0', 'negative_rate': negative_rate, + 'foreign_screen_fraction': foreign_screen_fraction, 'schema_extension': SCHEMA_EXTENSION, + 'catalog_sha256': hashlib.sha256((data/'catalog.json').read_bytes()).hexdigest(), + 'renderer_sha256': hashlib.sha256((data/'native/render_game_components.py').read_bytes()).hexdigest(), + 'font_manifest_sha256': hashlib.sha256(font_manifest.read_bytes()).hexdigest() if font_manifest.exists() else None, + 'scope': 'native card UI on generated list screens; header and backgrounds approximate; SDF rasterization approximate'}) + shutil.rmtree(output/'.parts', ignore_errors=True) + progress(True) + print(json.dumps({'complete': True, 'screenshots': len(records), 'elapsed_s': round(time.time()-started, 1)}), flush=True) + + +def verify(dataset, threads=16): + """Check completeness, per-image SHA-256 and count of a written dataset.""" + dataset = Path(dataset) + document = read(dataset/'truth.json') + rows = document['screenshots'] + + def check(row): + path = dataset/row['file'] + return path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == row['sha256'] + with ThreadPoolExecutor(threads) as pool: + results = list(pool.map(check, rows)) + files = {row['file'] for row in rows} + stray = sorted(p.name for p in dataset.iterdir() if p.is_file() and p.suffix in ('.png', '.webp') and p.name not in files) + report = {'dataset': str(dataset), 'complete': bool(document.get('complete')), 'screenshots': len(rows), + 'declared_count': document.get('count'), 'hash_ok': sum(results), 'hash_failed': len(results)-sum(results), + 'duplicate_files': len(rows)-len(files), 'stray_images': len(stray), + 'ok': bool(document.get('complete')) and all(results) and len(rows) == len(files) and not stray} + return report + + +def statistics(dataset): + """Counts by kind, rarity, display mode and language; foreign and held-out tallies.""" + dataset = Path(dataset) + document = read(dataset/'truth.json') + out = defaultdict(Counter) + heldout_ids = defaultdict(Counter) + for row in document['screenshots']: + out['screens']['total'] += 1 + out['screens_by_kind'][row.get('kind')] += 1 + out['screens_by_locale'][row['locale']] += 1 + out['screens_by_mode'][row.get('display_mode')] += 1 + out['screen_resolution']['x'.join(map(str, row.get('screen_resolution', [])))] += 1 + out['screens']['negative'] += int(bool(row.get('negative'))) + out['screens']['with_foreign'] += int(bool(row.get('foreign_cards'))) + out['screens']['with_header'] += int(bool(row.get('layout', {}).get('header'))) + out['background'][row.get('background', {}).get('type')] += 1 + for element in row.get('ui_elements', []): + out['ui_elements'][element['type']] += 1 + for card in row['cards']: + key = card['kind'] + out['cards_by_kind'][key] += 1 + out['cards_by_rarity'][f"{key}:{card.get('rarity')}"] += 1 + out['cards_by_mode'][f"{key}:{card['display_mode']}"] += 1 + out['cards_by_locale'][f"{key}:{row['locale']}"] += 1 + out['visible_fields'][f"{key}:level"] += card['level'] is not None + out['visible_fields'][f"{key}:awake_count"] += card['awake_count'] is not None + out['visible_fields'][f"{key}:card_rank"] += card['card_rank'] is not None + if card['id'] % 7 == 0: + out['heldout_cards'][key] += 1 + heldout_ids[key][card['id']] += 1 + for card in row.get('foreign_cards', []): + out['foreign_by_kind'][card['kind']] += 1 + out['foreign_by_method'][card['foreign_method']] += 1 + out['foreign_by_rarity'][f"{card['kind']}:{card['rarity']}"] += 1 + out['ignored']['cards'] += len(row.get('ignored_cards', [])) + out['ignored']['foreign'] += len(row.get('ignored_foreign_cards', [])) + ids = defaultdict(set) + for row in document['screenshots']: + for card in row['cards']: + ids[card['kind']].add(card['id']) + out['identities'] = Counter({k: len(v) for k, v in ids.items()}) + total = sum(out['cards_by_kind'].values())+sum(out['foreign_by_kind'].values()) + files = [dataset/row['file'] for row in document['screenshots']] + report = {k: dict(sorted(v.items(), key=lambda kv: str(kv[0]))) for k, v in out.items()} + report['heldout_ids'] = {k: dict(sorted(v.items())) for k, v in heldout_ids.items()} + report['foreign_fraction'] = sum(out['foreign_by_kind'].values())/max(1, total) + report['image_bytes'] = sum(p.stat().st_size for p in files if p.exists()) + report['profile'] = document['profile'] + report['complete'] = document['complete'] + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='action', required=True) + p = sub.add_parser('generate') + p.add_argument('--data', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--count', type=int, required=True) + p.add_argument('--seed', type=int, required=True) + p.add_argument('--profile', choices=list(PROFILES), required=True) + p.add_argument('--modes', default=','.join(MODES)) + p.add_argument('--locales', default=','.join(LOCALE_COLUMNS)) + p.add_argument('--workers', type=int, default=1) + p.add_argument('--chunk', type=int, default=16) + p.add_argument('--format', choices=['webp', 'png'], default='webp') + p.add_argument('--negative-rate', type=float, default=.05) + p.add_argument('--foreign-screen-fraction', type=float, default=.6) + p = sub.add_parser('verify') + p.add_argument('dataset', type=Path, nargs='+') + p = sub.add_parser('stats') + p.add_argument('dataset', type=Path) + p.add_argument('--output', type=Path) + a = parser.parse_args() + if a.action == 'generate': + modes = tuple(a.modes.split(',')) + locales = tuple(a.locales.split(',')) + if not set(modes) <= set(MODES) or not any(m not in MEMBER_ONLY_MODES for m in modes): + parser.error('Unsupported display modes') + if not set(locales) <= set(LOCALE_COLUMNS): + parser.error('Unsupported locale') + generate(a.data, a.output, a.count, a.seed, a.profile, modes, locales, a.workers, a.chunk, a.format, + a.negative_rate, a.foreign_screen_fraction) + elif a.action == 'verify': + reports = [verify(d) for d in a.dataset] + for report in reports: + print(json.dumps(report), flush=True) + if not all(r['ok'] for r in reports): + raise SystemExit(1) + else: + report = statistics(a.dataset) + if a.output: + write(a.output, report) + print(json.dumps(report, ensure_ascii=False), flush=True) + + +if __name__ == '__main__': + main() diff --git a/docs/TRAINING.md b/docs/TRAINING.md index 6a0f933..04ceee1 100644 --- a/docs/TRAINING.md +++ b/docs/TRAINING.md @@ -26,6 +26,18 @@ python -m bdon_vision.audit_merge --dataset synthetic/validation --observations 数据生成最后才将 `complete` 置为 true。训练裁块构建检查全部截图哈希。评估要求身份及 bbox IoU≥0.5，记录漏卡、多报、字段正确／错误／未知、隐藏字段误报，以及按语言、卡片尺寸、压缩和分辨率分层。`audit_merge` 额外检查 box 合并后的字段、凭空出现的值、虚假冲突和真实冲突丢失。 +## 列表整屏场景 + +`bdon_vision.scenes` 按列表组件的原生几何放置原生卡块：1920×1080 参考画布按 `min(W/1920, H/1080)` 缩放，成员每行 6 张、留影每行 4 张并水平居中，顶部留白 180 个参考单位，再加滚动偏移。画面另含带雾感的模糊背景、顶栏标题与排序／视图按钮、少量界面干扰、图库外美术卡块和无卡负样本。可选的场景背景放在 `DATA/backgrounds/adv-stage/*.webp`，由使用者自行准备；缺失时只用卡面模糊、噪声和渐变背景。 + +```bash +python -m bdon_vision.scenes generate --data DATA --output scenes/train --count 16000 --seed 1 --profile train --workers 48 +python -m bdon_vision.scenes verify scenes/train +python -m bdon_vision.scenes stats scenes/train --output scenes/train-stats.json +``` + +真值仍是 `bdon-synthetic/2`：`cards`、`ignored_cards` 语义不变，新增字段列在 `truth.json` 的 `schema_extension` 中。`visible_fraction` 同时扣除屏外部分和界面遮挡；`foreign_cards` 记录图库外美术的卡框和字段，不含 `id`。train 档排除 `id % 7 == 0` 的身份，外来卡来源和卡面背景也排除；编号是 5 的倍数的场景背景只用于非训练档。`final` 采集档的分辨率和压缩质量与其他档都不同，只在模型冻结后使用。图片可存为无损 WebP，像素与施加采集损伤后的结果一致。每张图由 profile、seed 和序号单独决定，与进程数和分块方式无关。 + ## 场景微调 GPU 环境使用已验证的 PyTorch／ONNX 依赖；不要把 GPU 包加入普通 CPU 使用者的运行依赖。实际训练脚本及模型的 JSON 报告保存参数、步数、样本量和数据指纹。 diff --git a/tests/test_scenes.py b/tests/test_scenes.py new file mode 100644 index 0000000..dadfdb2 --- /dev/null +++ b/tests/test_scenes.py @@ -0,0 +1,66 @@ +"""List-screen geometry, coverage accounting and dataset verification for scene generation.""" +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +import numpy as np + +from bdon_vision.scenes import _box_fraction, list_layout, verify + + +class ListLayoutTests(unittest.TestCase): + def test_member_rows_are_centred_on_a_4_by_3_screen(self): + layout = list_layout('member', 1441, 1080) + self.assertAlmostEqual(layout['scale'], 1441/1920) + self.assertEqual(layout['columns'], 6) + self.assertAlmostEqual(layout['origin'][0], 1441/2-1479*1441/1920/2) + self.assertAlmostEqual(layout['origin'][1], 180*1441/1920) + self.assertAlmostEqual(layout['pitch'][0], 251*1441/1920) + self.assertAlmostEqual(layout['pitch'][1], 321*1441/1920) + + def test_snap_rows_hold_four_cells(self): + layout = list_layout('snap', 1920, 1080) + self.assertEqual(layout['columns'], 4) + self.assertAlmostEqual(layout['origin'][0], (1920-1412)/2) + self.assertEqual(layout['tile'], (326., 184.)) + + def test_wide_screens_keep_height_scale_and_fixed_columns(self): + layout = list_layout('member', 2340, 1080) + self.assertAlmostEqual(layout['scale'], 1.) + self.assertAlmostEqual(layout['origin'][0], (2340-1479)/2) + + def test_ui_scale_multiplies_canvas_scale(self): + self.assertAlmostEqual(list_layout('snap', 1920, 1080, .9)['scale'], .9) + + +class CoverageTests(unittest.TestCase): + def test_visible_box_is_uncovered(self): + mask = np.zeros((100, 200), bool) + self.assertEqual(_box_fraction(mask, (10, 10, 50, 20), 200, 100), 0.) + + def test_mask_and_screen_edges_count_as_covered(self): + mask = np.zeros((100, 200), bool) + mask[:, :20] = True + self.assertAlmostEqual(_box_fraction(mask, (10, 10, 40, 10), 200, 100), .25) + self.assertAlmostEqual(_box_fraction(mask, (180, 90, 40, 20), 200, 100), .75) + + +class VerifyTests(unittest.TestCase): + def test_hash_mismatch_and_stray_images_fail(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root/'000000.png').write_bytes(b'first') + row = {'file': '000000.png', 'sha256': hashlib.sha256(b'first').hexdigest(), 'cards': []} + (root/'truth.json').write_text(json.dumps({'complete': True, 'count': 1, 'screenshots': [row]}), encoding='utf-8') + self.assertTrue(verify(root)['ok']) + (root/'000001.png').write_bytes(b'stray') + self.assertFalse(verify(root)['ok']) + (root/'000001.png').unlink() + (root/'000000.png').write_bytes(b'changed') + self.assertEqual(verify(root)['hash_failed'], 1) + + +if __name__ == '__main__': + unittest.main() From 3dc4d79fb76f925732a84484c5d2e9b78aa676bd Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:06:57 +0800 Subject: [PATCH 2/7] feat: add card-frame locator A stride-8 CenterNet-style network (MobileNetV3-Large with a light FPN) predicts member and snap tile centres, sizes and offsets in a whole screenshot without a reference gallery. Includes NumPy target encoding and decoding with visibility and aspect filters, scene-set training with scale, crop, mosaic, resampling and compression augmentation, detection metrics with ignore regions, ONNX export with a CPU consistency check, and unit tests on synthetic inputs. --- bdon_vision/evaluate_locator.py | 233 ++++++++++++++++++++ bdon_vision/export_locator.py | 90 ++++++++ bdon_vision/locator.py | 199 +++++++++++++++++ bdon_vision/locator_eval.py | 218 ++++++++++++++++++ bdon_vision/locator_model.py | 83 +++++++ bdon_vision/train_locator.py | 378 ++++++++++++++++++++++++++++++++ tests/test_locator.py | 130 +++++++++++ 7 files changed, 1331 insertions(+) create mode 100644 bdon_vision/evaluate_locator.py create mode 100644 bdon_vision/export_locator.py create mode 100644 bdon_vision/locator.py create mode 100644 bdon_vision/locator_eval.py create mode 100644 bdon_vision/locator_model.py create mode 100644 bdon_vision/train_locator.py create mode 100644 tests/test_locator.py diff --git a/bdon_vision/evaluate_locator.py b/bdon_vision/evaluate_locator.py new file mode 100644 index 0000000..425599e --- /dev/null +++ b/bdon_vision/evaluate_locator.py @@ -0,0 +1,233 @@ +"""Evaluate card-frame locators and the RootSIFT baseline on scene and real sets. + +Writes one JSON report with recall / precision at IoU 0.5 and 0.75, card kind +accuracy, strata by card width and resolution, and per-image results for +real screenshots. The score threshold is chosen on the first scene set +(normally validation) and then applied unchanged to every set. +""" +import argparse +import json +import time +from multiprocessing import Pool +from pathlib import Path + +import cv2 +import numpy as np + +from .locator import CLASSES, Locator, decode, letterbox, to_tensor +from .locator_eval import Accumulator, load_scenes, match +from .train_locator import expand, write_json + + +def load_real(directory): + """Real screenshots with approximate tile rectangles (``screenshots[].cards[].bbox``).""" + directory = Path(directory) + truth = json.loads((directory / 'truth.json').read_text(encoding='utf-8')) + scenes = [] + for row in truth['screenshots']: + cards = row.get('cards', []) + [dict(c, foreign=True) for c in row.get('foreign_cards', [])] + scenes.append({'path': str(directory / row['file']), 'chunk': directory.name, + 'boxes': np.asarray([c['bbox'] for c in cards], np.float32).reshape(-1, 4), + 'labels': np.asarray([CLASSES.index(c['kind']) for c in cards], np.int64), + 'ids': [c.get('id') for c in cards], + 'foreign': np.asarray([bool(c.get('foreign')) for c in cards], bool), + 'ignored_rows': row.get('ignored_cards', []), 'source_resolution': None, 'profile': 'real'}) + return scenes + + +def read_rgb(path): + image = cv2.imread(path, cv2.IMREAD_COLOR) + if image is None: + raise ValueError(f'unreadable image: {path}') + return cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + + +class TorchPredictor: + def __init__(self, weights, width=96): + import torch + from .locator_model import LocatorNet + self.torch = torch + self.model = LocatorNet(width=width) + self.model.load_state_dict(torch.load(weights, map_location='cpu', weights_only=True)) + self.model = self.model.cuda().eval() + + def raw(self, image, long_edge): + torch = self.torch + canvas, scale = letterbox(image, long_edge) + with torch.inference_mode(): + heat, size, offset = self.model(torch.from_numpy(to_tensor(canvas)).cuda()) + heat = torch.sigmoid(heat)[0].cpu().numpy() + size = (torch.exp(size.clamp(max=8)) * 8)[0].cpu().numpy() + offset = offset[0].cpu().numpy() + return heat, size, offset, scale + + +class OnnxPredictor: + def __init__(self, model, threads=4): + self.locators = {} + self.model, self.threads = model, threads + + def raw(self, image, long_edge): + if long_edge not in self.locators: + self.locators[long_edge] = Locator(self.model, long_edge, self.threads) + return self.locators[long_edge].raw(image) + + +def evaluate_sets(predictor, sets, sizes, threshold=None, real=None, overlay=None, aspect_tolerance=None): + report = {'sizes': {}} + for size in sizes: + accs, per_image, timings = {}, {}, [] + for name, scenes in sets: + acc = Accumulator() + for scene in scenes: + image = read_rgb(scene['path']) + h, w = image.shape[:2] + started = time.perf_counter() + heat, wh, offset, scale = predictor.raw(image, size) + timings.append(time.perf_counter() - started) + acc.add(decode(heat, wh, offset, scale, .05, image_size=(w, h), aspect_tolerance=aspect_tolerance), + scene, w, h, scale) + accs[name] = acc + chosen = threshold if threshold is not None else accs[sets[0][0]].best_threshold(.5) + entry = {'threshold': chosen, 'threshold_source': 'fixed' if threshold is not None else f'best F1@0.5 on {sets[0][0]}', + 'aspect_tolerance': aspect_tolerance, 'sets': {}} + for name, acc in accs.items(): + entry['sets'][name] = {'ap50': acc.average_precision(.5), 'ap75': acc.average_precision(.75), + 'strata': acc.summary(chosen)} + if real: + entry['real'] = evaluate_real(predictor, real, size, chosen, overlay, aspect_tolerance) + report['sizes'][str(size)] = entry + print(json.dumps({'size': size, 'threshold': chosen, + **{n: {'ap50': round(e['ap50'], 4), 'ap75': round(e['ap75'], 4), + 'overall': e['strata'].get('overall')} for n, e in entry['sets'].items()}}), flush=True) + return report + + +def _top_bar(box, height): + return bool(box[1] + box[3] / 2 < .1 * height) + + +def evaluate_real(predictor, scenes, size, threshold, overlay=None, aspect_tolerance=None): + rows = [] + for scene in scenes: + image = read_rgb(scene['path']) + h, w = image.shape[:2] + heat, wh, offset, scale = predictor.raw(image, size) + detections = decode(heat, wh, offset, scale, threshold, image_size=(w, h), aspect_tolerance=aspect_tolerance) + rows.append(real_row(scene, image, detections, overlay, f'{size}')) + return rows + + +def real_row(scene, image, detections, overlay=None, tag=''): + h, w = image.shape[:2] + result, truth = match(detections, scene['boxes'], scene['labels'], np.zeros((0, 4))) + order = sorted(range(len(detections)), key=lambda i: -detections[i]['score']) + false_positives = [dict(detections[order[i]], top_bar=_top_bar(detections[order[i]]['bbox'], h)) + for i, r in enumerate(result) if r[.5] == 'fp'] + cards = [] + for g in range(len(scene['boxes'])): + m = truth[.5][g] + cards.append({'kind': CLASSES[int(scene['labels'][g])], 'id': scene['ids'][g] if 'ids' in scene else None, + 'found': m is not None, 'kind_correct': bool(m and m[1]), + 'found_iou75': truth[.75][g] is not None, 'score': m[0] if m else None}) + if overlay: + canvas = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + for b in scene['boxes']: + x, y, bw, bh = map(int, b) + cv2.rectangle(canvas, (x, y), (x + bw, y + bh), (0, 200, 0), 2) + for d in detections: + x, y, bw, bh = map(int, d['bbox']) + color = (255, 80, 0) if d['kind'] == 'member' else (0, 80, 255) + cv2.rectangle(canvas, (x, y), (x + bw, y + bh), color, 2) + cv2.putText(canvas, f"{d['kind'][0]}{d['score']:.2f}", (x + 3, y + 18), cv2.FONT_HERSHEY_SIMPLEX, .6, color, 2) + Path(overlay).mkdir(parents=True, exist_ok=True) + cv2.imwrite(str(Path(overlay) / f'{tag}-{Path(scene["path"]).stem}.jpg'), canvas, [cv2.IMWRITE_JPEG_QUALITY, 85]) + return {'file': Path(scene['path']).name, 'truth': len(scene['boxes']), 'detections': len(detections), + 'found': sum(c['found'] for c in cards), 'kind_correct': sum(c['kind_correct'] for c in cards), + 'false_positives': len(false_positives), 'false_positives_in_top_bar': sum(f['top_bar'] for f in false_positives), + 'false_positive_boxes': false_positives, 'cards': cards} + + +_ENGINE = None + + +def _baseline_worker(args): + data, path, grid, excluded = args + global _ENGINE + if _ENGINE is None: + from .engine import Engine + _ENGINE = Engine(data, threads=1, excluded_identities=excluded or None) + image = cv2.imread(path, cv2.IMREAD_COLOR) + started = time.perf_counter() + items = _ENGINE.identify(image) + if grid: + items = _ENGINE.fill_grid(image, items) + elapsed = time.perf_counter() - started + out = [] + for item in items: + x, y, w, h = _ENGINE.ui_bbox(item) + s = w / (212 if item['kind'] == 'member' else 314) + out.append({'kind': item['kind'], 'score': float(item['identity_confidence']), 'id': item['id'], + 'bbox': [x - 6 * s, y - 6 * s, w + 12 * s, h + 12 * s]}) + return out, elapsed + + +def evaluate_baseline(data, sets, real=None, workers=8, grid=False, overlay=None, excluded=()): + excluded = {(kind, int(i)) for kind, i in excluded} + report = {'method': 'RootSIFT + FLANN gallery matching' + (' + grid fill' if grid else ''), + 'excluded_identities': sorted(excluded), + 'tile_box': 'matched artwork window expanded by 6 logical pixels per side', 'sets': {}} + with Pool(workers) as pool: + for name, scenes in sets: + results = pool.map(_baseline_worker, [(data, s['path'], grid, excluded) for s in scenes], chunksize=2) + acc = Accumulator() + for scene, (detections, _) in zip(scenes, results): + image = cv2.imread(scene['path'], cv2.IMREAD_UNCHANGED) + h, w = image.shape[:2] + acc.add(detections, scene, w, h, 640 / max(w, h)) + report['sets'][name] = {'strata': acc.summary(-1.), + 'median_s': float(np.median([e for _, e in results]))} + print(json.dumps({'baseline': name, 'overall': report['sets'][name]['strata'].get('overall')}), flush=True) + if real: + results = pool.map(_baseline_worker, [(data, s['path'], grid, excluded) for s in real], chunksize=1) + report['real'] = [real_row(scene, read_rgb(scene['path']), detections, overlay, 'baseline') + for scene, (detections, _) in zip(real, results)] + return report + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + p.add_argument('--model', help='Locator state dict (.pt, GPU) or exported .onnx (CPU)') + p.add_argument('--width', type=int, default=96) + p.add_argument('--set', action='append', default=[], metavar='NAME=GLOB', + help='Scene set; the first one selects the score threshold') + p.add_argument('--real', nargs='*', default=[], help='Directories with real screenshots and truth.json') + p.add_argument('--sizes', type=int, nargs='+', default=[640]) + p.add_argument('--threshold', type=float) + p.add_argument('--aspect-tolerance', type=float, help='Drop tiles whose aspect differs from the class by more') + p.add_argument('--baseline-data', help='Data directory for the RootSIFT baseline') + p.add_argument('--baseline-grid', action='store_true') + p.add_argument('--baseline-exclude', nargs='*', default=[], metavar='KIND:ID', + help='Remove identities from the baseline gallery (simulates cards added after it was built)') + p.add_argument('--workers', type=int, default=8, help='Baseline worker processes') + p.add_argument('--threads', type=int, default=4, help='onnxruntime CPU threads') + p.add_argument('--overlay', help='Directory for annotated real screenshots') + p.add_argument('--output', required=True) + a = p.parse_args() + sets = [] + for spec in a.set: + name, pattern = spec.split('=', 1) + sets.append((name, load_scenes(expand(pattern.split(','))))) + real = [s for d in a.real for s in load_real(d)] + report = {'sets': {name: len(scenes) for name, scenes in sets}, 'real_images': len(real)} + if a.model: + predictor = OnnxPredictor(a.model, a.threads) if a.model.endswith('.onnx') else TorchPredictor(a.model, a.width) + report['locator'] = evaluate_sets(predictor, sets, a.sizes, a.threshold, real, a.overlay, a.aspect_tolerance) + if a.baseline_data: + excluded = [item.split(':') for item in a.baseline_exclude] + report['baseline'] = evaluate_baseline(a.baseline_data, sets, real, a.workers, a.baseline_grid, a.overlay, excluded) + write_json(a.output, report) + + +if __name__ == '__main__': + main() diff --git a/bdon_vision/export_locator.py b/bdon_vision/export_locator.py new file mode 100644 index 0000000..6397886 --- /dev/null +++ b/bdon_vision/export_locator.py @@ -0,0 +1,90 @@ +"""Export a trained locator to ONNX and check it against PyTorch on CPU. + +The exported graph takes ``image`` float32 [1, 3, H, W] (RGB, 0..1; H and W +multiples of 32) and returns ``heatmap`` [1, 2, H/8, W/8] (sigmoid), +``size`` [1, 2, H/8, W/8] (tile width and height in input pixels) and +``offset`` [1, 2, H/8, W/8] (tile center relative to the cell origin, in +cells). ``bdon_vision.locator.decode`` documents the decoding. +""" +import argparse +import hashlib +import json +import time +from pathlib import Path + +import numpy as np +import torch + +from .locator import Locator, decode, letterbox, to_tensor +from .locator_model import ExportedLocator, LocatorNet, export_onnx + + +def compare(net, onnx_path, images, sizes, threshold=.3): + import cv2 + model = ExportedLocator(net).eval().cpu() + rows = [] + for path in images: + image = cv2.cvtColor(cv2.imread(str(path), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + for size in sizes: + locator = Locator(onnx_path, size, threads=1) + canvas, scale = letterbox(image, size) + with torch.no_grad(): + ref = [t[0].numpy() for t in model(torch.from_numpy(to_tensor(canvas)))] + got = locator.raw(image)[:3] + a, b = decode(*ref, scale, threshold), decode(*got, scale, threshold) + same = len(a) == len(b) and all(x['kind'] == y['kind'] and np.allclose(x['bbox'], y['bbox'], atol=.05) + for x, y in zip(a, b)) + rows.append({'image': Path(path).name, 'size': size, 'input': list(canvas.shape[:2]), + 'max_abs_diff': {n: float(np.abs(r - g).max()) for n, r, g in zip(('heatmap', 'size', 'offset'), ref, got)}, + 'detections': len(a), 'decoded_identical': bool(same)}) + return rows + + +def benchmark(onnx_path, sizes, aspect=(4, 3), repeats=20, threads=1): + import onnxruntime as ort + options = ort.SessionOptions() + options.intra_op_num_threads, options.inter_op_num_threads = threads, 1 + session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=['CPUExecutionProvider']) + out = {} + for size in sizes: + h = -(-round(size * aspect[1] / aspect[0]) // 32) * 32 + x = np.random.default_rng(0).random((1, 3, h, size), dtype=np.float32) + session.run(None, {'image': x}) + times = [] + for _ in range(repeats): + started = time.perf_counter() + session.run(None, {'image': x}) + times.append(time.perf_counter() - started) + out[f'{size}x{h}'] = {'median_ms': 1000 * float(np.median(times)), 'min_ms': 1000 * float(np.min(times))} + return out + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + p.add_argument('--weights', required=True) + p.add_argument('--output', required=True) + p.add_argument('--width', type=int, default=96) + p.add_argument('--images', nargs='*', default=[]) + p.add_argument('--sizes', type=int, nargs='+', default=[640]) + p.add_argument('--benchmark', action='store_true') + a = p.parse_args() + net = LocatorNet(width=a.width) + net.load_state_dict(torch.load(a.weights, map_location='cpu', weights_only=True)) + net.eval() + export_onnx(net, a.output) + import onnx + graph = onnx.load(a.output) + onnx.checker.check_model(graph) + report = {'onnx': a.output, 'sha256': hashlib.sha256(Path(a.output).read_bytes()).hexdigest(), + 'bytes': Path(a.output).stat().st_size, 'opset': graph.opset_import[0].version, + 'operators': sorted({n.op_type for n in graph.graph.node}), + 'parameters': int(sum(t.numel() for t in net.parameters()))} + if a.images: + report['parity'] = compare(net, a.output, a.images, a.sizes) + if a.benchmark: + report['cpu_single_thread'] = benchmark(a.output, a.sizes) + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/bdon_vision/locator.py b/bdon_vision/locator.py new file mode 100644 index 0000000..b8f644a --- /dev/null +++ b/bdon_vision/locator.py @@ -0,0 +1,199 @@ +"""Card-frame locator: input preparation, training targets and output decoding. + +The locator network predicts, on a stride-8 grid, one center heatmap per card +kind, the tile width and height in network-input pixels, and the sub-cell +offset of the tile center. It locates card tiles and their kind only; card +identity is resolved by a separate retrieval step. + +This module depends on NumPy only, so decoding can be reused and tested +without a deep-learning framework. ``Locator`` runs an exported ONNX model +with onnxruntime. +""" +import numpy as np + +CLASSES = ('member', 'snap') +# Tile width / height of each class in the native list layout. +TILE_ASPECT = {'member': 224 / 294, 'snap': 326 / 184} +STRIDE = 8 +MULTIPLE = 32 +PAD_VALUE = 128 +# Gaussian radius factor relative to the tile size (TTFNet-style elliptical +# targets). The regression area uses the same ellipse. +GAUSSIAN_ALPHA = .54 +REGRESSION_FLOOR = .05 + + +def letterbox(image, long_edge, multiple=MULTIPLE, pad_value=PAD_VALUE): + """Resize ``image`` (H, W, 3 uint8) so its long edge equals ``long_edge``. + + The resized image is placed at the top-left corner of a canvas whose sides + are rounded up to ``multiple``. Returns ``(canvas, scale)``; an input pixel + coordinate maps back to the original image by dividing by ``scale``. + """ + import cv2 + h, w = image.shape[:2] + scale = long_edge / max(h, w) + nw, nh = max(1, round(w * scale)), max(1, round(h * scale)) + resized = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_AREA if scale < 1 else cv2.INTER_LINEAR) + ch, cw = -(-nh // multiple) * multiple, -(-nw // multiple) * multiple + canvas = np.full((ch, cw, 3), pad_value, np.uint8) + canvas[:nh, :nw] = resized + return canvas, scale + + +def to_tensor(canvas): + """HWC uint8 RGB -> float32 [1, 3, H, W] in 0..1.""" + return np.ascontiguousarray(canvas.transpose(2, 0, 1)[None], dtype=np.float32) / 255. + + +def _gaussian_ellipse(width, height, alpha=GAUSSIAN_ALPHA): + return max(alpha * width / STRIDE / 6, .35), max(alpha * height / STRIDE / 6, .35) + + +def encode_targets(boxes, labels, ignore, shape, classes=len(CLASSES)): + """Build dense training targets on the stride-8 grid. + + ``boxes`` are tile rectangles ``[x, y, w, h]`` in input pixels, ``labels`` + their class indices and ``ignore`` rectangles where background loss is not + applied (for example tiles that are less than half visible). ``shape`` is + the input ``(H, W)``. + + Returns a dict with ``heatmap`` [C, h, w], ``loss_mask`` [h, w] (1 where the + heatmap loss applies), ``box`` [4, h, w] (target center x, center y, width, + height in input pixels) and ``weight`` [h, w] (regression weight, each + object's weights sum to one). + """ + gh, gw = shape[0] // STRIDE, shape[1] // STRIDE + heat = np.zeros((classes, gh, gw), np.float32) + mask = np.ones((gh, gw), np.float32) + box = np.zeros((4, gh, gw), np.float32) + weight = np.zeros((gh, gw), np.float32) + owner = np.zeros((gh, gw), np.float32) + ys, xs = np.mgrid[0:gh, 0:gw].astype(np.float32) + for x, y, w, h in np.asarray(ignore, np.float32).reshape(-1, 4): + x0, y0 = int(np.floor(x / STRIDE)), int(np.floor(y / STRIDE)) + x1, y1 = int(np.ceil((x + w) / STRIDE)), int(np.ceil((y + h) / STRIDE)) + mask[max(0, y0):max(0, y1), max(0, x0):max(0, x1)] = 0 + for (x, y, w, h), label in zip(np.asarray(boxes, np.float32).reshape(-1, 4), labels): + if w <= 0 or h <= 0: + continue + cx, cy = x + w / 2, y + h / 2 + px = min(max(int(np.floor(cx / STRIDE)), 0), gw - 1) + py = min(max(int(np.floor(cy / STRIDE)), 0), gh - 1) + sx, sy = _gaussian_ellipse(w, h) + g = np.exp(-((xs - px) ** 2 / (2 * sx * sx) + (ys - py) ** 2 / (2 * sy * sy))) + g[g < .01] = 0 + heat[label] = np.maximum(heat[label], g) + heat[label, py, px] = 1. + inside = (xs + .5 >= x / STRIDE) & (xs + .5 <= (x + w) / STRIDE) & (ys + .5 >= y / STRIDE) & (ys + .5 <= (y + h) / STRIDE) + region = (g > REGRESSION_FLOOR) & inside + region[py, px] = True + take = region & (g > owner) + owner[take] = g[take] + local = np.where(region, g, 0.) + local /= local.sum() + weight[take] = local[take] + box[0][take], box[1][take], box[2][take], box[3][take] = cx, cy, w, h + mask[heat.max(axis=0) > 0] = 1 + return {'heatmap': heat, 'loss_mask': mask, 'box': box, 'weight': weight} + + +def box_iou(a, b): + """IoU matrix between ``[N, 4]`` and ``[M, 4]`` xywh rectangles.""" + a = np.asarray(a, np.float64).reshape(-1, 4) + b = np.asarray(b, np.float64).reshape(-1, 4) + x0 = np.maximum(a[:, None, 0], b[None, :, 0]) + y0 = np.maximum(a[:, None, 1], b[None, :, 1]) + x1 = np.minimum(a[:, None, 0] + a[:, None, 2], b[None, :, 0] + b[None, :, 2]) + y1 = np.minimum(a[:, None, 1] + a[:, None, 3], b[None, :, 1] + b[None, :, 3]) + inter = np.clip(x1 - x0, 0, None) * np.clip(y1 - y0, 0, None) + union = a[:, None, 2] * a[:, None, 3] + b[None, :, 2] * b[None, :, 3] - inter + return inter / np.maximum(union, 1e-9) + + +def _overlap_smaller(a, b): + x0, y0 = max(a[0], b[0]), max(a[1], b[1]) + x1, y1 = min(a[0] + a[2], b[0] + b[2]), min(a[1] + a[3], b[1] + b[3]) + inter = max(0., x1 - x0) * max(0., y1 - y0) + return inter / max(min(a[2] * a[3], b[2] * b[3]), 1e-9) + + +def find_peaks(heatmap, threshold, max_peaks=300): + """Local maxima (3x3) of a [C, h, w] heatmap at or above ``threshold``. + + Returns arrays ``(classes, rows, cols, scores)`` sorted by descending score. + """ + c, h, w = heatmap.shape + padded = np.pad(heatmap, ((0, 0), (1, 1), (1, 1)), constant_values=-1.) + neighborhood = np.max(np.stack([padded[:, dy:dy + h, dx:dx + w] for dy in range(3) for dx in range(3)]), axis=0) + keep = (heatmap >= neighborhood) & (heatmap >= threshold) + cls, rows, cols = np.nonzero(keep) + scores = heatmap[cls, rows, cols] + order = np.argsort(-scores, kind='stable')[:max_peaks] + return cls[order], rows[order], cols[order], scores[order] + + +def visible_fraction(rect, image_size): + """Share of an [x, y, w, h] rectangle inside a (width, height) image.""" + x, y, w, h = rect + inside = max(0., min(image_size[0], x + w) - max(0., x)) * max(0., min(image_size[1], y + h) - max(0., y)) + return inside / max(w * h, 1e-9) + + +def decode(heatmap, size, offset, scale=1., threshold=.4, overlap=.5, max_detections=200, + image_size=None, min_visible=.5, aspect_tolerance=None): + """Turn raw network outputs (batch removed) into card detections. + + ``heatmap`` [C, h, w] is sigmoid-activated, ``size`` [2, h, w] holds tile + width and height in input pixels, ``offset`` [2, h, w] holds the tile center + relative to the cell origin in cells. Coordinates are divided by ``scale`` + to return to the original image. + + With ``image_size`` = (width, height), tiles less than ``min_visible`` + inside the image are dropped, matching the positive-tile definition. With + ``aspect_tolerance`` t, tiles whose width / height differs from the class + aspect by more than a factor 1 + t are dropped. Remaining detections whose + intersection covers more than ``overlap`` of the smaller box are + suppressed across classes, since card tiles never overlap on screen. + """ + cls, rows, cols, scores = find_peaks(heatmap, threshold) + out = [] + for k, i, j, s in zip(cls, rows, cols, scores): + w, h = float(size[0, i, j]), float(size[1, i, j]) + cx, cy = (j + float(offset[0, i, j])) * STRIDE, (i + float(offset[1, i, j])) * STRIDE + rect = [(cx - w / 2) / scale, (cy - h / 2) / scale, w / scale, h / scale] + kind = CLASSES[int(k)] + if image_size is not None and visible_fraction(rect, image_size) < min_visible: + continue + if aspect_tolerance is not None: + ratio = w / max(h, 1e-9) / TILE_ASPECT[kind] + if not 1 / (1 + aspect_tolerance) <= ratio <= 1 + aspect_tolerance: + continue + if any(_overlap_smaller(rect, d['bbox']) > overlap for d in out): + continue + out.append({'kind': kind, 'score': float(s), 'bbox': rect}) + if len(out) >= max_detections: + break + return out + + +class Locator: + """CPU onnxruntime wrapper around an exported locator model.""" + + def __init__(self, model, long_edge=640, threads=1, threshold=.4, aspect_tolerance=None): + import onnxruntime as ort + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + self.session = ort.InferenceSession(str(model), sess_options=options, providers=['CPUExecutionProvider']) + self.long_edge, self.threshold, self.aspect_tolerance = long_edge, threshold, aspect_tolerance + + def raw(self, image_rgb): + canvas, scale = letterbox(image_rgb, self.long_edge) + heat, size, offset = self.session.run(['heatmap', 'size', 'offset'], {'image': to_tensor(canvas)}) + return heat[0], size[0], offset[0], scale + + def __call__(self, image_rgb, threshold=None): + heat, size, offset, scale = self.raw(image_rgb) + return decode(heat, size, offset, scale, self.threshold if threshold is None else threshold, + image_size=image_rgb.shape[1::-1], aspect_tolerance=self.aspect_tolerance) diff --git a/bdon_vision/locator_eval.py b/bdon_vision/locator_eval.py new file mode 100644 index 0000000..5359e13 --- /dev/null +++ b/bdon_vision/locator_eval.py @@ -0,0 +1,218 @@ +"""Detection metrics for card-frame locators and scene truth loading. + +Scene truth follows the ``bdon-synthetic/2`` layout: each screenshot lists +``cards`` (tile ``bbox`` = [x, y, w, h], ``kind``, ``visible_fraction``), +optional ``foreign_cards`` (tiles whose artwork is outside the reference +gallery; counted as positives), ``ignored_cards`` and +``ignored_foreign_cards`` (less than half visible; neither positive nor +negative) and optional ``ui_elements`` (non-card interface rectangles, used +to attribute false positives). +""" +import json +from collections import defaultdict +from pathlib import Path + +import numpy as np + +from .locator import CLASSES, box_iou + +WIDTH_BINS = (0, 40, 56, 72, 96, 128, 192, 100000) + + +def _cluster(values, tolerance): + groups = [] + for v in sorted(values): + if groups and abs(v - groups[-1][-1]) < tolerance: + groups[-1].append(v) + else: + groups.append([v]) + return [float(np.median(g)) for g in groups] + + +def grid_ignore_regions(boxes, width, height): + """Grid cells that are 0-50% visible, inferred from a regular card grid. + + Used when truth lists partially visible cards without rectangles. Cells + that could hold such a card are excluded from both loss and metrics. + """ + if not len(boxes): + return [[0., 0., float(width), float(height)]] + b = np.asarray(boxes, np.float64) + w, h = np.median(b[:, 2]), np.median(b[:, 3]) + xs, ys = _cluster(b[:, 0], .3 * w), _cluster(b[:, 1], .3 * h) + px = min(np.diff(xs)) if len(xs) > 1 else 1.11 * w + py = min(np.diff(ys)) if len(ys) > 1 else 1.10 * h + cols = np.arange(np.floor((-w - xs[0]) / px), np.ceil((width - xs[0]) / px) + 1) * px + xs[0] + rows = np.arange(np.floor((-h - ys[0]) / py), np.ceil((height - ys[0]) / py) + 1) * py + ys[0] + out = [] + for y in rows: + for x in cols: + vis = max(0, min(width, x + w) - max(0, x)) * max(0, min(height, y + h) - max(0, y)) / (w * h) + if 0 < vis < .5: + out.append([float(x), float(y), float(w), float(h)]) + return out + + +def load_scenes(directories): + """Read every ``truth.json`` under the given chunk directories.""" + scenes = [] + for directory in directories: + directory = Path(directory) + truth = json.loads((directory / 'truth.json').read_text(encoding='utf-8')) + if not truth.get('complete', False): + raise ValueError(f'incomplete truth: {directory}') + for row in truth['screenshots']: + acquisition = row.get('acquisition', {}) + cards = [dict(c, foreign=False) for c in row.get('cards', [])] + cards += [dict(c, foreign=True) for c in row.get('foreign_cards', [])] + size = acquisition.get('output_resolution') + scenes.append({'path': str(directory / row['file']), 'chunk': directory.name, + 'boxes': np.asarray([c['bbox'] for c in cards], np.float32).reshape(-1, 4), + 'labels': np.asarray([CLASSES.index(c['kind']) for c in cards], np.int64), + 'foreign': np.asarray([c['foreign'] for c in cards], bool), + 'visible': np.asarray([c.get('visible_fraction', 1.) for c in cards], np.float32), + 'ignored_rows': row.get('ignored_cards', []) + row.get('ignored_foreign_cards', []), + 'ui': np.asarray([e['bbox'] for e in row.get('ui_elements', [])], np.float32).reshape(-1, 4), + 'negative': bool(row.get('negative', False)), 'size': size, + 'source_resolution': acquisition.get('source_resolution', size), + 'profile': acquisition.get('profile', truth.get('profile'))}) + return scenes + + +def resolve_ignore(scene, width, height): + rows = scene['ignored_rows'] + boxes = [r['bbox'] for r in rows if 'bbox' in r] + if any('bbox' not in r for r in rows): + boxes += grid_ignore_regions(scene['boxes'], width, height) + return np.asarray(boxes, np.float32).reshape(-1, 4) + + +def _cover(pred, regions): + """Largest fraction of each predicted box covered by one of ``regions``.""" + if not len(regions) or not len(pred): + return np.zeros(len(pred)) + x0 = np.maximum(pred[:, None, 0], regions[None, :, 0]) + y0 = np.maximum(pred[:, None, 1], regions[None, :, 1]) + x1 = np.minimum(pred[:, None, 0] + pred[:, None, 2], regions[None, :, 0] + regions[None, :, 2]) + y1 = np.minimum(pred[:, None, 1] + pred[:, None, 3], regions[None, :, 1] + regions[None, :, 3]) + inter = np.clip(x1 - x0, 0, None) * np.clip(y1 - y0, 0, None) + return inter.max(axis=1) / np.maximum(pred[:, 2] * pred[:, 3], 1e-9) + + +def match(detections, boxes, labels, ignore, ui=()): + """Greedy score-ordered, class-agnostic matching at IoU 0.5 and 0.75. + + Returns per-detection rows and per-truth rows. Because matching visits + detections in score order, the result at any higher score threshold is the + restriction of this result. + """ + order = sorted(range(len(detections)), key=lambda i: -detections[i]['score']) + pred = np.asarray([detections[i]['bbox'] for i in order], np.float64).reshape(-1, 4) + kinds = [CLASSES.index(detections[i]['kind']) for i in order] + iou = box_iou(pred, boxes) if len(boxes) else np.zeros((len(pred), 0)) + cover = _cover(pred, np.asarray(ignore, np.float64).reshape(-1, 4)) + # A box mostly inside a UI element, or one overlapping a UI element. + on_ui = np.zeros(len(pred), bool) + ui = np.asarray(ui, np.float64).reshape(-1, 4) + if len(ui) and len(pred): + on_ui = (_cover(pred, ui) >= .3) | (box_iou(pred, ui).max(axis=1) >= .3) + rows = [] + truth = {t: [None] * len(boxes) for t in (.5, .75)} + for p in range(len(pred)): + row = {'score': detections[order[p]]['score'], 'kind': kinds[p], 'on_ui': bool(on_ui[p])} + for t in (.5, .75): + free = [g for g in range(len(boxes)) if truth[t][g] is None and iou[p, g] >= t] + if free: + g = max(free, key=lambda g: iou[p, g]) + truth[t][g] = (row['score'], kinds[p] == int(labels[g])) + row[t] = 'tp' if kinds[p] == int(labels[g]) else 'tp_wrong_kind' + elif cover[p] >= .5: + row[t] = 'ignored' + else: + row[t] = 'fp' + rows.append(row) + return rows, truth + + +class Accumulator: + """Collects matching results and reports metrics at a score threshold.""" + + def __init__(self): + self.preds, self.truth = [], [] + + def add(self, detections, scene, width, height, input_scale=1.): + boxes, labels = scene['boxes'], scene['labels'] + rows, truth = match(detections, boxes, labels, resolve_ignore(scene, width, height), scene.get('ui', ())) + resolution = f'{width}x{height}' + long_edge = max(width, height) + image_keys = ['overall', f'output_long_edge:{int(long_edge // 480 * 480)}', + f'source_resolution:{scene["source_resolution"]}'] + if scene.get('negative'): + image_keys = image_keys + ['screen:card_free'] + for row in rows: + # Predictions also count under their predicted kind for per-kind precision. + row['keys'] = image_keys + [f'kind:{CLASSES[row["kind"]]}'] + self.preds.append(row) + for g in range(len(boxes)): + w_input = float(boxes[g][2]) * input_scale + bin_index = int(np.searchsorted(WIDTH_BINS, w_input, side='right')) - 1 + keys = image_keys + [f'kind:{CLASSES[int(labels[g])]}', + f'input_width:{WIDTH_BINS[bin_index]}-{WIDTH_BINS[bin_index + 1]}', + 'artwork:' + ('foreign' if scene['foreign'][g] else 'gallery')] + self.truth.append({'keys': keys, .5: truth[.5][g], .75: truth[.75][g]}) + return rows, truth, resolution + + def summary(self, threshold, strata=True): + out = {} + groups = defaultdict(lambda: defaultdict(int)) + for row in self.preds: + if row['score'] < threshold: + continue + for key in row['keys']: + for t in (.5, .75): + groups[key][f'{row[t]}@{t}'] += 1 + groups[key][f'fp_on_ui@{t}'] += int(row[t] == 'fp' and row.get('on_ui', False)) + for row in self.truth: + for key in row['keys']: + groups[key]['truth'] += 1 + for t in (.5, .75): + m = row[t] + if m is not None and m[0] >= threshold: + groups[key][f'recalled@{t}'] += 1 + groups[key][f'kind_correct@{t}'] += int(m[1]) + for key, g in groups.items(): + item = {'truth': g['truth']} + for t in (.5, .75): + tp = g[f'tp@{t}'] + g[f'tp_wrong_kind@{t}'] + fp = g[f'fp@{t}'] + item[f'iou{t}'] = {'recall': g[f'recalled@{t}'] / max(g['truth'], 1), + 'precision': tp / max(tp + fp, 1), 'false_positives': fp, + 'false_positives_on_ui': g[f'fp_on_ui@{t}'], + 'kind_accuracy': g[f'kind_correct@{t}'] / max(g[f'recalled@{t}'], 1), + 'recalled': g[f'recalled@{t}'], 'kind_correct': g[f'kind_correct@{t}']} + out[key] = item + if not strata: + return out.get('overall') + return dict(sorted(out.items())) + + def average_precision(self, t): + rows = sorted(self.preds, key=lambda r: -r['score']) + positives = len(self.truth) + tp = np.cumsum([r[t] in ('tp', 'tp_wrong_kind') for r in rows if r[t] != 'ignored']) + fp = np.cumsum([r[t] == 'fp' for r in rows if r[t] != 'ignored']) + if not len(tp) or not positives: + return 0. + recall = tp / positives + precision = tp / np.maximum(tp + fp, 1) + precision = np.maximum.accumulate(precision[::-1])[::-1] + grid = np.linspace(0, 1, 101) + idx = np.searchsorted(recall, grid, side='left') + return float(np.mean([precision[i] if i < len(precision) else 0. for i in idx])) + + def best_threshold(self, t=.5): + best = (-1, .3) + for threshold in np.arange(.1, .91, .02): + s = self.summary(float(threshold), strata=False)[f'iou{t}'] + f1 = 2 * s['recall'] * s['precision'] / max(s['recall'] + s['precision'], 1e-9) + best = max(best, (f1, round(float(threshold), 2))) + return best[1] diff --git a/bdon_vision/locator_model.py b/bdon_vision/locator_model.py new file mode 100644 index 0000000..18778de --- /dev/null +++ b/bdon_vision/locator_model.py @@ -0,0 +1,83 @@ +"""CenterNet-style card-frame locator network (MobileNetV3-Large + light FPN).""" +import math + +import torch +from torch import nn +from torch.nn import functional as F + +from .locator import CLASSES, STRIDE + + +def _conv_bn(cin, cout, kernel=1, groups=1): + return nn.Sequential(nn.Conv2d(cin, cout, kernel, padding=kernel // 2, groups=groups, bias=False), + nn.BatchNorm2d(cout), nn.ReLU(inplace=True)) + + +def _separable(channels): + return nn.Sequential(_conv_bn(channels, channels, 3, groups=channels), _conv_bn(channels, channels, 1)) + + +def _head(channels, outputs, bias=0.): + layer = nn.Conv2d(channels, outputs, 1) + nn.init.normal_(layer.weight, std=.01) + nn.init.constant_(layer.bias, bias) + return nn.Sequential(_conv_bn(channels, channels, 3, groups=channels), _conv_bn(channels, channels, 1), layer) + + +class LocatorNet(nn.Module): + """Stride-8 dense predictor. + + ``forward`` returns raw training outputs: heatmap logits [N, C, h, w], + log tile size in cells [N, 2, h, w] and center offset in cells + [N, 2, h, w]. ``ExportedLocator`` converts them to the deployment format. + """ + + def __init__(self, width=96, backbone_weights=None): + super().__init__() + from torchvision.models import mobilenet_v3_large + backbone = mobilenet_v3_large(weights=None) + if backbone_weights: + backbone.load_state_dict(torch.load(backbone_weights, map_location='cpu', weights_only=True)) + features = backbone.features + self.stem, self.c3, self.c4, self.c5 = features[:4], features[4:7], features[7:13], features[13:16] + self.l3, self.l4, self.l5 = _conv_bn(40, width), _conv_bn(112, width), _conv_bn(160, width) + self.s4, self.s3 = _separable(width), _separable(width) + self.context = nn.Sequential(_separable(width), _separable(width)) + self.heat = _head(width, len(CLASSES), bias=-math.log((1 - .05) / .05)) + self.size = _head(width, 2, bias=2.) + self.offset = _head(width, 2, bias=.5) + self.register_buffer('mean', torch.tensor([.485, .456, .406]).view(1, 3, 1, 1)) + self.register_buffer('std', torch.tensor([.229, .224, .225]).view(1, 3, 1, 1)) + + def forward(self, x): + x = self.stem((x - self.mean) / self.std) + c3 = self.c3(x) + c4 = self.c4(c3) + c5 = self.c5(c4) + p4 = self.s4(self.l4(c4) + F.interpolate(self.l5(c5), scale_factor=2., mode='nearest')) + p3 = self.s3(self.l3(c3) + F.interpolate(p4, scale_factor=2., mode='nearest')) + p3 = p3 + self.context(p3) + return self.heat(p3), self.size(p3), self.offset(p3) + + +class ExportedLocator(nn.Module): + """Deployment outputs: sigmoid heatmap, tile size in input pixels, offset in cells.""" + + def __init__(self, net): + super().__init__() + self.net = net + + def forward(self, image): + heat, size, offset = self.net(image) + return torch.sigmoid(heat), torch.exp(size.clamp(max=8.)) * STRIDE, offset + + +def export_onnx(net, path, height=384, width=640, opset=17): + model = ExportedLocator(net).eval().cpu() + # Batch is fixed to one image; height and width are any multiple of 32. + dims = {2: 'height', 3: 'width'} + out_dims = {2: 'grid_height', 3: 'grid_width'} + torch.onnx.export(model, torch.zeros(1, 3, height, width), str(path), input_names=['image'], + output_names=['heatmap', 'size', 'offset'], + dynamic_axes={'image': dims, 'heatmap': out_dims, 'size': out_dims, 'offset': out_dims}, + opset_version=opset, dynamo=False) diff --git a/bdon_vision/train_locator.py b/bdon_vision/train_locator.py new file mode 100644 index 0000000..fc8dcb9 --- /dev/null +++ b/bdon_vision/train_locator.py @@ -0,0 +1,378 @@ +"""Train the card-frame locator on synthetic scene sets. + +Example:: + + python -m bdon_vision.train_locator --train 'scenes/train-*' \ + --validation 'scenes/validation-*' --output runs/locator --steps 12000 + +Scenes are decoded once into memory (long edge capped), then sampled with +scale, crop, two-scene mosaic, color, blur, noise and JPEG augmentation. +""" +import argparse +import glob +import json +import math +import random +import time +from multiprocessing import Pool +from pathlib import Path + +import cv2 +import numpy as np +import torch +from torch.nn import functional as F + +from .locator import CLASSES, PAD_VALUE, STRIDE, decode, encode_targets, letterbox +from .locator_eval import Accumulator, load_scenes, resolve_ignore +from .locator_model import LocatorNet, export_onnx + +CACHE_EDGE = 1280 + + +def _decode_scene(args): + path, edge, jpeg = args + image = cv2.imread(path, cv2.IMREAD_COLOR) + if image is None: + raise ValueError(f'unreadable image: {path}') + h, w = image.shape[:2] + scale = min(1., edge / max(h, w)) + if scale < 1: + image = cv2.resize(image, (round(w * scale), round(h * scale)), interpolation=cv2.INTER_AREA) + if jpeg: + # Large sets are held as high-quality JPEG bytes to bound memory. + return cv2.imencode('.jpg', image, [cv2.IMWRITE_JPEG_QUALITY, jpeg])[1], (w, h), scale + return cv2.cvtColor(image, cv2.COLOR_BGR2RGB), (w, h), scale + + +def scene_image(scene): + """Cached RGB image of a scene (long edge capped, see ``cache_scale``).""" + image = scene['image'] + if image.ndim == 1: + return cv2.cvtColor(cv2.imdecode(image, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + return image + + +def cache_scenes(scenes, edge=CACHE_EDGE, workers=8, jpeg=0): + with Pool(workers) as pool: + decoded = pool.map(_decode_scene, [(s['path'], edge, jpeg) for s in scenes], chunksize=8) + for scene, (image, (w, h), scale) in zip(scenes, decoded): + scene['image'], scene['width'], scene['height'], scene['cache_scale'] = image, w, h, scale + scene['ignore'] = resolve_ignore(scene, w, h) + return scenes + + +def _visible(boxes, rect): + x0 = np.maximum(boxes[:, 0], rect[0]) + y0 = np.maximum(boxes[:, 1], rect[1]) + x1 = np.minimum(boxes[:, 0] + boxes[:, 2], rect[0] + rect[2]) + y1 = np.minimum(boxes[:, 1] + boxes[:, 3], rect[1] + rect[3]) + return np.clip(x1 - x0, 0, None) * np.clip(y1 - y0, 0, None) / np.maximum(boxes[:, 2] * boxes[:, 3], 1e-9) + + +class SceneSampler(torch.utils.data.Dataset): + """Random augmented square views; every item is independently sampled.""" + + def __init__(self, scenes, size=512, length=10 ** 7, edge_range=(448, 832), mosaic=.3, seed=0): + self.scenes, self.size, self.length = scenes, size, length + self.edge_range, self.mosaic, self.seed = edge_range, mosaic, seed + + def __len__(self): + return self.length + + def view(self, rng): + scene = self.scenes[rng.randrange(len(self.scenes))] + image = scene_image(scene) + s = self.size + long_edge = rng.uniform(*self.edge_range) + r = rng.random() + if r < .3: + long_edge *= rng.uniform(1., 2.6) + elif r < .4: + long_edge *= rng.uniform(.65, 1.) + scale = long_edge / max(scene['width'], scene['height']) + k = scale / scene['cache_scale'] + nh, nw = max(1, round(image.shape[0] * k)), max(1, round(image.shape[1] * k)) + # Crop before resizing when enlarging, to keep the work bounded. + ox = rng.uniform(0, max(0., nw - s)) if nw > s else -rng.uniform(0, s - nw) * (rng.random() < .5) + oy = rng.uniform(0, max(0., nh - s)) if nh > s else -rng.uniform(0, s - nh) * (rng.random() < .5) + src = (max(0, int(ox / k)), max(0, int(oy / k))) + src_end = (min(image.shape[1], int(math.ceil((ox + s) / k)) + 1), min(image.shape[0], int(math.ceil((oy + s) / k)) + 1)) + patch = image[src[1]:src_end[1], src[0]:src_end[0]] + pw, ph = max(1, round(patch.shape[1] * k)), max(1, round(patch.shape[0] * k)) + # Mixed resampling keeps the model robust to browser canvas scaling. + interp = rng.choice([cv2.INTER_AREA, cv2.INTER_AREA, cv2.INTER_LINEAR, cv2.INTER_CUBIC] if k < 1 else [cv2.INTER_LINEAR, cv2.INTER_CUBIC]) + patch = cv2.resize(patch, (pw, ph), interpolation=interp) + canvas = np.full((s, s, 3), PAD_VALUE, np.uint8) + dx, dy = round(src[0] * k - ox), round(src[1] * k - oy) + x0, y0 = max(0, dx), max(0, dy) + x1, y1 = min(s, dx + pw), min(s, dy + ph) + if x1 > x0 and y1 > y0: + canvas[y0:y1, x0:x1] = patch[y0 - dy:y1 - dy, x0 - dx:x1 - dx] + shift = np.array([-ox, -oy, 0, 0], np.float32) + boxes = scene['boxes'] * scale + shift + ignore = scene['ignore'] * scale + shift + content = [-ox, -oy, scene['width'] * scale, scene['height'] * scale] + # Share of each tile that is unoccluded within the screenshot bounds. + occlusion = np.minimum(scene['visible'] / np.maximum(_visible(boxes, content), 1e-6), 1.) if len(boxes) else scene['visible'] + return canvas, boxes, scene['labels'], ignore, content, occlusion + + def split(self, a, b, rng): + """Join two views along a vertical or horizontal seam.""" + s = self.size + cut = round(rng.uniform(.3, .7) * s) + vertical = rng.random() < .5 + canvas = a[0].copy() + regions = ([0, 0, cut, s], [cut, 0, s - cut, s]) if vertical else ([0, 0, s, cut], [0, cut, s, s - cut]) + rx, ry, rw, rh = regions[1] + canvas[ry:ry + rh, rx:rx + rw] = b[0][ry:ry + rh, rx:rx + rw] + parts = [] + for view, region in zip((a, b), regions): + _, boxes, labels, ignore, content, occlusion = view + cx0, cy0 = max(content[0], region[0]), max(content[1], region[1]) + cx1 = min(content[0] + content[2], region[0] + region[2]) + cy1 = min(content[1] + content[3], region[1] + region[3]) + parts.append((boxes, labels, ignore, [cx0, cy0, max(0, cx1 - cx0), max(0, cy1 - cy0)], occlusion)) + return canvas, parts + + def photometric(self, image, rng): + x = image.astype(np.float32) + if rng.random() < .8: + x = (x - 128) * rng.uniform(.75, 1.3) + 128 + rng.uniform(-30, 30) + if rng.random() < .5: + gray = x.mean(axis=2, keepdims=True) + x = gray + (x - gray) * rng.uniform(.5, 1.4) + if rng.random() < .3: + x = x * np.array([rng.uniform(.88, 1.12) for _ in range(3)], np.float32) + x = np.clip(x, 0, 255) + if rng.random() < .25: + x = cv2.GaussianBlur(x, (0, 0), rng.uniform(.3, 1.3)) + if rng.random() < .25: + x = x + np.random.default_rng(rng.randrange(2 ** 31)).normal(0, rng.uniform(2, 8), x.shape).astype(np.float32) + x = np.clip(x, 0, 255).astype(np.uint8) + if rng.random() < .6: + ok, buf = cv2.imencode('.jpg', x, [cv2.IMWRITE_JPEG_QUALITY, rng.randint(28, 95)]) + x = cv2.imdecode(buf, cv2.IMREAD_UNCHANGED) + return x + + def __getitem__(self, index): + rng = random.Random(self.seed * 1000003 + index) + first = self.view(rng) + if rng.random() < self.mosaic: + canvas, parts = self.split(first, self.view(rng), rng) + else: + canvas = first[0] + parts = [first[1:]] + boxes, labels, ignore = [], [], [] + for b, l, ig, content, occlusion in parts: + vis = _visible(b, content) * occlusion if len(b) else np.zeros(0) + keep = vis >= .5 + boxes.append(b[keep]) + labels.append(l[keep]) + ignore.append(b[(vis > 0) & ~keep]) + if len(ig): + ignore.append(ig[_visible(ig, content) > 0]) + canvas = self.photometric(canvas, rng) + target = encode_targets(np.concatenate(boxes), np.concatenate(labels), np.concatenate(ignore), canvas.shape[:2]) + x = torch.from_numpy(canvas).permute(2, 0, 1).contiguous() + return (x, torch.from_numpy(target['heatmap']), torch.from_numpy(target['loss_mask']), + torch.from_numpy(target['box']), torch.from_numpy(target['weight'])) + + +def focal_loss(logits, target, mask): + p = torch.sigmoid(logits).clamp(1e-4, 1 - 1e-4) + pos = target.eq(1).float() + neg = (1 - pos) * mask[:, None] + pos_loss = -torch.log(p) * (1 - p) ** 2 * pos + neg_loss = -torch.log(1 - p) * p ** 2 * (1 - target) ** 4 * neg + return (pos_loss.sum() + neg_loss.sum()) / pos.sum().clamp(min=1) + + +def regression_loss(size, offset, box, weight): + n, _, h, w = size.shape + ys = torch.arange(h, device=size.device, dtype=torch.float32).view(1, h, 1) + xs = torch.arange(w, device=size.device, dtype=torch.float32).view(1, 1, w) + valid = weight > 0 + if not valid.any(): + return size.sum() * 0, size.sum() * 0 + tw, th = box[:, 2].clamp(min=1), box[:, 3].clamp(min=1) + tox, toy = box[:, 0] / STRIDE - xs, box[:, 1] / STRIDE - ys + tlw, tlh = torch.log(tw / STRIDE), torch.log(th / STRIDE) + l1 = ((size[:, 0] - tlw).abs() + (size[:, 1] - tlh).abs() + (offset[:, 0] - tox).abs() + (offset[:, 1] - toy).abs()) + pw, ph = torch.exp(size[:, 0].clamp(max=8)) * STRIDE, torch.exp(size[:, 1].clamp(max=8)) * STRIDE + pcx, pcy = (xs + offset[:, 0]) * STRIDE, (ys + offset[:, 1]) * STRIDE + px0, py0, px1, py1 = pcx - pw / 2, pcy - ph / 2, pcx + pw / 2, pcy + ph / 2 + tx0, ty0, tx1, ty1 = box[:, 0] - tw / 2, box[:, 1] - th / 2, box[:, 0] + tw / 2, box[:, 1] + th / 2 + inter = (torch.min(px1, tx1) - torch.max(px0, tx0)).clamp(min=0) * (torch.min(py1, ty1) - torch.max(py0, ty0)).clamp(min=0) + union = pw * ph + tw * th - inter + hull = (torch.max(px1, tx1) - torch.min(px0, tx0)) * (torch.max(py1, ty1) - torch.min(py0, ty0)) + giou = inter / union - (hull - union) / hull.clamp(min=1e-6) + objects = weight.sum().clamp(min=1) + return (l1 * weight)[valid].sum() / objects, ((1 - giou) * weight)[valid].sum() / objects + + +def predict(model, image_rgb, long_edge, threshold=.05, device='cuda'): + canvas, scale = letterbox(image_rgb, long_edge) + x = torch.from_numpy(canvas).permute(2, 0, 1)[None].to(device).float().div_(255) + with torch.inference_mode(): + heat, size, offset = model(x) + heat = torch.sigmoid(heat.float())[0].cpu().numpy() + size = (torch.exp(size.float().clamp(max=8)) * STRIDE)[0].cpu().numpy() + offset = offset.float()[0].cpu().numpy() + return decode(heat, size, offset, scale, threshold, image_size=image_rgb.shape[1::-1]), scale + + +def evaluate_model(model, scenes, long_edge, device='cuda'): + was_training = model.training + model.eval() + acc = Accumulator() + for scene in scenes: + detections, _ = predict(model, scene_image(scene), long_edge, device=device) + for d in detections: + d['bbox'] = [v / scene['cache_scale'] for v in d['bbox']] + acc.add(detections, scene, scene['width'], scene['height'], long_edge / max(scene['width'], scene['height'])) + model.train(was_training) + return acc + + +def expand(patterns): + out = [] + for pattern in patterns: + found = sorted(p for p in glob.glob(pattern) if Path(p, 'truth.json').exists()) + if not found: + raise ValueError(f'no scene chunks match {pattern}') + out += found + return out + + +def _plain(value): + if isinstance(value, np.generic): + return value.item() + if isinstance(value, np.ndarray): + return value.tolist() + raise TypeError(f'not serializable: {type(value).__name__}') + + +def write_json(path, value): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(value, indent=2, default=_plain), encoding='utf-8') + tmp.replace(path) + + +def train(args): + torch.manual_seed(args.seed) + random.seed(args.seed) + np.random.seed(args.seed) + out = Path(args.output) + out.mkdir(parents=True, exist_ok=True) + started = time.time() + train_scenes = cache_scenes(load_scenes(expand(args.train)), workers=args.workers, jpeg=args.cache_jpeg) + valid_scenes = load_scenes(expand(args.validation)) if args.validation else [] + if args.validation_limit and len(valid_scenes) > args.validation_limit: + step = len(valid_scenes) / args.validation_limit + valid_scenes = [valid_scenes[int(i * step)] for i in range(args.validation_limit)] + valid_scenes = cache_scenes(valid_scenes, workers=args.workers, jpeg=args.cache_jpeg) if valid_scenes else [] + print(json.dumps({'train_scenes': len(train_scenes), 'validation_scenes': len(valid_scenes), + 'cache_s': round(time.time() - started, 1)}), flush=True) + sampler = SceneSampler(train_scenes, size=args.crop, length=args.steps * args.batch, seed=args.seed) + device = torch.device(args.device) + cuda = device.type == 'cuda' + if args.threads: + torch.set_num_threads(args.threads) + loader = torch.utils.data.DataLoader(sampler, batch_size=args.batch, shuffle=False, num_workers=args.workers, + pin_memory=cuda, drop_last=True, persistent_workers=True, prefetch_factor=4) + model = LocatorNet(width=args.width, backbone_weights=args.backbone).to(device).to(memory_format=torch.channels_last) + if args.initial: + model.load_state_dict(torch.load(args.initial, map_location='cpu', weights_only=True)) + backbone = [p for n, p in model.named_parameters() if n.startswith(('stem', 'c3', 'c4', 'c5'))] + rest = [p for n, p in model.named_parameters() if not n.startswith(('stem', 'c3', 'c4', 'c5'))] + optimizer = torch.optim.AdamW([{'params': backbone, 'lr': args.lr * args.backbone_lr}, + {'params': rest, 'lr': args.lr}], weight_decay=args.weight_decay) + base = [g['lr'] for g in optimizer.param_groups] + history, best = [], None + t0 = time.time() + for step, (x, heat, mask, box, weight) in enumerate(loader, 1): + frac = step / args.steps + factor = min(1., step / args.warmup) * (.5 * (1 + math.cos(math.pi * frac)) * .98 + .02) + for g, lr in zip(optimizer.param_groups, base): + g['lr'] = lr * factor + x = x.to(device, non_blocking=True).float().div_(255).contiguous(memory_format=torch.channels_last) + heat, mask = heat.to(device, non_blocking=True), mask.to(device, non_blocking=True) + box, weight = box.to(device, non_blocking=True), weight.to(device, non_blocking=True) + with torch.autocast(device.type, dtype=torch.bfloat16, enabled=cuda and args.bf16): + logits, size, offset = model(x) + logits, size, offset = logits.float(), size.float(), offset.float() + lh = focal_loss(logits, heat, mask) + l1, giou = regression_loss(size, offset, box, weight) + loss = lh + args.l1_weight * l1 + args.giou_weight * giou + optimizer.zero_grad(set_to_none=True) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 10.) + optimizer.step() + if step % 50 == 0 or step == 1: + status = {'step': step, 'steps': args.steps, 'loss': round(float(loss.detach()), 4), + 'heat': round(float(lh.detach()), 4), 'l1': round(float(l1.detach()), 4), + 'giou': round(float(giou.detach()), 4), 'lr': optimizer.param_groups[1]['lr'], + 'elapsed_s': round(time.time() - t0, 1), + 'gpu_max_gib': round(torch.cuda.max_memory_allocated() / 2 ** 30, 2) if cuda else None} + write_json(out / 'progress.json', status) + print(json.dumps(status), flush=True) + if valid_scenes and (step % args.eval_every == 0 or step == args.steps): + acc = evaluate_model(model, valid_scenes, args.eval_edge, device) + s = acc.summary(.3, strata=False) + ap50, ap75 = acc.average_precision(.5), acc.average_precision(.75) + score = ap50 + ap75 + s['iou0.75']['recall'] * s['iou0.75']['precision'] + s['iou0.5']['recall'] * s['iou0.5']['precision'] + row = {'step': step, 'ap50': ap50, 'ap75': ap75, 'at_0.3': s, 'score': score} + history.append(row) + write_json(out / 'validation-history.json', history) + print(json.dumps({'validation': row}), flush=True) + torch.save(model.state_dict(), out / 'last.pt') + # Later checkpoints win ties: they were trained at a lower learning rate. + if best is None or score >= best['score']: + best = row + torch.save(model.state_dict(), out / 'best.pt') + if (out / 'STOP').exists(): + print(json.dumps({'stopped_at': step}), flush=True) + break + if not valid_scenes: + torch.save(model.state_dict(), out / 'best.pt') + torch.save(model.state_dict(), out / 'last.pt') + model.load_state_dict(torch.load(out / 'best.pt', map_location='cpu', weights_only=True)) + export_onnx(model.float().to(memory_format=torch.contiguous_format), out / 'locator.onnx') + write_json(out / 'training.json', {'args': vars(args), 'best': best, 'train_scenes': len(train_scenes), + 'validation_scenes': len(valid_scenes), 'elapsed_s': round(time.time() - started, 1), + 'classes': list(CLASSES), 'stride': STRIDE}) + print(json.dumps({'done': True, 'best': best}), flush=True) + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + p.add_argument('--train', nargs='+', required=True, help='Glob(s) of scene chunk directories') + p.add_argument('--validation', nargs='*', default=[]) + p.add_argument('--validation-limit', type=int, default=0, help='Evenly subsample validation scenes') + p.add_argument('--output', required=True) + p.add_argument('--backbone', help='torchvision MobileNetV3-Large ImageNet state dict') + p.add_argument('--initial', help='Locator state dict to continue from') + p.add_argument('--steps', type=int, default=12000) + p.add_argument('--batch', type=int, default=32) + p.add_argument('--crop', type=int, default=512) + p.add_argument('--width', type=int, default=96) + p.add_argument('--lr', type=float, default=2e-3) + p.add_argument('--backbone-lr', type=float, default=.5, help='Backbone learning-rate multiplier') + p.add_argument('--weight-decay', type=float, default=1e-4) + p.add_argument('--warmup', type=int, default=500) + p.add_argument('--l1-weight', type=float, default=.5) + p.add_argument('--giou-weight', type=float, default=2.) + p.add_argument('--eval-every', type=int, default=1000) + p.add_argument('--eval-edge', type=int, default=640) + p.add_argument('--workers', type=int, default=8) + p.add_argument('--cache-jpeg', type=int, default=0, help='Hold cached scenes as JPEG of this quality (0 = raw)') + p.add_argument('--seed', type=int, default=1) + p.add_argument('--device', default='cuda') + p.add_argument('--threads', type=int, default=0, help='CPU threads for PyTorch (0 = default)') + p.add_argument('--no-bf16', dest='bf16', action='store_false') + train(p.parse_args()) + + +if __name__ == '__main__': + main() diff --git a/tests/test_locator.py b/tests/test_locator.py new file mode 100644 index 0000000..4a2ecf3 --- /dev/null +++ b/tests/test_locator.py @@ -0,0 +1,130 @@ +"""Card-frame locator: target encoding, decoding and metrics on synthetic inputs.""" +import importlib.util +import unittest + +import numpy as np + +from bdon_vision.locator import CLASSES, STRIDE, box_iou, decode, encode_targets, find_peaks, letterbox +from bdon_vision.locator_eval import Accumulator, grid_ignore_regions, match + + +def outputs_from_targets(target): + """Ideal network outputs for the given dense targets.""" + heat = target['heatmap'] + _, h, w = heat.shape + ys, xs = np.mgrid[0:h, 0:w].astype(np.float32) + box = target['box'] + size = np.stack([box[2], box[3]]) + offset = np.stack([box[0] / STRIDE - xs, box[1] / STRIDE - ys]) + return heat, size, offset + + +class LetterboxTests(unittest.TestCase): + def test_long_edge_and_padding(self): + image = np.zeros((290, 500, 3), np.uint8) + canvas, scale = letterbox(image, 640) + self.assertEqual(canvas.shape, (384, 640, 3)) + self.assertAlmostEqual(scale, 640 / 500) + self.assertTrue((canvas[384 - 1, 0] == 128).all()) + + +class TargetRoundTripTests(unittest.TestCase): + def test_decode_recovers_encoded_tiles(self): + boxes = np.array([[37.5, 40.25, 74, 97], [140, 44, 74, 97], [20, 200, 110, 62], [150.5, 210, 110, 62]], np.float32) + labels = np.array([0, 0, 1, 1]) + target = encode_targets(boxes, labels, np.zeros((0, 4)), (320, 288)) + self.assertEqual(target['heatmap'].shape, (len(CLASSES), 40, 36)) + self.assertEqual(int((target['heatmap'] == 1).sum()), 4) + for label, (x, y, w, h) in zip(labels, boxes): + self.assertAlmostEqual(target['weight'][(target['box'][0] == x + w / 2) & (target['box'][1] == y + h / 2)].sum(), 1., places=5) + detections = decode(*outputs_from_targets(target), scale=2., threshold=.5) + self.assertEqual(len(detections), 4) + recovered = sorted((d['kind'], *np.round(d['bbox'], 4)) for d in detections) + expected = sorted((CLASSES[k], *np.round(b / 2, 4)) for k, b in zip(labels, boxes)) + for a, b in zip(recovered, expected): + self.assertEqual(a[0], b[0]) + np.testing.assert_allclose(a[1:], b[1:], atol=1e-3) + + def test_ignore_region_masks_background_only(self): + target = encode_targets(np.array([[0, 0, 64, 80]]), [0], np.array([[0, 0, 160, 96]]), (96, 160)) + mask, heat = target['loss_mask'], target['heatmap'].max(axis=0) + self.assertTrue((mask[heat > 0] == 1).all()) + self.assertEqual(mask[:, 10:].sum(), 0) + + def test_overlapping_detections_are_suppressed_across_kinds(self): + heat = np.zeros((2, 6, 6), np.float32) + heat[0, 2, 2], heat[1, 2, 4] = .9, .6 + size = np.full((2, 6, 6), 40., np.float32) + offset = np.full((2, 6, 6), .5, np.float32) + detections = decode(heat, size, offset, threshold=.3) + self.assertEqual([d['kind'] for d in detections], ['member']) + + def test_visibility_and_aspect_filters(self): + heat = np.zeros((2, 8, 8), np.float32) + heat[0, 0, 0], heat[0, 4, 4], heat[1, 4, 1] = .9, .8, .7 + size = np.stack([np.full((8, 8), 30., np.float32), np.full((8, 8), 40., np.float32)]) + offset = np.full((2, 8, 8), .5, np.float32) + kept = decode(heat, size, offset, threshold=.3, image_size=(64, 64)) + self.assertEqual([round(d['score'], 1) for d in kept], [.8, .7]) + kept = decode(heat, size, offset, threshold=.3, image_size=(64, 64), aspect_tolerance=.2) + self.assertEqual([d['kind'] for d in kept], ['member']) + + def test_peaks_are_local_maxima(self): + heat = np.zeros((1, 5, 5), np.float32) + heat[0, 1, 1], heat[0, 1, 2], heat[0, 4, 4] = .8, .7, .5 + cls, rows, cols, scores = find_peaks(heat, .4) + self.assertEqual(list(zip(rows.tolist(), cols.tolist())), [(1, 1), (4, 4)]) + + +class MetricTests(unittest.TestCase): + def grid(self, rows, cols, w=74, h=97, x0=30, y0=20): + return np.array([[x0 + c * w * 1.11, y0 + r * h * 1.10, w, h] for r in range(rows) for c in range(cols)], np.float32) + + def test_grid_ignore_regions_cover_partial_row(self): + boxes = self.grid(3, 4, x0=5, y0=5) + regions = grid_ignore_regions(boxes, 330, 360) + self.assertEqual(len(regions), 4) + self.assertTrue(all(abs(r[1] - (5 + 3 * 97 * 1.10)) < 1e-3 for r in regions)) + + def test_perfect_predictions_and_ignored_false_positive(self): + boxes = self.grid(2, 3) + scene = {'boxes': boxes, 'labels': np.zeros(len(boxes), np.int64), 'foreign': np.zeros(len(boxes), bool), + 'ignored_rows': [{'kind': 'member', 'bbox': [30, 250, 74, 97]}], 'source_resolution': [400, 300]} + detections = [{'kind': 'member', 'score': .9, 'bbox': b.tolist()} for b in boxes] + detections.append({'kind': 'member', 'score': .8, 'bbox': [31, 251, 74, 97]}) + detections.append({'kind': 'snap', 'score': .7, 'bbox': [300, 200, 50, 30]}) + acc = Accumulator() + acc.add(detections, scene, 400, 300) + overall = acc.summary(.5)['overall'] + self.assertEqual(overall['iou0.5']['recall'], 1.) + self.assertEqual(overall['iou0.5']['false_positives'], 1) + self.assertEqual(acc.summary(.75)['overall']['iou0.5']['false_positives'], 0) + self.assertEqual(overall['iou0.75']['kind_accuracy'], 1.) + + def test_wrong_kind_counts_as_localized_but_not_kind_correct(self): + boxes = self.grid(1, 2) + rows, truth = match([{'kind': 'snap', 'score': .9, 'bbox': boxes[0].tolist()}], boxes, np.zeros(2, np.int64), np.zeros((0, 4))) + self.assertEqual(rows[0][.5], 'tp_wrong_kind') + self.assertEqual(truth[.5][0], (.9, False)) + self.assertIsNone(truth[.5][1]) + + def test_box_iou(self): + self.assertAlmostEqual(float(box_iou([[0, 0, 10, 10]], [[5, 0, 10, 10]])[0, 0]), 50 / 150) + + +@unittest.skipUnless(importlib.util.find_spec('torch') and importlib.util.find_spec('torchvision'), 'torch not installed') +class ModelShapeTests(unittest.TestCase): + def test_output_grid_is_stride_eight(self): + import torch + from bdon_vision.locator_model import ExportedLocator, LocatorNet + model = ExportedLocator(LocatorNet()).eval() + with torch.no_grad(): + heat, size, offset = model(torch.rand(1, 3, 96, 160)) + self.assertEqual(tuple(heat.shape), (1, len(CLASSES), 12, 20)) + self.assertEqual(tuple(size.shape), (1, 2, 12, 20)) + self.assertTrue(bool(((heat >= 0) & (heat <= 1)).all())) + self.assertTrue(bool((size > 0).all())) + + +if __name__ == '__main__': + unittest.main() From 37ad3e58429d911ac1976053f11e1a6f44bae102 Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:07:45 +0800 Subject: [PATCH 3/7] feat: add per-kind open-set artwork encoders Member and snap tiles get separate retrieval encoders whose inputs follow each kind's artwork window (member 160x128, snap 128x224). Includes per-kind crop banks built from scene truth files, open-set retrieval metrics with leave-true-out rejection, training with optional distillation and cross-kind identities, threshold selection, and ONNX export with gallery vectors bound to the artwork bytes they came from. --- bdon_vision/evaluate_kind_encoder.py | 151 +++++++++ bdon_vision/export_kind_encoder.py | 130 +++++++ bdon_vision/kind_corpus.py | 227 +++++++++++++ bdon_vision/kind_encoder.py | 127 +++++++ bdon_vision/kind_geometry.py | 164 +++++++++ bdon_vision/kind_metrics.py | 136 ++++++++ bdon_vision/train_kind_encoder.py | 487 +++++++++++++++++++++++++++ tests/test_kind_encoder.py | 217 ++++++++++++ 8 files changed, 1639 insertions(+) create mode 100644 bdon_vision/evaluate_kind_encoder.py create mode 100644 bdon_vision/export_kind_encoder.py create mode 100644 bdon_vision/kind_corpus.py create mode 100644 bdon_vision/kind_encoder.py create mode 100644 bdon_vision/kind_geometry.py create mode 100644 bdon_vision/kind_metrics.py create mode 100644 bdon_vision/train_kind_encoder.py create mode 100644 tests/test_kind_encoder.py diff --git a/bdon_vision/evaluate_kind_encoder.py b/bdon_vision/evaluate_kind_encoder.py new file mode 100644 index 0000000..4450d1f --- /dev/null +++ b/bdon_vision/evaluate_kind_encoder.py @@ -0,0 +1,151 @@ +"""Compare per-kind encoders with the shared encoder on identical crop banks. + +Thresholds are always chosen on the development split named by +``--select-on`` (separately for each model) and then applied unchanged to +every other bank. The shared encoder sees its own input format: member crops +are identical; snap windows and the 512x288 snap artwork are squashed to its +160x128 portrait input. +""" +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from .assets import read, write +from .kind_corpus import load_bank +from .kind_encoder import kind_cards, load_model, references +from .kind_metrics import open_set_curve, retrieval_table, select_thresholds, summarize +from .train_kind_encoder import embed + +SHARED_THRESHOLDS = {'similarity': .72, 'margin': .06} + + +def shared_model(path): + from .train import CardEncoder + model = CardEncoder() + model.load_state_dict(torch.load(path, map_location='cpu', weights_only=True)) + return model.eval() + + +def shared_references(data, cards): + rows = [] + for card in cards: + with Image.open(Path(data) / card.get('match_file', card['file'])) as image: + rows.append(np.asarray(image.convert('RGB').resize((128, 160), Image.Resampling.BILINEAR), np.float32)) + return np.stack(rows).transpose(0, 3, 1, 2) / 255. + + +def stratified(table, meta, mask, t, m): + """Summaries per tile width, visibility, occlusion and foreign-art method.""" + def sub(sel): + return summarize({k: v[sel] for k, v in table.items()}, t, m) + out = {} + width = meta['tile_width'][mask] + bins = np.digitize(width, [80, 120, 170, 240]) + for b, name in enumerate(['<80px', '80-120px', '120-170px', '170-240px', '>=240px']): + if (bins == b).any(): + out[f'tile_width {name}'] = sub(bins == b) + partial = meta['window_inside'][mask] < .999 + for flag, name in [(False, 'fully inside'), (True, 'partly outside')]: + if (partial == flag).any(): + out[name] = sub(partial == flag) + if 'occluded_fraction' in meta: + occluded = meta['occluded_fraction'][mask] > .05 + for flag, name in [(False, 'not occluded'), (True, 'occluded > 5%')]: + if (occluded == flag).any(): + out[name] = sub(occluded == flag) + if 'foreign_method' in meta: + method = meta['foreign_method'][mask] + for name in sorted(set(method.tolist()) - {''}): + out[f'foreign {name}'] = sub(method == name) + return out + + +def per_crop(table, meta, mask, cards): + rows = [] + files = meta['source'][mask] + boxes = meta['bbox'][mask] + for i in range(len(table['labels'])): + label = int(table['labels'][i]) + rows.append({'file': str(files[i]).split(':', 1)[1], 'bbox': [round(float(v), 1) for v in boxes[i]], + 'truth': cards[label]['id'] if label >= 0 else None, + 'top1': cards[int(table['pred'][i])]['id'], 'similarity': round(float(table['s1'][i]), 4), + 'margin': round(float(table['s1'][i] - table['s2'][i]), 4), + 'true_similarity': None if label < 0 else round(float(table['true_score'][i]), 4), + 'without_true_top1': cards[int(table['loo_pred'][i])]['id'], + 'without_true_similarity': round(float(table['loo_s1'][i]), 4), + 'without_true_margin': round(float(table['loo_s1'][i] - table['loo_s2'][i]), 4)}) + return rows + + +def evaluate(args): + catalog = read(args.data / 'catalog.json')['cards'] + cards = kind_cards(catalog, args.kind) + held = np.array([c['id'] % 7 == 0 for c in cards]) + models = {} + if args.model: + models['kind'] = (load_model(args.kind, args.model).to(args.device), torch.from_numpy(references(args.data, cards, args.kind)), + 'art', hashlib.sha256(args.model.read_bytes()).hexdigest()) + if args.shared: + crops = 'art' if args.kind == 'member' else 'baseline' + models['shared'] = (shared_model(args.shared).to(args.device), torch.from_numpy(shared_references(args.data, cards)).float(), + crops, hashlib.sha256(args.shared.read_bytes()).hexdigest()) + banks = dict(item.split('=', 1) for item in args.banks) + report = {'kind': args.kind, 'select_on': args.select_on, 'banks': {}, 'models': {}} + for name, (model, refs, crops, digest) in models.items(): + gallery = embed(model, refs.float()) + tables = {} + for bank_name, path in banks.items(): + arrays, labels, meta, manifest = load_bank(path, catalog, args.kind, (crops,)) + mask = np.ones(len(labels), bool) if bank_name in args.all_crops else meta['window_inside'] >= .999 + queries = embed(model, np.ascontiguousarray(arrays[crops][mask])) + tables[bank_name] = (retrieval_table(queries, gallery, labels[mask], held), meta, mask, manifest) + choice = select_thresholds(tables[args.select_on][0]) + result = {'sha256': digest, 'input': crops, 'selected_thresholds': choice, + 'open_set_curve_on_selection_split': [{k: v for k, v in row.items() if k != 'summary'} | ( + {'known_accepted_correct': row['summary']['known']['accepted_correct_rate']} if 'summary' in row else {}) + for row in open_set_curve(tables[args.select_on][0])], 'banks': {}} + for bank_name, (table, meta, mask, manifest) in tables.items(): + entry = {'crops': int(mask.sum()), 'fingerprint': manifest['source_fingerprint'], + 'top1_only': summarize(table, 2., 1.)} + if choice['feasible']: + entry['at_selected'] = summarize(table, choice['similarity'], choice['margin']) + entry['strata_at_selected'] = stratified(table, meta, mask, choice['similarity'], choice['margin']) + if name == 'shared': + entry['at_shared_runtime_thresholds'] = summarize(table, SHARED_THRESHOLDS['similarity'], + SHARED_THRESHOLDS['margin']) + if bank_name in args.per_crop: + entry['per_crop'] = per_crop(table, meta, mask, cards) + result['banks'][bank_name] = entry + report['banks'][bank_name] = str(path) + report['models'][name] = result + write(args.output, report) + brief = {name: {b: {'seen_top1': r['top1_only']['seen']['top1'], 'unseen_top1': r['top1_only']['unseen']['top1'], + 'sel': res['selected_thresholds'].get('similarity'), + 'known_acc_correct': r.get('at_selected', {}).get('known', {}).get('accepted_correct_rate'), + 'unseen_open_far': r.get('at_selected', {}).get('unseen', {}).get('open_set_false_accept')} + for b, r in res['banks'].items()} for name, res in report['models'].items()} + print(json.dumps(brief, indent=1), flush=True) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--kind', choices=['member', 'snap'], required=True) + p.add_argument('--data', type=Path, required=True) + p.add_argument('--model', type=Path, help='per-kind encoder state dict') + p.add_argument('--shared', type=Path, help='shared 160x128 encoder state dict') + p.add_argument('--banks', nargs='+', required=True, help='name=crop-bank-root') + p.add_argument('--select-on', default='validation') + p.add_argument('--per-crop', nargs='*', default=[], help='bank names to list crop by crop') + p.add_argument('--all-crops', nargs='*', default=[], help='bank names that keep partly visible windows') + p.add_argument('--output', type=Path, required=True) + p.add_argument('--device', default='cuda') + evaluate(p.parse_args()) + + +if __name__ == '__main__': + main() diff --git a/bdon_vision/export_kind_encoder.py b/bdon_vision/export_kind_encoder.py new file mode 100644 index 0000000..6a9c6a4 --- /dev/null +++ b/bdon_vision/export_kind_encoder.py @@ -0,0 +1,130 @@ +"""Export a per-kind encoder to ONNX and build its gallery vectors. + +Outputs in ``--output``: + +* ``-encoder.onnx``: input ``image`` float32 [N, 3, H, W] RGB in [0, 1], + output ``embedding`` float32 [N, 128], L2-normalised; opset 17, dynamic N. +* ``-gallery.json`` / ``-gallery.npz``: one vector per catalog + card, each bound to the SHA-256 of the artwork bytes it was computed from. +* ``-export.json``: file hashes, operator list and consistency checks. + +Gallery vectors are computed with the exported ONNX model on CPU, i.e. the +same graph a browser client runs. +""" +import argparse +import hashlib +import json +from pathlib import Path + +import cv2 +import numpy as np +import torch +from PIL import Image + +from .assets import read, write +from .kind_encoder import export_onnx, gallery_document, kind_cards, load_model, reference_source +from .kind_geometry import INPUT, MEMBER_CANONICAL, WINDOW, center_crop_box, reference_image + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def alternative_reference(kind, path): + """Reference input built with OpenCV area resampling instead of PIL. + + Used only to measure how sensitive gallery vectors are to a client's + resampling implementation. + """ + h, w = INPUT[kind] + with Image.open(path) as image: + rgb = np.asarray(image.convert('RGB')) + if kind == 'member': + if (rgb.shape[1], rgb.shape[0]) != MEMBER_CANONICAL: + box = center_crop_box(rgb.shape[1], rgb.shape[0], MEMBER_CANONICAL[0] / MEMBER_CANONICAL[1]) + else: + box = (0, 0, rgb.shape[1], rgb.shape[0]) + else: + box = center_crop_box(rgb.shape[1], rgb.shape[0], WINDOW[kind][0] / WINDOW[kind][1]) + left, top, right, bottom = box + matrix = np.array([[(right - left) / w, 0, left], [0, (bottom - top) / h, top]], np.float32) + if (right - left) / w > 1.5: + # Area-average to the target size first, as a downscaling client would. + crop = rgb[int(round(top)):int(round(bottom)), int(round(left)):int(round(right))] + out = cv2.resize(crop, (w, h), interpolation=cv2.INTER_AREA) + else: + out = cv2.warpAffine(rgb, matrix, (w, h), flags=cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP, + borderMode=cv2.BORDER_REPLICATE) + return out.astype(np.float32).transpose(2, 0, 1) / 255. + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--kind', choices=['member', 'snap'], required=True) + p.add_argument('--data', type=Path, required=True) + p.add_argument('--model', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--thresholds', type=Path, help='JSON with selected similarity/margin thresholds to embed') + a = p.parse_args() + import onnx + import onnxruntime as ort + + kind = a.kind + a.output.mkdir(parents=True, exist_ok=True) + model = load_model(kind, a.model) + onnx_path = a.output / f'{kind}-encoder.onnx' + export_onnx(model, onnx_path, kind) + graph = onnx.load(str(onnx_path)) + onnx.checker.check_model(graph) + operators = sorted({node.op_type for node in graph.graph.node}) + session = ort.InferenceSession(str(onnx_path), providers=['CPUExecutionProvider']) + + catalog = read(a.data / 'catalog.json')['cards'] + cards = kind_cards(catalog, kind) + sources = [reference_source(a.data, c) for c in cards] + inputs = [] + for path in sources: + with Image.open(path) as image: + inputs.append(np.asarray(reference_image(kind, image), np.float32).transpose(2, 0, 1) / 255.) + inputs = np.stack(inputs).astype(np.float32) + vectors = np.concatenate([session.run(None, {'image': inputs[i:i + 16]})[0] for i in range(0, len(inputs), 16)]) + with torch.no_grad(): + reference = model(torch.from_numpy(inputs)).numpy() + single = np.concatenate([session.run(None, {'image': inputs[i:i + 1]})[0] for i in range(len(inputs))]) + rng = np.random.default_rng(1) + noise = rng.random((7, 3, *INPUT[kind]), dtype=np.float32) + with torch.no_grad(): + noise_ref = model(torch.from_numpy(noise)).numpy() + noise_onnx = session.run(None, {'image': noise})[0] + alternative = np.stack([alternative_reference(kind, s) for s in sources]).astype(np.float32) + alt_vectors = np.concatenate([session.run(None, {'image': alternative[i:i + 16]})[0] + for i in range(0, len(alternative), 16)]) + alt_cos = (alt_vectors * vectors).sum(1) + alt_rank = (alt_vectors @ vectors.T).argmax(1) == np.arange(len(vectors)) + checks = { + 'torch_vs_onnx_max_abs_diff': float(np.abs(reference - vectors).max()), + 'torch_vs_onnx_random_max_abs_diff': float(np.abs(noise_ref - noise_onnx).max()), + 'batch_vs_single_max_abs_diff': float(np.abs(single - vectors).max()), + 'vector_norm_range': [float(np.linalg.norm(vectors, axis=1).min()), float(np.linalg.norm(vectors, axis=1).max())], + 'alternative_resampling_cosine_min': float(alt_cos.min()), + 'alternative_resampling_cosine_mean': float(alt_cos.mean()), + 'alternative_resampling_self_retrieval': int(alt_rank.sum()), 'cards': len(cards), + } + model_sha = sha256(onnx_path) + document = gallery_document(a.data, cards, kind, vectors, model_sha) + if a.thresholds: + document['thresholds'] = read(a.thresholds) + gallery_json = a.output / f'{kind}-gallery.json' + gallery_json.write_text(json.dumps(document, ensure_ascii=False, separators=(',', ':')), encoding='utf-8') + gallery_npz = a.output / f'{kind}-gallery.npz' + np.savez(gallery_npz, ids=np.array([c['id'] for c in cards], np.int32), vectors=vectors.astype(np.float32), + source_sha256=np.array([r['source_sha256'] for r in document['cards']]), model_sha256=np.array(model_sha)) + manifest = {'kind': kind, 'input_hw': list(INPUT[kind]), 'opset': 17, 'operators': operators, + 'files': {p.name: sha256(p) for p in [onnx_path, gallery_json, gallery_npz]}, + 'source_state_dict_sha256': sha256(a.model), 'checks': checks} + write(a.output / f'{kind}-export.json', manifest) + print(json.dumps(manifest, indent=1), flush=True) + + +if __name__ == '__main__': + main() diff --git a/bdon_vision/kind_corpus.py b/bdon_vision/kind_corpus.py new file mode 100644 index 0000000..a01bcef --- /dev/null +++ b/bdon_vision/kind_corpus.py @@ -0,0 +1,227 @@ +"""Per-kind artwork crop banks from scene truth files. + +Every tile listed in a screenshot's ``cards`` becomes a query crop labelled +with its catalog index. Tiles listed in ``foreign_cards`` (artwork outside the +catalog) are stored with label -1 and a ``foreign`` flag; they are evaluation +material for open-set rejection and never identity supervision. + +Bank layout (one directory per kind; ``--arrays`` selects the crop arrays):: + + art.npy uint8 [N, H, W, 3] exact artwork window at encoder input size + context.npy uint8 [N, Hc, Wc, 3] window expanded by CONTEXT_MARGIN per side + baseline.npy uint8 [N, 160, 128, 3] snap only: window squashed to the shared + portrait input of the single-encoder model + labels.npy int32 [N] catalog index, -1 for foreign artwork + meta.npz per-crop source, visibility and acquisition fields + manifest.json catalog order, dataset fingerprints and counts +""" +import argparse +import hashlib +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +import cv2 +import numpy as np +from numpy.lib.format import open_memmap + +from .assets import read, write +from .kind_geometry import INPUT, KINDS, art_window, inside_fraction, query_crop, sample_rect + +CONTEXT_MARGIN = .125 +ARRAYS = ('art', 'context', 'baseline') +BASELINE_SIZE = (128, 160) # (w, h) of the shared portrait encoder input +LOCALES = ['ja', 'en', 'zh-Hant', 'zh-Hans', 'ko'] +MIN_WINDOW_INSIDE = .6 + + +def context_shape(kind): + h, w = INPUT[kind] + return round(h * (1 + 2 * CONTEXT_MARGIN)), round(w * (1 + 2 * CONTEXT_MARGIN)) + + +def _tiles(row, kind): + for index, card in enumerate(row.get('cards', [])): + if card.get('kind', row.get('kind')) == kind: + yield index, card, False + for index, card in enumerate(row.get('foreign_cards', [])): + if card.get('kind', row.get('kind')) == kind: + yield index, card, True + + +def _crop_screenshot(job): + path, row, kind, verify, arrays, bbox_key, resample = job + raw = (path / row['file']).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if verify and row.get('sha256') and row['sha256'] != digest: + raise ValueError(f"Image hash mismatch: {path / row['file']}") + image = cv2.imdecode(np.frombuffer(raw, np.uint8), cv2.IMREAD_COLOR) + if image is None: + raise ValueError(f"Cannot decode {path / row['file']}") + ih, iw = image.shape[:2] + out = [] + for index, card, foreign in _tiles(row, kind): + bbox = [float(v) for v in card.get(bbox_key) or card['bbox']] + window = art_window(kind, bbox) + window_inside = inside_fraction(window, (iw, ih)) + if window_inside < MIN_WINDOW_INSIDE: + out.append(None) + continue + item = { + 'art': query_crop(image, kind, bbox, resample=resample) if 'art' in arrays else None, + 'context': query_crop(image, kind, bbox, CONTEXT_MARGIN) if 'context' in arrays else None, + 'foreign': foreign, 'id': card.get('id'), 'card_index': index, + 'window_inside': window_inside, 'tile_inside': inside_fraction(bbox, (iw, ih)), + 'visible_fraction': float(card.get('visible_fraction', 1.)), + 'occluded_fraction': float(card.get('occluded_fraction', 0.)), + 'foreign_method': str(card.get('foreign_method', '')) if foreign else '', + 'tile_width': bbox[2], 'display_mode': card.get('display_mode', row.get('display_mode', '')), + 'bbox': bbox, + } + if kind == 'snap' and 'baseline' in arrays: + item['baseline'] = sample_rect(image, window, BASELINE_SIZE, resample=resample) + out.append(item) + return digest, out + + +def build(data, datasets, output, kind, workers=8, allow_incomplete=False, arrays=ARRAYS, stride=1, bbox_key='bbox', + resample='linear'): + """Write the crop bank of one kind. ``arrays`` selects which crop arrays to + store; ``stride`` keeps every n-th screenshot that shows this kind; + ``bbox_key`` names an alternative tile rectangle field (falls back to bbox); + ``resample`` selects how evaluation crops are shrunk (training context is + always bilinear).""" + output = Path(output) / kind + output.mkdir(parents=True, exist_ok=True) + catalog = read(Path(data) / 'catalog.json')['cards'] + lookup = {(c['kind'], c['id']): i for i, c in enumerate(catalog)} + manifests = [] + fingerprint = hashlib.sha256() + for path in datasets: + path = Path(path) + document = read(path / 'truth.json') + if not document.get('complete', False) and not allow_incomplete: + raise ValueError(f'Incomplete dataset: {path}') + fingerprint.update((path / 'truth.json').read_bytes()) + manifests.append((path, document)) + selected = [(d, s) for d, (_, document) in enumerate(manifests) + for s, row in enumerate(document['screenshots']) if any(True for _ in _tiles(row, kind))][::stride] + jobs = [(manifests[d][0], manifests[d][1]['screenshots'][s], kind, True, tuple(arrays), bbox_key, resample) for d, s in selected] + capacity = sum(1 for job in jobs for _ in _tiles(job[1], kind)) + h, w = INPUT[kind] + ch, cw = context_shape(kind) + capacity = max(capacity, 1) + art = open_memmap(output / 'art.npy', mode='w+', dtype='uint8', shape=(capacity, h, w, 3)) if 'art' in arrays else None + context = open_memmap(output / 'context.npy', mode='w+', dtype='uint8', + shape=(capacity, ch, cw, 3)) if 'context' in arrays else None + baseline = open_memmap(output / 'baseline.npy', mode='w+', dtype='uint8', + shape=(capacity, BASELINE_SIZE[1], BASELINE_SIZE[0], 3)) if kind == 'snap' and 'baseline' in arrays else None + labels = np.full(capacity, -1, np.int32) + columns = {name: [] for name in ['dataset', 'screenshot', 'card_index', 'foreign', 'window_inside', + 'tile_inside', 'visible_fraction', 'occluded_fraction', 'tile_width', 'locale', + 'mode', 'profile', 'foreign_method']} + bboxes = [] + files = [] + count = 0 + skipped = Counter() + unknown_ids = Counter() + profiles = Counter() + modes = [] + dataset_of_job = [d for d, _ in selected] + screenshot_of_job = [s for _, s in selected] + with ProcessPoolExecutor(max_workers=workers) as pool: + for j, (digest, items) in enumerate(pool.map(_crop_screenshot, jobs, chunksize=4)): + fingerprint.update(bytes.fromhex(digest)) + row = jobs[j][1] + document = manifests[dataset_of_job[j]][1] + for item in items: + if item is None: + skipped['window_outside_image'] += 1 + continue + if not item['foreign']: + key = (kind, item['id']) + if key not in lookup: + unknown_ids[str(item['id'])] += 1 + continue + labels[count] = lookup[key] + if art is not None: + art[count] = item['art'] + if context is not None: + context[count] = item['context'] + if baseline is not None: + baseline[count] = item['baseline'] + mode = item['display_mode'] + if mode not in modes: + modes.append(mode) + profile = document.get('profile', 'real') + profiles[profile] += 1 + for name, value in [('dataset', dataset_of_job[j]), ('screenshot', screenshot_of_job[j]), + ('card_index', item['card_index']), ('foreign', item['foreign']), + ('window_inside', item['window_inside']), ('tile_inside', item['tile_inside']), + ('visible_fraction', item['visible_fraction']), + ('occluded_fraction', item['occluded_fraction']), ('tile_width', item['tile_width']), + ('locale', LOCALES.index(row['locale']) if row.get('locale') in LOCALES else -1), + ('mode', modes.index(mode)), ('profile', profile), + ('foreign_method', item['foreign_method'])]: + columns[name].append(value) + bboxes.append(item['bbox']) + files.append(f"{dataset_of_job[j]}:{row['file']}") + count += 1 + if (j + 1) % 200 == 0: + print({'kind': kind, 'screenshots': j + 1, 'of': len(jobs), 'crops': count}, flush=True) + for array in (art, context, baseline): + if array is not None: + array.flush() + np.save(output / 'labels.npy', labels[:count]) + np.savez(output / 'meta.npz', bbox=np.array(bboxes, np.float32).reshape(-1, 4), source=np.array(files), + **{k: np.array(v) for k, v in columns.items()}) + kind_cards = [[c['kind'], c['id']] for c in catalog] + manifest = { + 'schema': 'bdon-kind-crop-bank/1', 'kind': kind, 'input_hw': list(INPUT[kind]), + 'context_hw': list(context_shape(kind)), 'context_margin': CONTEXT_MARGIN, + 'source_fingerprint': fingerprint.hexdigest(), 'catalog_ids': kind_cards, + 'capacity': capacity, 'count': count, 'arrays': [a for a in ARRAYS if a in arrays and (a != 'baseline' or kind == 'snap')], + 'screenshot_stride': stride, 'bbox_key': bbox_key, 'resample': resample, 'known': int((labels[:count] >= 0).sum()), + 'foreign': int((labels[:count] < 0).sum()), 'screenshots': len(jobs), 'profiles': dict(profiles), + 'display_modes': modes, 'locale_names': LOCALES, 'skipped': dict(skipped), + 'ids_outside_catalog': dict(unknown_ids), 'datasets': [str(p) for p, _ in manifests], 'complete': True, + } + write(output / 'manifest.json', manifest) + print(manifest, flush=True) + return manifest + + +def load_bank(path, catalog, kind, arrays=('art',)): + """Load a bank as numpy arrays plus kind-local labels (-1 foreign).""" + path = Path(path) / kind + manifest = read(path / 'manifest.json') + if not manifest.get('complete'): + raise ValueError(f'Incomplete crop bank: {path}') + if manifest['catalog_ids'] != [[c['kind'], c['id']] for c in catalog]: + raise ValueError(f'Catalog order differs from crop bank {path}; rebuild it') + count = manifest['count'] + kind_index = {i: k for k, i in enumerate(i for i, c in enumerate(catalog) if c['kind'] == kind)} + labels = np.load(path / 'labels.npy') + local = np.array([kind_index[int(v)] if v >= 0 else -1 for v in labels], np.int64) + out = {name: np.load(path / f'{name}.npy', mmap_mode='r')[:count] for name in arrays} + meta = dict(np.load(path / 'meta.npz', allow_pickle=False)) + return out, local, meta, manifest + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--data', type=Path, required=True) + p.add_argument('--datasets', nargs='+', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--kinds', nargs='+', choices=KINDS, default=list(KINDS)) + p.add_argument('--workers', type=int, default=8) + p.add_argument('--allow-incomplete', action='store_true', + help='accept truth files without a complete flag (hand-labelled regression sets)') + p.add_argument('--arrays', nargs='+', choices=ARRAYS, default=list(ARRAYS), + help='crop arrays to store (training needs context; evaluation needs art, plus baseline for snap)') + p.add_argument('--stride', type=int, default=1, help='keep every n-th screenshot that shows the kind') + p.add_argument('--bbox-key', default='bbox', help='card field holding the tile rectangle') + p.add_argument('--resample', choices=['linear', 'area'], default='linear', help='shrinking method of art crops') + a = p.parse_args() + for kind in a.kinds: + build(a.data, a.datasets, a.output, kind, a.workers, a.allow_incomplete, a.arrays, a.stride, a.bbox_key, a.resample) diff --git a/bdon_vision/kind_encoder.py b/bdon_vision/kind_encoder.py new file mode 100644 index 0000000..de54666 --- /dev/null +++ b/bdon_vision/kind_encoder.py @@ -0,0 +1,127 @@ +"""Per-kind open-set artwork encoders. + +``KindEncoder`` keeps the MobileNetV3-Small feature extractor and the +``2x2 average pool -> Linear(576*4, 128) -> L2`` head of the shared encoder, +so both kinds can start from the same weights. The pooling window is chosen +per input size so the head always sees a 2x2 grid: + +* member 160x128 -> 5x4 features, AvgPool kernel (3, 2) stride (2, 2) +* snap 128x224 -> 4x7 features, AvgPool kernel (2, 4) stride (2, 3) + +Both are plain ONNX ``AveragePool`` operators. +""" +import hashlib +from pathlib import Path + +import numpy as np +from PIL import Image + +from .kind_geometry import INPUT, reference_image + +POOL = {'member': ((3, 2), (2, 2)), 'snap': ((2, 4), (2, 3))} + + +def feature_grid(height, width): + """MobileNetV3 feature map size (stride 32, 'same'-style padding).""" + for _ in range(5): + height, width = (height + 1) // 2, (width + 1) // 2 + return height, width + + +def pooled_grid(kind): + (kh, kw), (sh, sw) = POOL[kind] + fh, fw = feature_grid(*INPUT[kind]) + return (fh - kh) // sh + 1, (fw - kw) // sw + 1 + + +def build_model(kind, pretrained=False): + import torch + from torch import nn + from torch.nn import functional as F + + class KindEncoder(nn.Module): + def __init__(self): + super().__init__() + from torchvision.models import mobilenet_v3_small, MobileNet_V3_Small_Weights + weights = MobileNet_V3_Small_Weights.DEFAULT if pretrained else None + self.kind = kind + self.features = mobilenet_v3_small(weights=weights).features + self.projection = nn.Linear(576 * 4, 128) + self.register_buffer('mean', torch.tensor([.485, .456, .406]).view(1, 3, 1, 1)) + self.register_buffer('std', torch.tensor([.229, .224, .225]).view(1, 3, 1, 1)) + self.pool_kernel, self.pool_stride = POOL[kind] + + def embed(self, x): + x = self.features((x - self.mean) / self.std) + x = F.avg_pool2d(x, self.pool_kernel, self.pool_stride).flatten(1) + return self.projection(x) + + def forward(self, x): + return F.normalize(self.embed(x), dim=1) + + return KindEncoder() + + +def load_model(kind, path): + import torch + model = build_model(kind) + state = torch.load(path, map_location='cpu', weights_only=True) + if isinstance(state, dict) and 'model' in state: + state = state['model'] + model.load_state_dict(state) + return model.eval() + + +def freeze_batchnorm(model): + """Keep BatchNorm running statistics fixed (affine parameters still train).""" + from torch import nn + for module in model.modules(): + if isinstance(module, nn.BatchNorm2d): + module.eval() + return model + + +def reference_array(kind, path): + """float32 [3, H, W] reference input for a catalog artwork file.""" + with Image.open(path) as image: + return np.asarray(reference_image(kind, image), np.float32).transpose(2, 0, 1) / 255. + + +def kind_cards(catalog, kind): + return [c for c in catalog if c['kind'] == kind] + + +def reference_source(data, card): + """Artwork file a reference is built from: member square sprite or snap thumbnail.""" + data = Path(data) + if card['kind'] == 'member': + square = data / f"native/assets/member-{card['asset_id']}-square.webp" + if square.exists(): + return square + return data / card.get('match_file', card['file']) + return data / card['file'] + + +def references(data, cards, kind): + return np.stack([reference_array(kind, reference_source(data, c)) for c in cards]) + + +def export_onnx(model, path, kind): + import torch + model = model.eval().cpu() + h, w = INPUT[kind] + torch.onnx.export(model, torch.zeros(1, 3, h, w), str(path), input_names=['image'], output_names=['embedding'], + dynamic_axes={'image': {0: 'batch'}, 'embedding': {0: 'batch'}}, opset_version=17, dynamo=False) + + +def gallery_document(data, cards, kind, vectors, model_sha256): + """JSON-serialisable gallery bound to the artwork bytes each vector came from.""" + rows = [] + for card, vector in zip(cards, vectors): + source = reference_source(data, card) + rows.append({'kind': kind, 'id': card['id'], 'asset_id': card['asset_id'], + 'source': source.name, 'source_sha256': hashlib.sha256(source.read_bytes()).hexdigest(), + 'vector': [round(float(v), 6) for v in vector]}) + h, w = INPUT[kind] + return {'schema': 'bdon-kind-gallery/1', 'kind': kind, 'input_hw': [h, w], 'dimension': int(vectors.shape[1]), + 'model_sha256': model_sha256, 'cards': rows} diff --git a/bdon_vision/kind_geometry.py b/bdon_vision/kind_geometry.py new file mode 100644 index 0000000..97ee0ee --- /dev/null +++ b/bdon_vision/kind_geometry.py @@ -0,0 +1,164 @@ +"""Per-kind card geometry and encoder input preparation. + +Member and Snap list tiles have different native layouts. Each kind gets an +encoder input with roughly its own artwork aspect ratio instead of a shared +portrait resize: + +* member: tile 224x294 logical units, artwork window inset by 6 -> 212x282, + encoder input 160x128 (H x W). +* snap: tile 326x184, artwork window 314x172, encoder input 128x224 (H x W). + The 512x288 source artwork fills the window with a centered aspect crop. + +Query crops come from a tile rectangle ``[x, y, w, h]`` in image pixels. +Reference inputs come from the catalog artwork with the same preprocessing a +browser client can reproduce (see ``reference_image``). +""" +import cv2 +import numpy as np +from PIL import Image, ImageOps + +KINDS = ('member', 'snap') +TILE = {'member': (224, 294), 'snap': (326, 184)} +ART_INSET = 6 +WINDOW = {'member': (212, 282), 'snap': (314, 172)} +# Encoder input (height, width). +INPUT = {'member': (160, 128), 'snap': (128, 224)} +# Member references are built from the square sprite with an aspect fill. +MEMBER_CANONICAL = (212, 282) +# Normalized (x0, y0, x1, y1) regions inside the artwork window that the list +# UI draws over: attribute badge, level/field text and card-rank icon. +OVERLAYS = { + 'member': ((0., 0., .17, .13), (0., .82, .50, 1.), (.73, .80, 1., 1.)), + 'snap': ((0., 0., .11, .21), (0., .70, .31, 1.), (.84, .67, 1., 1.)), +} + + +def tile_scale(kind, bbox): + """Image pixels per logical unit along x and y.""" + tw, th = TILE[kind] + return bbox[2] / tw, bbox[3] / th + + +def art_window(kind, bbox): + """Artwork window ``(left, top, width, height)`` for a tile rectangle.""" + x, y, w, h = (float(v) for v in bbox) + sx, sy = tile_scale(kind, bbox) + return x + ART_INSET * sx, y + ART_INSET * sy, w - 2 * ART_INSET * sx, h - 2 * ART_INSET * sy + + +def expand(rect, margin): + """Grow ``(left, top, width, height)`` by ``margin`` of its size per side.""" + left, top, width, height = rect + return left - width * margin, top - height * margin, width * (1 + 2 * margin), height * (1 + 2 * margin) + + +def inside_fraction(rect, image_size): + """Fraction of a rectangle's area that lies inside an image ``(w, h)``.""" + left, top, width, height = rect + iw, ih = image_size + ox = max(0., min(iw, left + width) - max(0., left)) + oy = max(0., min(ih, top + height) - max(0., top)) + return ox * oy / max(width * height, 1e-9) + + +def sample_rect(image, rect, size, border=cv2.BORDER_REPLICATE, resample='linear'): + """Resample ``rect`` from a BGR image into ``size`` (w, h); returns RGB. + + ``linear`` samples bilinearly at the output pixel centres. ``area`` first + cuts the rectangle at source resolution and then area-averages it down, + which is closer to what an image-smoothing canvas does when shrinking. + """ + left, top, width, height = rect + if resample == 'area' and (width > size[0] or height > size[1]): + inner = (max(size[0], int(np.ceil(width))), max(size[1], int(np.ceil(height)))) + matrix = np.array([[width / inner[0], 0, left], [0, height / inner[1], top]], np.float32) + cut = cv2.warpAffine(image, matrix, inner, flags=cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP, borderMode=border) + out = cv2.resize(cut, size, interpolation=cv2.INTER_AREA) + else: + matrix = np.array([[width / size[0], 0, left], [0, height / size[1], top]], np.float32) + out = cv2.warpAffine(image, matrix, size, flags=cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP, borderMode=border) + return np.ascontiguousarray(out[:, :, ::-1]) + + +def query_crop(image, kind, bbox, margin=0., resample='linear'): + """Encoder query crop (RGB uint8, H x W x 3) for a tile in a BGR screenshot. + + With ``margin > 0`` the window is expanded on every side and the output is + scaled by ``1 + 2 * margin`` so the central region still has input size. + ``resample`` is passed to :func:`sample_rect`. + """ + h, w = INPUT[kind] + rect = art_window(kind, bbox) + if margin: + rect = expand(rect, margin) + w, h = round(w * (1 + 2 * margin)), round(h * (1 + 2 * margin)) + return sample_rect(image, rect, (w, h), resample=resample) + + +def center_crop_box(width, height, aspect): + """Largest centered ``(left, top, right, bottom)`` box with ``aspect`` = w / h.""" + if width / height > aspect: + cw = height * aspect + left = (width - cw) / 2 + return left, 0., left + cw, float(height) + ch = width / aspect + top = (height - ch) / 2 + return 0., top, float(width), top + ch + + +def reference_image(kind, image): + """Reference encoder input (PIL RGB, input size) from catalog artwork. + + member: the square sprite (or an already fitted 212x282 canonical image) is + aspect-filled to 212x282 with LANCZOS, then resized to 128x160 bilinear. + snap: the 512x288 artwork is center-cropped to the 314:172 window aspect, + then resized to 224x128 bilinear. + """ + image = image.convert('RGB') + h, w = INPUT[kind] + if kind == 'member': + if image.size != MEMBER_CANONICAL: + image = ImageOps.fit(image, MEMBER_CANONICAL, Image.Resampling.LANCZOS) + return image.resize((w, h), Image.Resampling.BILINEAR) + ww, wh = WINDOW[kind] + box = center_crop_box(image.width, image.height, ww / wh) + return image.resize((w, h), Image.Resampling.BILINEAR, box=box) + + +def cross_window(kind, source_kind): + """Central ``(x, y)`` fraction of a ``source_kind`` artwork window that has + the window aspect ratio of ``kind``.""" + target = WINDOW[kind][0] / WINDOW[kind][1] + source = WINDOW[source_kind][0] / WINDOW[source_kind][1] + return (target / source, 1.) if source > target else (1., source / target) + + +def cross_reference_image(kind, source_kind, image): + """Reference input for ``kind`` cut from the window view of another kind's + artwork (the same region :func:`cross_window` takes from a query crop).""" + image = image.convert('RGB') + if source_kind == 'member': + if image.size != MEMBER_CANONICAL: + image = ImageOps.fit(image, MEMBER_CANONICAL, Image.Resampling.LANCZOS) + left, top, right, bottom = 0., 0., float(image.width), float(image.height) + else: + sw, sh = WINDOW[source_kind] + left, top, right, bottom = center_crop_box(image.width, image.height, sw / sh) + fx, fy = cross_window(kind, source_kind) + cx, cy = (left + right) / 2, (top + bottom) / 2 + half_w, half_h = (right - left) * fx / 2, (bottom - top) * fy / 2 + h, w = INPUT[kind] + return image.resize((w, h), Image.Resampling.BILINEAR, box=(cx - half_w, cy - half_h, cx + half_w, cy + half_h)) + + +def to_chw(array): + """uint8 H x W x 3 RGB -> float32 3 x H x W in [0, 1].""" + return np.ascontiguousarray(np.asarray(array, np.float32).transpose(2, 0, 1) / 255.) + + +def overlay_mask(kind, height, width): + """Boolean H x W mask of the UI overlay regions inside the artwork window.""" + mask = np.zeros((height, width), bool) + for x0, y0, x1, y1 in OVERLAYS[kind]: + mask[int(round(y0 * height)):int(round(y1 * height)), int(round(x0 * width)):int(round(x1 * width))] = True + return mask diff --git a/bdon_vision/kind_metrics.py b/bdon_vision/kind_metrics.py new file mode 100644 index 0000000..a779bd9 --- /dev/null +++ b/bdon_vision/kind_metrics.py @@ -0,0 +1,136 @@ +"""Retrieval, acceptance and open-set metrics for gallery encoders (numpy only). + +A query is accepted when its best gallery similarity is at least ``t`` and +exceeds the second best by at least ``m``. Open-set behaviour is measured by +removing the query's own card from the gallery ("leave-true-out") and counting +how often the remaining gallery still produces an accepted answer, plus the +accept rate on tiles whose artwork is outside the catalog. +""" +import numpy as np + +T_GRID = np.round(np.arange(.30, .96, .01), 2) +M_GRID = np.round(np.arange(0., .31, .01), 2) + + +def top2(scores): + order = np.argsort(-scores, axis=1)[:, :2] + rows = np.arange(len(scores))[:, None] + values = scores[rows, order] + return order[:, 0], values[:, 0], values[:, 1] + + +def retrieval_table(queries, gallery, labels, held): + """Per-query retrieval facts. + + queries [N, D], gallery [K, D] (L2-normalised), labels [N] gallery index or + -1 for artwork outside the catalog, held [K] bool marking identities that + never entered training. + """ + labels = np.asarray(labels) + scores = np.asarray(queries, np.float64) @ np.asarray(gallery, np.float64).T + known = labels >= 0 + pred, s1, s2 = top2(scores) + loo = scores.copy() + loo[np.flatnonzero(known), labels[known]] = -np.inf + loo_pred, loo_s1, loo_s2 = top2(loo) + true_score = np.full(len(labels), np.nan) + true_score[known] = scores[np.flatnonzero(known), labels[known]] + group = np.where(~known, 'foreign', np.where(np.asarray(held)[np.maximum(labels, 0)], 'unseen', 'seen')) + return {'pred': pred, 's1': s1, 's2': s2, 'correct': known & (pred == labels), 'labels': labels, + 'loo_pred': loo_pred, 'loo_s1': loo_s1, 'loo_s2': loo_s2, 'true_score': true_score, 'group': group} + + +def accepted(s1, s2, t, m): + return (s1 >= t) & (s1 - s2 >= m) + + +def _rate(numerator, denominator): + return None if denominator == 0 else numerator / denominator + + +def summarize(table, t, m): + """Closed-set, acceptance and open-set numbers per identity group.""" + out = {'thresholds': {'similarity': float(t), 'margin': float(m)}} + acc = accepted(table['s1'], table['s2'], t, m) + loo_acc = accepted(table['loo_s1'], table['loo_s2'], t, m) + for name in ('seen', 'unseen', 'known', 'foreign'): + mask = table['group'] != 'foreign' if name == 'known' else table['group'] == name + n = int(mask.sum()) + row = {'count': n} + if name != 'foreign': + correct = table['correct'][mask] + a = acc[mask] + row.update({ + 'top1': _rate(int(correct.sum()), n), + 'accept_rate': _rate(int(a.sum()), n), + 'accepted_precision': _rate(int((a & correct).sum()), int(a.sum())), + 'accepted_correct_rate': _rate(int((a & correct).sum()), n), + 'wrong_accepts': int((a & ~correct).sum()), + 'open_set_false_accept': _rate(int(loo_acc[mask].sum()), n), + 'median_true_similarity': None if n == 0 else float(np.median(table['true_score'][mask])), + 'median_margin': None if n == 0 else float(np.median((table['s1'] - table['s2'])[mask])), + }) + else: + row['false_accept'] = _rate(int(acc[mask].sum()), n) + out[name] = row + return out + + +def select_thresholds(table, max_false_accept=.01, min_precision=.995, tolerance=.001): + """Pick (t, m) maximising known accepted-correct rate under open-set limits. + + Limits: unseen leave-true-out false accepts and foreign-artwork accepts each + at most ``max_false_accept``; accepted precision over known queries at + least ``min_precision``. Seen and unseen identities are weighted equally. + Among settings within ``tolerance`` of the best score, those with the + fewest open-set accepts are kept and the median similarity threshold (then + the median margin at that threshold) is returned, which stays away from + the edge of the feasible region. + """ + group = table['group'] + seen, unseen, foreign = group == 'seen', group == 'unseen', group == 'foreign' + known = ~foreign + margin, loo_margin = table['s1'] - table['s2'], table['loo_s1'] - table['loo_s2'] + rows = [] + for t in T_GRID: + hi = table['s1'] >= t + loo_hi = table['loo_s1'] >= t + for m in M_GRID: + acc = hi & (margin >= m) + loo = loo_hi & (loo_margin >= m) + far_unseen = float(loo[unseen].mean()) if unseen.any() else 0. + far_foreign = float(acc[foreign].mean()) if foreign.any() else 0. + a = acc[known] + precision = (a & table['correct'][known]).sum() / max(a.sum(), 1) + if far_unseen > max_false_accept or far_foreign > max_false_accept or precision < min_precision: + continue + good = acc & table['correct'] + parts = [good[g].mean() for g in (seen, unseen) if g.any()] + rows.append((float(np.mean(parts)) if parts else 0., far_unseen + far_foreign, float(t), float(m))) + if not rows: + return {'similarity': None, 'margin': None, 'score': 0., 'feasible': False, + 'max_false_accept': max_false_accept, 'min_precision': min_precision} + best = max(r[0] for r in rows) + tied = [r for r in rows if r[0] >= best - tolerance] + low = min(r[1] for r in tied) + tied = [r for r in tied if r[1] <= low + 1e-12] + ts = sorted({r[2] for r in tied}) + t = ts[(len(ts) - 1) // 2] + ms = sorted(r[3] for r in tied if r[2] == t) + m = ms[(len(ms) - 1) // 2] + score = next(r[0] for r in tied if r[2] == t and r[3] == m) + return {'similarity': t, 'margin': m, 'score': score, 'feasible': True, + 'max_false_accept': max_false_accept, 'min_precision': min_precision, + 'tied_similarity_range': [ts[0], ts[-1]]} + + +def open_set_curve(table, points=(.005, .01, .02, .05)): + """Known accepted-correct rate at several open-set false-accept limits.""" + rows = [] + for limit in points: + choice = select_thresholds(table, max_false_accept=limit) + row = {'max_false_accept': limit, **choice} + if choice['feasible']: + row['summary'] = summarize(table, choice['similarity'], choice['margin']) + rows.append(row) + return rows diff --git a/bdon_vision/train_kind_encoder.py b/bdon_vision/train_kind_encoder.py new file mode 100644 index 0000000..11613ca --- /dev/null +++ b/bdon_vision/train_kind_encoder.py @@ -0,0 +1,487 @@ +"""Train a per-kind open-set artwork encoder on scene crop banks. + +Queries are artwork-window crops from complete synthetic scenes; references +are catalog artwork prepared with ``kind_geometry.reference_image``. Only +training identities (``id % 7 != 0``) enter either branch. Held-out +identities and foreign artwork are used only by the development evaluation, +where held-out cards are added to the gallery as plain vectors. + +Objective per step (training references are encoded with gradients, or +without gradients and refreshed periodically with ``--ref-mode frozen``): + +* margin softmax over training references plus a fixed "unknown" logit: + the true card must beat every other card and the acceptance level; +* leave-true-out softmax: with the true card removed, the unknown logit must + beat every remaining card (open-set rejection); +* cosine consistency between each query and its reference; +* optional cosine distillation towards the initial model's embeddings, which + keeps held-out identities from drifting while training identities adapt. + +Other cards of the same character get an additive logit bonus in both +softmax terms, so the closest confusers are kept further apart. BatchNorm +statistics stay frozen so query and reference branches share one forward. +""" +import argparse +import hashlib +import json +import math +import time +from pathlib import Path + +import numpy as np +import torch +from PIL import Image +from torch.nn import functional as F + +from .assets import read, write +from .kind_corpus import CONTEXT_MARGIN, load_bank +from .kind_encoder import build_model, freeze_batchnorm, kind_cards, reference_source, references +from .kind_geometry import INPUT, OVERLAYS, cross_reference_image, cross_window +from .kind_metrics import open_set_curve, retrieval_table, select_thresholds, summarize + + +def seed_all(seed): + import random + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def _uniform(n, low, high, device): + return torch.empty(n, device=device).uniform_(low, high) + + +def warp(x, out_hw, base, shift, scale, aspect): + """Batched crop of the central ``base`` fraction of each image. + + ``base`` is a fraction of the input size, either one number or an + ``(x, y)`` pair. ``shift`` is the maximum translation as a fraction of the + cropped window, ``scale`` and ``aspect`` the relative size and aspect jitter. + """ + n = x.shape[0] + bx, by = (base, base) if isinstance(base, (int, float)) else base + s = _uniform(n, 1 - scale, 1 + scale, x.device) + a = _uniform(n, -aspect, aspect, x.device) + theta = torch.zeros(n, 2, 3, device=x.device) + theta[:, 0, 0] = bx * s * (1 + a) + theta[:, 1, 1] = by * s * (1 - a) + theta[:, :, 2] = (torch.rand(n, 2, device=x.device) * 2 - 1) * 2 * shift * torch.tensor([bx, by], device=x.device) + grid = F.affine_grid(theta, (n, x.shape[1], *out_hw), align_corners=False) + return F.grid_sample(x, grid, mode='bilinear', padding_mode='border', align_corners=False) + + +def photometric(x, strength): + n = x.shape[0] + dev = x.device + gray = x.mean(1, keepdim=True) + sat = _uniform(n, 1 - .25 * strength, 1 + .2 * strength, dev).view(n, 1, 1, 1) + x = gray + (x - gray) * sat + contrast = _uniform(n, 1 - .2 * strength, 1 + .2 * strength, dev).view(n, 1, 1, 1) + bright = _uniform(n, 1 - .2 * strength, 1 + .15 * strength, dev).view(n, 1, 1, 1) + x = ((x - .5) * contrast + .5) * bright + tint = 1 + (torch.rand(n, 3, 1, 1, device=dev) * 2 - 1) * .04 * strength + x = (x * tint).clamp(0, 1) + gamma = torch.exp(_uniform(n, -.15 * strength, .15 * strength, dev)).view(n, 1, 1, 1) + return x.clamp(1e-4, 1) ** gamma + + +def resolution(x, low, prob, chunks=4): + """Downscale-then-upscale random chunks of the batch (small or blurry tiles).""" + n, _, h, w = x.shape + out = [] + for part in x.chunk(chunks): + if torch.rand(()) < prob: + f = float(torch.empty(()).uniform_(low, 1.)) + small = F.interpolate(part, size=(max(8, round(h * f)), max(8, round(w * f))), mode='area') + part = F.interpolate(small, size=(h, w), mode='bilinear', align_corners=False) + out.append(part) + return torch.cat(out) + + +def occlude(x, kind, overlay_prob, erase_prob, edge_prob): + """Display-layer occlusion: UI overlay zones, random rectangles and cut edges.""" + n, _, h, w = x.shape + dev = x.device + donor = x.roll(1, 0) + x = x.clone() + for x0, y0, x1, y1 in OVERLAYS[kind]: + pick = torch.rand(n, device=dev) + r0, r1 = int(y0 * h), int(math.ceil(y1 * h)) + c0, c1 = int(x0 * w), int(math.ceil(x1 * w)) + solid = pick < overlay_prob * .5 + patch = (pick >= overlay_prob * .5) & (pick < overlay_prob) + if solid.any(): + x[solid, :, r0:r1, c0:c1] = torch.rand(int(solid.sum()), 3, 1, 1, device=dev) + if patch.any(): + x[patch, :, r0:r1, c0:c1] = donor[patch, :, r0:r1, c0:c1] + for i in (torch.rand(n) < erase_prob).nonzero().flatten().tolist(): + area = float(torch.empty(()).uniform_(.03, .14)) + ratio = math.exp(float(torch.empty(()).uniform_(-1., 1.))) + eh = min(h, max(2, int(round(math.sqrt(area * h * w / ratio))))) + ew = min(w, max(2, int(round(math.sqrt(area * h * w * ratio))))) + top = int(torch.randint(0, h - eh + 1, ())) + left = int(torch.randint(0, w - ew + 1, ())) + if torch.rand(()) < .5: + x[i, :, top:top + eh, left:left + ew] = torch.rand(3, 1, 1, device=dev) + else: + x[i, :, top:top + eh, left:left + ew] = torch.rand(3, eh, ew, device=dev) + for i in (torch.rand(n) < edge_prob).nonzero().flatten().tolist(): + side = int(torch.randint(0, 4, ())) + f = float(torch.empty(()).uniform_(.05, .3)) + fill = torch.rand(3, 1, 1, device=dev) + if side == 0: + x[i, :, :int(h * f)] = fill + elif side == 1: + x[i, :, h - int(h * f):] = fill + elif side == 2: + x[i, :, :, :int(w * f)] = fill + else: + x[i, :, :, w - int(w * f):] = fill + return x + + +def augment_queries(ctx, kind, cfg, window=(1., 1.)): + """Augmented encoder queries from context crops. + + ``window`` is the central fraction (x, y) of the artwork window to keep; + other-kind crops use the central part matching this kind's aspect ratio. + """ + h, w = INPUT[kind] + base = tuple(f / (1 + 2 * CONTEXT_MARGIN) for f in window) + x = warp(ctx, (h, w), base, cfg['shift'], cfg['scale'], cfg['aspect']) + x = photometric(x, cfg['color']) + x = resolution(x, cfg['min_resolution'], cfg['resolution_prob']) + x = occlude(x, kind, cfg['overlay_prob'], cfg['erase_prob'], cfg['edge_prob']) + x = x + torch.randn_like(x) * float(torch.empty(()).uniform_(0, cfg['noise'])) + return x.clamp(0, 1) + + +def augment_references(ref, kind): + h, w = INPUT[kind] + x = warp(ref, (h, w), 1., .02, .03, .02) + x = photometric(x, .3) + x = resolution(x, .5, .3, chunks=2) + return x.clamp(0, 1) + + +@torch.no_grad() +def embed(model, array, batch=512): + """Embed uint8 NHWC (numpy or tensor) or float NCHW tensors in eval mode.""" + device = next(model.parameters()).device + was = model.training + model.eval() + out = [] + for start in range(0, len(array), batch): + part = array[start:start + batch] + if isinstance(part, np.ndarray): + part = torch.from_numpy(np.ascontiguousarray(part)) + if part.dtype == torch.uint8: + part = part.to(device).permute(0, 3, 1, 2).float().div_(255) + out.append(model(part.to(device)).float().cpu()) + model.train(was) + if was: + freeze_batchnorm(model) + return torch.cat(out).numpy() + + +def evaluate_bank(model, bank, gallery_inputs, held, thresholds=None): + """Retrieval summary with thresholds chosen on this bank; if ``thresholds`` + (chosen elsewhere) are given, also report the bank at those thresholds.""" + gallery = embed(model, gallery_inputs) + queries = embed(model, bank['art']) + table = retrieval_table(queries, gallery, bank['labels'], held) + choice = select_thresholds(table) + result = {'selected_thresholds': choice, 'top1_only': summarize(table, 2., 1.)} + if choice['feasible']: + result['at_selected'] = summarize(table, choice['similarity'], choice['margin']) + if thresholds and thresholds.get('feasible'): + at = summarize(table, thresholds['similarity'], thresholds['margin']) + parts = [at[g]['accepted_correct_rate'] for g in ('seen', 'unseen') if at[g]['count']] + result['at_validation_thresholds'] = at + result['score_at_validation_thresholds'] = float(np.mean(parts)) if parts else 0. + return result, table + + +class QueryPool: + """Training queries of one source kind plus the matching reference inputs. + + The own kind uses whole artwork windows. With ``--cross-kind`` the training + identities of the other kind become extra identities: queries keep the + central part of their artwork window with this kind's aspect ratio and the + references are cut the same way, so identities and degradations both grow + without touching held-out cards. + """ + + def __init__(self, args, catalog, kind, source_kind, device, offset=0): + self.kind, self.source_kind, self.device = kind, source_kind, device + self.cards = kind_cards(catalog, source_kind) + held = np.array([c['id'] % 7 == 0 for c in self.cards]) + self.train_ids = np.flatnonzero(~held) + contexts, labels, self.fingerprints = [], [], [] + self.dropped = {'held_out_identity': 0, 'foreign': 0, 'window_partly_outside': 0} + for path in args.train: + arrays, local, meta, manifest = load_bank(path, catalog, source_kind, ('context',)) + if set(manifest['profiles']) - {'train'}: + raise ValueError(f'Training bank {path} contains non-train profiles {manifest["profiles"]}') + keep = local >= 0 + self.dropped['foreign'] += int((~keep).sum()) + held_q = keep & held[np.maximum(local, 0)] + self.dropped['held_out_identity'] += int(held_q.sum()) + keep &= ~held_q + partial = meta['window_inside'] < args.min_inside + self.dropped['window_partly_outside'] += int((keep & partial).sum()) + keep &= ~partial + idx = np.flatnonzero(keep) + contexts.append(torch.from_numpy(np.ascontiguousarray(arrays['context'][idx]))) + labels.append(local[idx]) + self.fingerprints.append(manifest['source_fingerprint']) + self.context = torch.cat(contexts) + self.label = torch.from_numpy(np.concatenate(labels)) + if held[self.label.numpy()].any(): + raise ValueError('Held-out identity in training queries') + self.on_gpu = device.type == 'cpu' or self.context.numel() < args.gpu_bank_bytes + self.context = self.context.to(device) if self.on_gpu else self.context.pin_memory() + self.order = torch.argsort(self.label) + self.counts = torch.bincount(self.label, minlength=len(self.cards)) + self.offsets = torch.cumsum(self.counts, 0) - self.counts + self.present = torch.tensor([int(k) for k in self.train_ids if self.counts[k] > 0]) + self.position = torch.full((len(self.cards),), -1, dtype=torch.long) + self.position[torch.from_numpy(self.train_ids)] = torch.arange(len(self.train_ids)) + offset + if source_kind == kind: + self.window = (1., 1.) + inputs = references(args.data, self.cards, kind)[self.train_ids] + else: + self.window = cross_window(kind, source_kind) + rows = [] + for k in self.train_ids: + with Image.open(reference_source(args.data, self.cards[k])) as image: + rows.append(np.asarray(cross_reference_image(kind, source_kind, image), np.float32) / 255.) + inputs = np.stack(rows).transpose(0, 3, 1, 2) + self.ref_inputs = torch.from_numpy(np.ascontiguousarray(inputs)).float().to(device) + + def sample(self, n, cfg): + """``n`` identity-balanced augmented queries and their reference positions.""" + ident = self.present[torch.randint(len(self.present), (n,))] + offset = (torch.rand(n) * self.counts[ident]).long().clamp_max(self.counts[ident] - 1) + rows = self.order[self.offsets[ident] + offset] + batch = self.context[rows.to(self.context.device)] if self.on_gpu else self.context[rows] + batch = batch.to(self.device, non_blocking=True).permute(0, 3, 1, 2).float().div_(255) + return augment_queries(batch, self.kind, cfg, self.window), self.position[ident].to(self.device) + + +def train(args): + device = torch.device(args.device) + if device.type == 'cuda' and not torch.cuda.is_available(): + raise RuntimeError('GPU required') + kind = args.kind + out = args.output + out.mkdir(parents=True, exist_ok=True) + seed_all(args.seed) + torch.set_num_threads(8) + catalog = read(args.data / 'catalog.json')['cards'] + cards = kind_cards(catalog, kind) + held = np.array([c['id'] % 7 == 0 for c in cards]) + train_ids = np.flatnonzero(~held) + ref_inputs = torch.from_numpy(references(args.data, cards, kind)).float().to(device) + + pools = [QueryPool(args, catalog, kind, kind, device)] + if args.cross_kind: + other = 'snap' if kind == 'member' else 'member' + pools.append(QueryPool(args, catalog, kind, other, device, offset=len(pools[0].train_ids))) + own = pools[0] + missing = [int(cards[k]['id']) for k in train_ids if own.counts[k] == 0] + + dev_sets = {} + for name, path in [('validation', args.validation), ('stress', args.stress)]: + if path is None: + continue + arrays, local, meta, manifest = load_bank(path, catalog, kind, ('art',)) + keep = meta['window_inside'] >= args.min_inside + dev_sets[name] = {'art': np.ascontiguousarray(arrays['art'][keep]), 'labels': local[keep], + 'fingerprint': manifest['source_fingerprint'], 'profiles': manifest['profiles']} + + model = build_model(kind) + state = torch.load(args.initial, map_location='cpu', weights_only=True) + model.load_state_dict(state) + model = model.to(device).train() + freeze_batchnorm(model) + teacher = None + if args.kd_weight: + teacher = build_model(kind) + teacher.load_state_dict(state) + teacher = teacher.to(device).eval().requires_grad_(False) + head = list(model.projection.parameters()) + backbone = [p for n, p in model.named_parameters() if not n.startswith('projection.')] + optimizer = torch.optim.AdamW([{'params': backbone, 'lr': args.lr}, {'params': head, 'lr': args.head_lr}], + weight_decay=args.weight_decay) + base_lrs = [args.lr, args.head_lr] + cfg = {'shift': args.shift, 'scale': args.scale_jitter, 'aspect': .03, 'color': 1., 'min_resolution': args.min_resolution, + 'resolution_prob': .5, 'overlay_prob': args.overlay_prob, 'erase_prob': args.erase_prob, + 'edge_prob': args.edge_prob, 'noise': .02} + config = {k: (str(v) if isinstance(v, Path) else ([str(p) for p in v] if isinstance(v, list) else v)) + for k, v in vars(args).items()} + config.update({'augmentation': cfg, 'train_queries': {p.source_kind: int(len(p.label)) for p in pools}, + 'dropped_queries': {p.source_kind: p.dropped for p in pools}, + 'reference_identities': {p.source_kind: [int(p.cards[k]['id']) for k in p.train_ids] for p in pools}, + 'train_identities': [int(cards[k]['id']) for k in train_ids], + 'held_out_identities': [int(c['id']) for c in cards if c['id'] % 7 == 0], + 'train_identities_without_queries': missing, + 'train_fingerprints': {p.source_kind: p.fingerprints for p in pools}, + 'initial_sha256': hashlib.sha256(args.initial.read_bytes()).hexdigest(), + 'bank_on_gpu': all(p.on_gpu for p in pools)}) + write(out / 'config.json', config) + print(json.dumps({k: config[k] for k in ['train_queries', 'dropped_queries', 'bank_on_gpu']}), flush=True) + + all_refs = torch.cat([p.ref_inputs for p in pools]) + names = [p.cards[k].get('name') for p in pools for k in p.train_ids] + same_name = torch.tensor([[a is not None and a == b for b in names] for a in names], device=device) + shares = [1.] if len(pools) == 1 else [args.own_fraction, 1 - args.own_fraction] + sizes = [round(args.batch * f) for f in shares] + sizes[-1] = args.batch - sum(sizes[:-1]) + curve = [] + started = time.perf_counter() + + def checkpoint(step, loss_value): + row = {'step': step, 'train_loss': loss_value, 'elapsed_s': round(time.perf_counter() - started, 1)} + choice = None + for name, bank in dev_sets.items(): + result, _ = evaluate_bank(model, bank, ref_inputs, held, choice) + row[name] = result + if name == 'validation': + choice = result['selected_thresholds'] + curve.append(row) + write(out / 'learning-curve.json', {'checkpoints': curve}) + torch.save({k: v.detach().cpu() for k, v in model.state_dict().items()}, out / f'step-{step:06d}.pt') + brief = {name: {'select': row[name]['selected_thresholds'], + 'score_at_validation_thresholds': row[name].get('score_at_validation_thresholds'), + 'top1_seen': row[name]['top1_only']['seen']['top1'], + 'top1_unseen': row[name]['top1_only']['unseen']['top1']} for name in dev_sets} + print(json.dumps({'checkpoint': step, **brief}), flush=True) + + if args.eval_every: + checkpoint(0, None) + for step in range(args.steps): + warm = min(1., (step + 1) / args.warmup) + cosine = .05 + .95 * .5 * (1 + math.cos(math.pi * step / args.steps)) + for group, lr in zip(optimizer.param_groups, base_lrs): + group['lr'] = lr * warm * cosine + parts = [p.sample(n, cfg) for p, n in zip(pools, sizes) if n] + queries = torch.cat([q for q, _ in parts]) + target = torch.cat([t for _, t in parts]) + if args.ref_mode == 'grad': + ref_batch = augment_references(all_refs, kind) + emb = model(torch.cat([queries, ref_batch])) + q, r = emb[:len(queries)], emb[len(queries):] + else: + if step % args.ref_every == 0: + with torch.no_grad(): + frozen_refs = model(all_refs) + q, r = model(queries), frozen_refs + sims = q @ r.T + onehot = F.one_hot(target, len(all_refs)).bool() + unknown = torch.full((len(q), 1), args.unknown_level, device=device) + # Cards of the same character are the closest confusers; their logits + # get an extra bonus so the loss keeps a wider gap to them. + hard = (same_name[target] & ~onehot).float() * args.same_name_margin + logits = torch.cat([sims - args.margin * onehot.float() + hard, unknown], 1) / args.temperature + loss_id = F.cross_entropy(logits, target) + loo = torch.cat([(sims + hard).masked_fill(onehot, -1e4), unknown], 1) / args.temperature + loss_open = F.cross_entropy(loo, torch.full_like(target, len(all_refs))) + loss_cos = (1 - sims[onehot]).mean() + loss = loss_id + args.open_weight * loss_open + args.cos_weight * loss_cos + loss_kd = torch.zeros((), device=device) + if teacher is not None: + # Stay close to the initial embedding so identities that never + # appear in training keep their place in the embedding space. + with torch.no_grad(): + q0 = teacher(queries) + loss_kd = (1 - (q * q0).sum(1)).mean() + if args.ref_mode == 'grad': + with torch.no_grad(): + r0 = teacher(ref_batch) + loss_kd = loss_kd + (1 - (r * r0).sum(1)).mean() + loss = loss + args.kd_weight * loss_kd + optimizer.zero_grad(set_to_none=True) + loss.backward() + optimizer.step() + if step == 0 or (step + 1) % 50 == 0: + status = {'step': step + 1, 'steps': args.steps, 'loss': loss.item(), 'identity': loss_id.item(), + 'open': loss_open.item(), 'cos': loss_cos.item(), 'kd': loss_kd.item(), + 'elapsed_s': round(time.perf_counter() - started, 1), + 'gpu_max_allocated_gib': round(torch.cuda.max_memory_allocated() / 2 ** 30, 2) if device.type == 'cuda' else 0} + write(out / 'progress.json', status) + print(json.dumps(status), flush=True) + if args.eval_every and ((step + 1) % args.eval_every == 0 or step + 1 == args.steps): + checkpoint(step + 1, loss.item()) + torch.save({k: v.detach().cpu() for k, v in model.state_dict().items()}, out / 'last.pt') + if curve: + best = select_checkpoint(curve) + write(out / 'selection.json', best) + print(json.dumps({'selected': best}), flush=True) + write(out / 'progress.json', {'step': args.steps, 'steps': args.steps, 'done': True, + 'elapsed_s': round(time.perf_counter() - started, 1)}) + + +def select_checkpoint(curve): + """Highest validation score (mean of seen and unseen accepted-correct rates + at the open-set-constrained thresholds chosen on validation), rounded to + 0.001; ties go to the higher stress score at those same thresholds, then + to the earlier step.""" + def key(row): + stress = row.get('stress', {}).get('score_at_validation_thresholds', 0.) + return round(row['validation']['selected_thresholds']['score'], 3), round(stress, 4), -row['step'] + row = max(curve, key=key) + return {'rule': 'max validation score = mean(seen, unseen) accepted-correct rate at thresholds with ' + 'unseen leave-true-out and foreign false accepts <= 1% and accepted precision >= 99.5%; ' + 'ties broken by stress score at the validation thresholds, then earlier step', + 'step': row['step'], 'validation_score': row['validation']['selected_thresholds']['score'], + 'stress_score': row.get('stress', {}).get('score_at_validation_thresholds'), + 'thresholds': row['validation']['selected_thresholds']} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--kind', choices=['member', 'snap'], required=True) + p.add_argument('--data', type=Path, required=True) + p.add_argument('--train', type=Path, nargs='+', required=True, help='crop bank roots (train profile only)') + p.add_argument('--validation', type=Path, required=True) + p.add_argument('--stress', type=Path) + p.add_argument('--initial', type=Path, required=True, help='state dict of the shared encoder') + p.add_argument('--output', type=Path, required=True) + p.add_argument('--steps', type=int, default=3000) + p.add_argument('--batch', type=int, default=256) + p.add_argument('--lr', type=float, default=1.5e-4) + p.add_argument('--head-lr', type=float, default=4e-4) + p.add_argument('--weight-decay', type=float, default=1e-4) + p.add_argument('--warmup', type=int, default=100) + p.add_argument('--temperature', type=float, default=.08) + p.add_argument('--margin', type=float, default=.1) + p.add_argument('--unknown-level', type=float, default=.55) + p.add_argument('--open-weight', type=float, default=.5) + p.add_argument('--cos-weight', type=float, default=.15) + p.add_argument('--same-name-margin', type=float, default=.05) + p.add_argument('--kd-weight', type=float, default=0., + help='cosine distillation to the initial model on query (and reference) embeddings') + p.add_argument('--ref-mode', choices=['grad', 'frozen'], default='grad', + help='encode training references with gradients every step, or without gradients every --ref-every steps') + p.add_argument('--ref-every', type=int, default=20) + p.add_argument('--cross-kind', action='store_true', + help='add the training identities of the other kind, cut to this aspect ratio') + p.add_argument('--own-fraction', type=float, default=.6, help='share of each batch from the own kind') + p.add_argument('--shift', type=float, default=.05) + p.add_argument('--scale-jitter', type=float, default=.07) + p.add_argument('--min-resolution', type=float, default=.35) + p.add_argument('--overlay-prob', type=float, default=.5) + p.add_argument('--erase-prob', type=float, default=.3) + p.add_argument('--edge-prob', type=float, default=.1) + p.add_argument('--min-inside', type=float, default=.999) + p.add_argument('--eval-every', type=int, default=250) + p.add_argument('--gpu-bank-bytes', type=float, default=4e9) + p.add_argument('--seed', type=int, default=52817) + p.add_argument('--device', default='cuda') + train(p.parse_args()) + + +if __name__ == '__main__': + main() diff --git a/tests/test_kind_encoder.py b/tests/test_kind_encoder.py new file mode 100644 index 0000000..fb005b8 --- /dev/null +++ b/tests/test_kind_encoder.py @@ -0,0 +1,217 @@ +"""Per-kind encoder geometry, crop banks and open-set metrics on synthetic inputs.""" +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +from bdon_vision import kind_geometry as geo +from bdon_vision.kind_metrics import retrieval_table, select_thresholds, summarize + +HAS_TORCH = importlib.util.find_spec('torch') is not None and importlib.util.find_spec('torchvision') is not None + + +class GeometryTests(unittest.TestCase): + def test_art_window_insets_six_logical_units(self): + self.assertEqual(geo.art_window('member', [10, 20, 224, 294]), (16., 26., 212., 282.)) + x, y, w, h = geo.art_window('snap', [0, 0, 652, 368]) + self.assertAlmostEqual(x, 12.) + self.assertAlmostEqual(y, 12.) + self.assertAlmostEqual(w, 628.) + self.assertAlmostEqual(h, 344.) + + def test_snap_reference_uses_centered_window_aspect(self): + left, top, right, bottom = geo.center_crop_box(512, 288, 314 / 172) + self.assertEqual((left, right), (0., 512.)) + self.assertAlmostEqual(bottom - top, 512 * 172 / 314) + self.assertAlmostEqual(top, 288 - bottom) + + def test_reference_sizes_and_center_content(self): + snap = np.zeros((288, 512, 3), np.uint8) + snap[:, :] = (0, 0, 255) + snap[4:284] = (255, 0, 0) + out = np.asarray(geo.reference_image('snap', Image.fromarray(snap))) + self.assertEqual(out.shape, (128, 224, 3)) + self.assertTrue((out[2:-2, :, 0] > 200).all()) + square = Image.new('RGB', (300, 300), (0, 255, 0)) + member = geo.reference_image('member', square) + self.assertEqual(member.size, (128, 160)) + + def test_member_fitted_canonical_is_not_refitted(self): + canonical = np.zeros((282, 212, 3), np.uint8) + canonical[:, :106] = 255 + out = np.asarray(geo.reference_image('member', Image.fromarray(canonical))) + self.assertGreater(out[:, :60].mean(), 250) + self.assertLess(out[:, 68:].mean(), 5) + + def test_query_crop_samples_the_artwork_window(self): + image = np.zeros((400, 600, 3), np.uint8) + image[:, :] = (255, 0, 0) # BGR blue frame and background + bbox = [100, 50, 326, 184] + x, y, w, h = geo.art_window('snap', bbox) + image[int(y):int(y + h), int(x):int(x + w)] = (0, 0, 255) # BGR red art + crop = geo.query_crop(image, 'snap', bbox) + self.assertEqual(crop.shape, (128, 224, 3)) + self.assertGreater(crop[2:-2, 2:-2, 0].mean(), 250) # RGB red + wide = geo.query_crop(image, 'snap', bbox, margin=.125) + self.assertEqual(wide.shape, (160, 280, 3)) + self.assertGreater(wide[0, :, 2].mean(), 200) # context reaches the blue frame + + def test_area_resampling_matches_bilinear_on_flat_regions(self): + image = np.zeros((800, 1200, 3), np.uint8) + bbox = [100, 100, 652, 368] # twice the logical snap tile size + x, y, w, h = geo.art_window('snap', bbox) + image[int(y):int(y + h), int(x):int(x + w)] = (0, 0, 255) + linear = geo.query_crop(image, 'snap', bbox).astype(int) + area = geo.query_crop(image, 'snap', bbox, resample='area').astype(int) + self.assertEqual(area.shape, linear.shape) + self.assertLessEqual(np.abs(linear[2:-2, 2:-2] - area[2:-2, 2:-2]).max(), 2) + + def test_overlay_mask_covers_corners_only(self): + for kind in geo.KINDS: + h, w = geo.INPUT[kind] + mask = geo.overlay_mask(kind, h, w) + self.assertTrue(mask[0, 0] and mask[-1, 0] and mask[-1, -1]) + self.assertFalse(mask[h // 2, w // 2]) + self.assertLess(mask.mean(), .3) + + def test_cross_kind_windows_keep_the_target_aspect(self): + fx, fy = geo.cross_window('member', 'snap') + self.assertEqual(fy, 1.) + self.assertAlmostEqual(314 * fx / (172 * fy), 212 / 282) + fx, fy = geo.cross_window('snap', 'member') + self.assertEqual(fx, 1.) + self.assertAlmostEqual(212 * fx / (282 * fy), 314 / 172) + + def test_cross_kind_reference_takes_the_center_of_the_window(self): + snap = np.zeros((288, 512, 3), np.uint8) + snap[:, :] = (0, 255, 0) + snap[:, 150:362] = (255, 0, 0) + out = np.asarray(geo.cross_reference_image('member', 'snap', Image.fromarray(snap))) + self.assertEqual(out.shape, (160, 128, 3)) + self.assertGreater(out[:, 4:-4, 0].mean(), 250) + member = geo.cross_reference_image('snap', 'member', Image.new('RGB', (300, 300), (0, 0, 255))) + self.assertEqual(member.size, (224, 128)) + + +class MetricTests(unittest.TestCase): + def setUp(self): + rng = np.random.default_rng(3) + gallery = rng.normal(size=(6, 16)) + self.gallery = gallery / np.linalg.norm(gallery, axis=1, keepdims=True) + labels = np.repeat(np.arange(6), 5) + queries = self.gallery[labels] + rng.normal(scale=.05, size=(30, 16)) + foreign = rng.normal(size=(4, 16)) + queries = np.concatenate([queries, foreign]) + self.queries = queries / np.linalg.norm(queries, axis=1, keepdims=True) + self.labels = np.concatenate([labels, [-1] * 4]) + self.held = np.array([False, False, False, False, True, True]) + + def test_groups_and_leave_true_out(self): + table = retrieval_table(self.queries, self.gallery, self.labels, self.held) + self.assertEqual(list(np.unique(table['group'])), ['foreign', 'seen', 'unseen']) + self.assertTrue(table['correct'][:30].all()) + self.assertTrue((table['loo_pred'][:30] != self.labels[:30]).all()) + self.assertTrue((table['loo_s1'][:30] < table['s1'][:30]).all()) + + def test_selected_thresholds_respect_open_set_limits(self): + table = retrieval_table(self.queries, self.gallery, self.labels, self.held) + choice = select_thresholds(table, max_false_accept=0.) + self.assertTrue(choice['feasible']) + summary = summarize(table, choice['similarity'], choice['margin']) + self.assertEqual(summary['unseen']['open_set_false_accept'], 0.) + self.assertEqual(summary['foreign']['false_accept'], 0.) + self.assertEqual(summary['known']['accepted_precision'], 1.) + self.assertEqual(summary['known']['accepted_correct_rate'], 1.) + + def test_identical_cards_cannot_be_accepted(self): + gallery = np.concatenate([self.gallery[:1], self.gallery[:1], self.gallery[1:]]) + labels = np.zeros(5, int) + table = retrieval_table(np.repeat(gallery[:1], 5, 0), gallery, labels, np.zeros(len(gallery), bool)) + summary = summarize(table, .5, .01) + self.assertEqual(summary['known']['accept_rate'], 0.) + + +class CropBankTests(unittest.TestCase): + def test_bank_labels_foreign_and_partial_tiles(self): + from bdon_vision.kind_corpus import build, load_bank + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + data, scenes, out = tmp / 'data', tmp / 'scenes', tmp / 'banks' + data.mkdir() + scenes.mkdir() + cards = [{'kind': 'snap', 'id': i, 'asset_id': i, 'file': f'snap-{i}.png'} for i in (1, 7)] + cards.append({'kind': 'member', 'id': 1, 'asset_id': 1, 'file': 'member-1.png'}) + (data / 'catalog.json').write_text(json.dumps({'cards': cards}), encoding='utf-8') + image = np.full((300, 800, 3), 40, np.uint8) + image[20:204, 20:346] = (0, 0, 200) + image[20:204, 400:726] = (0, 200, 0) + cv2.imwrite(str(scenes / 'a.png'), image) + truth = {'complete': True, 'profile': 'train', 'screenshots': [{ + 'file': 'a.png', 'locale': 'ja', + 'cards': [{'kind': 'snap', 'id': 7, 'bbox': [20, 20, 326, 184]}, + {'kind': 'snap', 'id': 1, 'bbox': [700, 200, 326, 184]}], + 'foreign_cards': [{'kind': 'snap', 'bbox': [400, 20, 326, 184], 'foreign_method': 'mix'}]}]} + (scenes / 'truth.json').write_text(json.dumps(truth), encoding='utf-8') + manifest = build(data, [scenes], out, 'snap', workers=1) + self.assertEqual(manifest['count'], 2) + self.assertEqual(manifest['foreign'], 1) + self.assertEqual(manifest['skipped'], {'window_outside_image': 1}) + arrays, labels, meta, _ = load_bank(out, cards, 'snap', ('art', 'context', 'baseline')) + self.assertEqual(list(labels), [1, -1]) + self.assertEqual(arrays['art'].shape[1:], (128, 224, 3)) + self.assertEqual(arrays['baseline'].shape[1:], (160, 128, 3)) + self.assertGreater(arrays['art'][0, ..., 0].mean(), 190) + self.assertGreater(arrays['art'][1, ..., 1].mean(), 190) + self.assertEqual(list(meta['foreign']), [False, True]) + self.assertEqual(list(meta['foreign_method']), ['', 'mix']) + + def test_stride_counts_only_screens_of_the_kind_and_arrays_are_optional(self): + from bdon_vision.kind_corpus import build, load_bank + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + data, scenes, out = tmp / 'data', tmp / 'scenes', tmp / 'banks' + data.mkdir() + scenes.mkdir() + cards = [{'kind': 'snap', 'id': 1, 'asset_id': 1, 'file': 'snap-1.png'}, + {'kind': 'member', 'id': 1, 'asset_id': 1, 'file': 'member-1.png'}] + (data / 'catalog.json').write_text(json.dumps({'cards': cards}), encoding='utf-8') + cv2.imwrite(str(scenes / 'a.png'), np.full((300, 400, 3), 90, np.uint8)) + rows = [] + for i in range(6): # member and snap screens alternate + kind = 'member' if i % 2 == 0 else 'snap' + bbox = [10, 10, 224, 294] if kind == 'member' else [10, 10, 326, 184] + rows.append({'file': 'a.png', 'cards': [{'kind': kind, 'id': 1, 'bbox': bbox}]}) + (scenes / 'truth.json').write_text(json.dumps({'complete': True, 'profile': 'train', 'screenshots': rows}), + encoding='utf-8') + manifest = build(data, [scenes], out, 'snap', workers=1, arrays=('context',), stride=2) + self.assertEqual(manifest['count'], 2) + self.assertEqual(manifest['arrays'], ['context']) + arrays, labels, meta, _ = load_bank(out, cards, 'snap', ('context',)) + self.assertEqual(arrays['context'].shape[1:], (160, 280, 3)) + self.assertEqual(list(meta['screenshot']), [1, 5]) + self.assertFalse((out / 'snap' / 'art.npy').exists()) + + +@unittest.skipUnless(HAS_TORCH, 'torch not installed') +class EncoderTests(unittest.TestCase): + def test_kind_encoders_share_the_shared_encoder_state_layout(self): + import torch + from bdon_vision.kind_encoder import build_model, pooled_grid + from bdon_vision.train import CardEncoder + state = CardEncoder().state_dict() + for kind in geo.KINDS: + self.assertEqual(pooled_grid(kind), (2, 2)) + model = build_model(kind) + model.load_state_dict(state) + out = model.eval()(torch.rand(3, 3, *geo.INPUT[kind])) + self.assertEqual(tuple(out.shape), (3, 128)) + self.assertTrue(torch.allclose(out.norm(dim=1), torch.ones(3), atol=1e-5)) + + +if __name__ == '__main__': + unittest.main() From 2a8f363d723bb11845dbfffdeae33d6a6abfc5ed Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:10:10 +0800 Subject: [PATCH 4/7] feat: add per-kind field and card-rank readers Member and snap cards get separate parameter-field readers (106 and 101 classes) and six-class rank-icon classifiers. Crops are defined relative to the card tile rectangle, so the same geometry serves crop banks, training and inference. - kind_fields: crop geometry, labels, acceptance rules, misread sweep and scene crop-bank builder (catalog and foreign-art cards) - kind_hardcrops: rendered acquisition-degraded crops with optional UI-panel occlusion labelled by covered glyph or icon pixels - kind_pose: locator pose error per kind and the induced crop shift - train_kind_fields / train_kind_rank: training with per-checkpoint zero-misread threshold selection, ONNX export (opset 17, variable batch) and ONNX Runtime consistency checks - select_kind_models: checkpoint selection on saved checkpoints - evaluate_kind_fields: stratified evaluation on scene sets, crop banks and detected tiles --- bdon_vision/evaluate_kind_fields.py | 513 ++++++++++++++++++++++++++++ bdon_vision/kind_fields.py | 347 +++++++++++++++++++ bdon_vision/kind_hardcrops.py | 259 ++++++++++++++ bdon_vision/kind_pose.py | 261 ++++++++++++++ bdon_vision/select_kind_models.py | 100 ++++++ bdon_vision/train_kind_fields.py | 384 +++++++++++++++++++++ bdon_vision/train_kind_rank.py | 171 ++++++++++ tests/test_kind_fields.py | 211 ++++++++++++ tests/test_kind_pose.py | 90 +++++ 9 files changed, 2336 insertions(+) create mode 100644 bdon_vision/evaluate_kind_fields.py create mode 100644 bdon_vision/kind_fields.py create mode 100644 bdon_vision/kind_hardcrops.py create mode 100644 bdon_vision/kind_pose.py create mode 100644 bdon_vision/select_kind_models.py create mode 100644 bdon_vision/train_kind_fields.py create mode 100644 bdon_vision/train_kind_rank.py create mode 100644 tests/test_kind_fields.py create mode 100644 tests/test_kind_pose.py diff --git a/bdon_vision/evaluate_kind_fields.py b/bdon_vision/evaluate_kind_fields.py new file mode 100644 index 0000000..84154bc --- /dev/null +++ b/bdon_vision/evaluate_kind_fields.py @@ -0,0 +1,513 @@ +"""Development evaluation of the per-kind field and rank readers. + +Compares, on identical crops and identical acceptance rules: + +* new per-kind field readers against the shared field model; +* new per-kind rank classifiers against native-sprite template matching + (``Engine.read_rank``) given the same truth-derived card pose. + +Scenes are read directly from ``truth.json`` and images; crops use the +deployed geometry in ``kind_fields``. The card pose is the truth tile, a +deterministically perturbed truth tile, or the tile of a stored detection +(``kind_pose detect``) paired with the truth tile; cards the detector misses +or assigns to the other kind count as unread. Results are stratified by kind, +locale, display mode, source resolution, acquisition profile, card width, +foreign artwork and held-out identity. Only development data belongs here. +""" +import argparse +import hashlib +import json +from collections import Counter, defaultdict +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +import cv2 +import numpy as np + +from .assets import read, write +from .kind_fields import (FIELD_CONFIDENCE, KINDS, RANK_CONFIDENCE, LOGICAL, MARGIN, RANK_INPUT, RANK_SIDE, crop_field, crop_rank, decide_field, + decide_rank, field_inside, field_label, latent_field, latent_rank, load_bank, misread_sweep, + rank_icon_inside, rank_label, scene_cards, softmax, to_input) +from .kind_pose import detections_for, paired_tiles + +STORED_RANK_MARGIN = MARGIN*RANK_INPUT//RANK_SIDE + + +def session(path, threads=8): + import onnxruntime as ort + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + return ort.InferenceSession(str(path), sess_options=options, providers=['CPUExecutionProvider']) + + +def run(sess, crops, batch=512): + if len(crops) == 0: + return np.zeros((0, sess.get_outputs()[0].shape[1] or 1), np.float64) + out = [softmax(sess.run(None, {sess.get_inputs()[0].name: to_input(crops[i:i+batch])})[0]) + for i in range(0, len(crops), batch)] + return np.concatenate(out) + + +def field_value(kind, decision): + if decision['level'] is not None: + return decision['level'] + if decision['awake_count'] is not None: + return 100+decision['awake_count'] + return 0 + + +def tally(counter, truth, value, prefix='', latent=-1): + """truth/value: 0 = no value; counts correct, wrong, unknown, false visible. + + ``latent`` (card state, analysis only; -1 unknown) splits values emitted + for hidden targets into those matching the card state and misreads. + """ + if truth > 0: + counter[prefix+'visible_total'] += 1 + counter[prefix+'correct'] += int(value == truth) + counter[prefix+'wrong'] += int(value > 0 and value != truth) + counter[prefix+'unknown'] += int(value == 0) + else: + counter[prefix+'hidden_total'] += 1 + counter[prefix+'false_visible'] += int(value > 0) + counter[prefix+'false_visible_latent_match'] += int(value > 0 and value == latent) + + +def finish(counter): + out = dict(counter) + for prefix in sorted({k[:-len('visible_total')] for k in counter if k.endswith('visible_total')}): + total = counter[prefix+'visible_total'] + out[prefix+'coverage'] = round(counter[prefix+'correct']/total, 6) if total else None + out[prefix+'misreads'] = (counter[prefix+'wrong']+counter[prefix+'false_visible'] + - counter[prefix+'false_visible_latent_match']) + return out + + +# ---------------------------------------------------------------- engine rank + +ENGINE = {} + + +def light_engine(data): + """Engine instance carrying only what ``read_rank`` needs.""" + from .engine import Engine + engine = Engine.__new__(Engine) + engine.data = Path(data) + engine.cards = read(engine.data/'catalog.json')['cards'] + engine.art = [cv2.imread(str(engine.data/c.get('match_file', c['file']))) for c in engine.cards] + engine.ranks = {} + for kind, prefix in [('member', 'CardRank'), ('snap', 'snaplimit_')]: + engine.ranks[kind] = [cv2.imread(str(engine.data/f'native/game-ui/sprites/{prefix}{i}.png'), cv2.IMREAD_UNCHANGED) + for i in range(6)] + engine.lookup = {(c['kind'], c['id']): i for i, c in enumerate(engine.cards)} + return engine + + +def art_item(engine, kind, card_id, tile): + """Matched-artwork item for ``Engine.read_rank`` from a tile rectangle.""" + index = engine.lookup.get((kind, card_id)) + if index is None: + return None + x, y, w, h = tile + s = w/LOGICAL[kind][0] + if kind == 'member': + return {'kind': kind, '_index': index, 'bbox': [x+6*s, y+6*s, 212*s, 282*s]} + wx, wy, ww, wh = x+6*s, y+6*s, 314*s, 172*s + ah, aw = engine.art[index].shape[:2] + if aw/ah < 314/172: + full_h = ww*ah/aw + return {'kind': kind, '_index': index, 'bbox': [wx, wy-(full_h-wh)/2, ww, full_h]} + full_w = wh*aw/ah + return {'kind': kind, '_index': index, 'bbox': [wx-(full_w-ww)/2, wy, full_w, wh]} + + +def _init_worker(data, jitter=0., tile_key='bbox', detections=None): + cv2.setNumThreads(1) + ENGINE['engine'] = light_engine(data) if data else None + ENGINE['jitter'] = jitter + ENGINE['tile_key'] = tile_key + ENGINE['detections'] = detections + + +def perturb(tile, jitter, key): + """Deterministic pose error: shift by up to ``jitter`` of the size, scale by up to ``jitter``.""" + if not jitter: + return tile + rng = np.random.default_rng(int(hashlib.sha256(key.encode()).hexdigest()[:12], 16)) + dx, dy, ds = rng.uniform(-jitter, jitter, 3) + x, y, w, h = tile + return [x+dx*w, y+dy*h, w*(1+ds), h*(1+ds)] + + +def _scene(job): + path, row, dataset = job + image = cv2.imread(str(Path(path)/row['file'])) + if image is None: + raise ValueError(f"Cannot decode {Path(path)/row['file']}") + ih, iw = image.shape[:2] + engine = ENGINE.get('engine') + tile_key = ENGINE.get('tile_key', 'bbox') + detections = ENGINE.get('detections') + located = paired_tiles(row, detections[dataset][row['file']], tile_key)[0] if detections is not None else None + acquisition = row.get('acquisition', {}) + rows = [] + for index, (card, foreign) in enumerate(scene_cards(row)): + kind = card['kind'] + reference = perturb([float(v) for v in card[tile_key]], ENGINE.get('jitter', 0.), f"{dataset}/{row['file']}/{index}") + tile = reference if located is None else located[index] + f_label = field_label(kind, card) + f_inside = field_inside(kind, reference, iw, ih) + r_inside = rank_icon_inside(kind, reference, iw, ih) + f_read = tile is not None and f_label is not None and f_inside and field_inside(kind, tile, iw, ih) + r_read = tile is not None and r_inside and rank_icon_inside(kind, tile, iw, ih) + item = {'dataset': dataset, 'file': row['file'], 'card': index, 'kind': kind, 'id': card.get('id'), + 'foreign': foreign, 'locale': row.get('locale', 'unknown'), 'mode': card.get('display_mode', 'level'), + 'profile': acquisition.get('profile', 'unknown'), + 'source_resolution': 'x'.join(map(str, acquisition.get('source_resolution', [iw, ih]))), + 'width': reference[2], 'field_label': f_label, 'field_inside': f_inside, 'located': tile is not None, + 'field_latent': latent_field(kind, card), 'rank_latent': latent_rank(card), + 'field_occluded': not card.get('field_visible', True), 'rank_occluded': card.get('rank_visible') is False, + 'rank_label': rank_label(card), 'rank_inside': r_inside, + 'field_crop': crop_field(image, kind, tile) if f_read else None, + 'rank_crop': crop_rank(image, kind, tile) if r_read else None, 'engine_rank': None} + if engine is not None and r_inside and not foreign: + if not r_read: + item['engine_rank'] = {'value': None, 'reason': 'not_located'} + else: + art = art_item(engine, kind, card.get('id'), tile) + if art is not None: + item['engine_rank'] = engine.read_rank(image, art) + rows.append(item) + return rows + + +def strata_keys(item): + keys = ['overall', 'locale:'+item['locale'], 'mode:'+item['mode'], 'profile:'+item['profile'], + 'source_resolution:'+item['source_resolution'], 'card_width_bin:'+str(int(item['width']//30)*30), + 'foreign:'+str(item['foreign'])] + if item['field_occluded'] and item['field_inside']: + keys.append('field_occluded') + if item['id'] is not None and not item['foreign']: + keys.append('identity:'+('heldout' if item['id'] % 7 == 0 else 'seen')) + return keys + + +def scatter(probabilities, index, total): + """Full probability table with unread cards set to certain class 0 (no value at any threshold).""" + if probabilities is None: + return None + out = np.zeros((total, probabilities.shape[1]), np.float64) + out[:, 0] = 1. + out[index] = probabilities + return out + + +def evaluate_scenes(datasets, models, baseline, data, output, workers=8, engine_cache=None, jitter=0., + tile_key='bbox', detections=None): + """``engine_cache``: JSON file reusing template-matching results for the same truth fingerprint and + pose. ``jitter``: deterministic relative pose error applied to every truth tile (0 = exact). + ``tile_key``: truth tile field of each card. ``detections``: stored detections (``kind_pose detect``) + whose paired tiles replace the truth tiles; the evaluated population stays defined by the truth tiles.""" + jobs = [] + fingerprint = hashlib.sha256() + stored = read(detections) if detections else None + located = {} if stored else None + for d, path in enumerate(map(Path, datasets)): + document = read(path/'truth.json') + if not document.get('complete', False): + raise ValueError(f'Incomplete dataset: {path}') + fingerprint.update((path/'truth.json').read_bytes()) + jobs += [(str(path), row, d) for row in document['screenshots']] + if stored: + located[d] = detections_for(stored, path) + pose = {'jitter': jitter, 'tile_key': tile_key, + 'detections': {'path': str(detections), 'sha256': hashlib.sha256(Path(detections).read_bytes()).hexdigest(), + 'locator': stored['locator']} if stored else None} + items = [] + cached = None + if engine_cache and Path(engine_cache).exists(): + cached = read(engine_cache) + same_pose = (cached.get('jitter', 0.) == jitter and cached.get('tile_key', 'bbox') == tile_key + and cached.get('detections') == (pose['detections'] or {}).get('sha256')) + if cached['truth_fingerprint'] != fingerprint.hexdigest() or cached['data'] != str(data) or not same_pose: + raise ValueError('Engine cache belongs to other scenes, data or pose') + run_engine = str(data) if data and cached is None else None + with ProcessPoolExecutor(max_workers=workers, initializer=_init_worker, + initargs=(run_engine, jitter, tile_key, located)) as pool: + for done, rows in enumerate(pool.map(_scene, jobs, chunksize=4)): + items += rows + if (done+1) % 500 == 0: + print(json.dumps({'scenes': done+1, 'total': len(jobs)}), flush=True) + key = lambda item: f"{item['dataset']}/{item['file']}/{item['card']}" + if cached is not None: + for item in items: + item['engine_rank'] = cached['results'].get(key(item)) + elif engine_cache and data: + write(engine_cache, {'truth_fingerprint': fingerprint.hexdigest(), 'data': str(data), 'jitter': jitter, + 'tile_key': tile_key, 'detections': (pose['detections'] or {}).get('sha256'), + 'results': {key(i): i['engine_rank'] for i in items if i['engine_rank'] is not None}}) + sessions = {name: session(path) for name, path in models.items()} + thresholds = acceptance(models) + base = session(baseline) if baseline else None + report = {'schema': 'ournotes-boxlens.kind-field-evaluation/1', 'datasets': [str(p) for p in datasets], + 'truth_fingerprint': fingerprint.hexdigest(), 'scenes': len(jobs), + 'models': {k: {'path': str(v), 'sha256': hashlib.sha256(Path(v).read_bytes()).hexdigest(), + 'confidence_threshold': thresholds[k]} for k, v in models.items()}, + 'baseline_fields': {'path': str(baseline), 'sha256': hashlib.sha256(Path(baseline).read_bytes()).hexdigest()} if baseline else None, + 'engine_rank_data': str(data) if data else None, 'pose_jitter': jitter, 'pose': pose, 'kinds': {}} + for kind in KINDS: + fields = [i for i in items if i['kind'] == kind and i['field_label'] is not None and i['field_inside']] + ranks = [i for i in items if i['kind'] == kind and i['rank_inside']] + strata = defaultdict(Counter) + read_fields = [n for n, i in enumerate(fields) if i['field_crop'] is not None] + field_crops = (np.stack([fields[n]['field_crop'] for n in read_fields]) if read_fields + else np.zeros((0, 56, 144, 3), np.uint8)) + new_p = scatter(run(sessions[f'{kind}-fields'], field_crops), read_fields, len(fields)) if f'{kind}-fields' in sessions else None + base_p = scatter(run(base, field_crops), read_fields, len(fields)) if base else None + for n, item in enumerate(fields): + truth = item['field_label'] + for key in strata_keys(item): + if new_p is not None: + tally(strata[key], truth, field_value(kind, decide_field(kind, new_p[n], thresholds[f'{kind}-fields'])), 'field_new_', + item['field_latent']) + if base_p is not None: + tally(strata[key], truth, field_value(kind, decide_field(kind, base_p[n])), 'field_baseline_', + item['field_latent']) + read_ranks = [n for n, i in enumerate(ranks) if i['rank_crop'] is not None] + rank_crops = (np.stack([ranks[n]['rank_crop'] for n in read_ranks]) if read_ranks + else np.zeros((0, 48, 48, 3), np.uint8)) + rank_p = scatter(run(sessions[f'{kind}-rank'], rank_crops), read_ranks, len(ranks)) if f'{kind}-rank' in sessions else None + examples = [] + for n, item in enumerate(ranks): + truth = item['rank_label'] + new_value = (decide_rank(rank_p[n], thresholds[f'{kind}-rank'])['value'] or 0) if rank_p is not None else None + engine_value = (item['engine_rank']['value'] or 0) if item['engine_rank'] is not None else None + for key in strata_keys(item): + latent = item['rank_latent'] + if new_value is not None: + tally(strata[key], truth, new_value, 'rank_new_', latent) + if engine_value is not None: + tally(strata[key], truth, engine_value, 'rank_engine_', latent) + if new_value is not None: + tally(strata[key], truth, new_value, 'rank_new_on_engine_cards_', latent) + if new_value is not None and new_value > 0 and new_value != truth and len(examples) < 50: + examples.append({k: item[k] for k in ('dataset', 'file', 'card', 'id', 'mode', 'width', 'rank_label')} | {'predicted': new_value}) + limit = 105 if kind == 'member' else 100 + f_truth = np.array([i['field_label'] for i in fields]) + f_latent = np.array([i['field_latent'] for i in fields]) + r_truth = np.array([i['rank_label'] for i in ranks]) + r_latent = np.array([i['rank_latent'] for i in ranks]) + sweeps = {} + if new_p is not None and len(fields): + sweeps['field_new'] = misread_sweep(new_p, f_truth, f_latent, limit, default=thresholds[f'{kind}-fields']) + if base_p is not None and len(fields): + sweeps['field_baseline'] = misread_sweep(base_p, f_truth, f_latent, limit) + if rank_p is not None and len(ranks): + sweeps['rank_new'] = misread_sweep(rank_p, r_truth, r_latent, margin=.5, default=thresholds[f'{kind}-rank']) + report['kinds'][kind] = {'field_crops': len(fields), 'rank_crops': len(ranks), 'zero_misread': sweeps, + 'field_cards_not_read_for_pose': len(fields)-len(read_fields), + 'rank_cards_not_read_for_pose': len(ranks)-len(read_ranks), + 'rank_cards_outside_image': sum(1 for i in items if i['kind'] == kind and not i['rank_inside']), + 'field_cards_outside_image': sum(1 for i in items if i['kind'] == kind and i['field_label'] is not None and not i['field_inside']), + 'strata': {k: finish(v) for k, v in sorted(strata.items())}, 'rank_error_examples': examples} + print(json.dumps({'kind': kind, 'overall': report['kinds'][kind]['strata'].get('overall')}), flush=True) + report['scope'] = (('Card pose from stored detections paired with truth tiles; cards missed or detected as the ' + 'other kind, or whose detected crop leaves the image, count as unread. ' if stored else + 'Truth-derived card pose for every method (oracle localization). ') + + 'Field targets: visible level/training values, 0 = other or hidden parameter. Rank targets: ' + 'visible icon 1..5, 0 = no icon; only icons fully inside the image are scored.') + write(output, report) + return report + + +def evaluate_bank(bank, kind, models, baseline, output): + """Crop-bank evaluation (e.g. acquisition-degraded validation crops).""" + report = {'bank': str(bank), 'kind': kind} + images, labels, meta = load_bank(bank, kind, 'field') + crops = np.asarray(images)[:, MARGIN:MARGIN+56, MARGIN:MARGIN+144] + thresholds = acceptance(models) + for name, path in [('new', models.get(f'{kind}-fields')), ('baseline', baseline)]: + if path is None: + continue + threshold = thresholds[f'{kind}-fields'] if name == 'new' else FIELD_CONFIDENCE + report['field_'+name+'_threshold'] = threshold + p = run(session(path), crops) + counter = Counter() + latent = meta.get('latent', np.full(len(labels), -1)) + for prob, truth, state in zip(p, labels, latent): + tally(counter, int(truth), field_value(kind, decide_field(kind, prob, threshold)), latent=int(state)) + report['field_'+name] = finish(counter) + report['field_'+name+'_zero_misread'] = misread_sweep(p, labels, latent, 105 if kind == 'member' else 100, + default=threshold) + if models.get(f'{kind}-rank'): + images, labels, meta = load_bank(bank, kind, 'rank') + m = STORED_RANK_MARGIN + crops = np.asarray(images)[:, m:m+RANK_INPUT, m:m+RANK_INPUT] + p = run(session(models[f'{kind}-rank']), crops) + counter = Counter() + latent = meta.get('latent', np.full(len(labels), -1)) + for prob, truth, state in zip(p, labels, latent): + tally(counter, int(truth), decide_rank(prob, thresholds[f'{kind}-rank'])['value'] or 0, latent=int(state)) + report['rank_new'] = finish(counter) + report['rank_new_threshold'] = thresholds[f'{kind}-rank'] + report['rank_new_zero_misread'] = misread_sweep(p, labels, latent, margin=.5, default=thresholds[f'{kind}-rank']) + write(output, report) + print(json.dumps(report), flush=True) + return report + + +def acceptance(models): + """Per-model confidence threshold from ``selection.json`` next to the ONNX file, else the default.""" + out = {} + for name, path in models.items(): + selection = Path(path).parent/'selection.json' + default = FIELD_CONFIDENCE if name.endswith('fields') else RANK_CONFIDENCE + out[name] = read(selection).get('acceptance', {}).get('confidence', default) if selection.exists() else default + return out + + +def parse_models(values): + models = {} + for value in values or []: + name, path = value.split('=', 1) + if name not in [f'{k}-{p}' for k in KINDS for p in ('fields', 'rank')]: + raise ValueError(f'Unknown model name: {name}') + models[name] = Path(path) + return models + + +def tile_from_window(kind, window): + """Card tile rectangle from the artwork window (member 212x282, Snap 314x172 logical).""" + x, y, w, h = window + s = w/(212 if kind == 'member' else 314) + return [x-6*s, y-6*s, LOGICAL[kind][0]*s, LOGICAL[kind][1]*s] + + +def iou(a, b): + ix = max(0., min(a[0]+a[2], b[0]+b[2])-max(a[0], b[0])) + iy = max(0., min(a[1]+a[3], b[1]+b[3])-max(a[1], b[1])) + inter = ix*iy + return inter/max(1e-9, a[2]*a[3]+b[2]*b[3]-inter) + + +def score(counter, truth, value, name): + if truth is None: + counter[name+'_hidden_predictions'] += int(value is not None) + return + counter[name+'_total'] += 1 + counter[name+'_correct'] += int(value == truth) + counter[name+'_wrong'] += int(value is not None and value != truth) + counter[name+'_unknown'] += int(value is None) + + +def evaluate_real(truth_files, data, models, output, truth_box='window'): + """Real screenshots: production localization, baseline readers and new readers per card. + + Cards are localized by the existing matcher; the new readers use the tile + implied by the matched artwork window, the baseline readers use the + production code path. ``truth_box`` only serves to pair duplicates. + """ + from .engine import Engine + engine = Engine(data, threads=4) + sessions = {name: session(path, 4) for name, path in models.items()} + thresholds = acceptance(models) + images_out, totals = [], Counter() + for truth_file in map(Path, truth_files): + document = read(truth_file) + for row in document['screenshots']: + image = cv2.imread(str(truth_file.parent/row['file'])) + if image is None: + raise ValueError(f"Cannot decode {truth_file.parent/row['file']}") + ih, iw = image.shape[:2] + items = engine.fill_grid(image, engine.identify(image)) + windows = [engine.ui_bbox(item) for item in items] + baseline_fields = engine.numbers.read_fields(image, [{'kind': i['kind'], 'bbox': w} for i, w in zip(items, windows)]) + detected = [] + for item, window, base in zip(items, windows, baseline_fields): + kind = item['kind'] + tile = tile_from_window(kind, window) + new = {'level': None, 'awake_count': None, 'card_rank': None} + if f'{kind}-fields' in sessions and field_inside(kind, tile, iw, ih): + probabilities = run(sessions[f'{kind}-fields'], crop_field(image, kind, tile)[None])[0] + decision = decide_field(kind, probabilities, thresholds[f'{kind}-fields']) + new['level'], new['awake_count'] = decision['level'], decision['awake_count'] + new['field_confidence'] = decision['confidence'] + if f'{kind}-rank' in sessions and rank_icon_inside(kind, tile, iw, ih): + decision = decide_rank(run(sessions[f'{kind}-rank'], crop_rank(image, kind, tile)[None])[0], + thresholds[f'{kind}-rank']) + new['card_rank'], new['rank_confidence'] = decision['value'], decision['confidence'] + detected.append({'kind': kind, 'id': item['id'], 'window': [round(v, 2) for v in window], 'tile': tile, + 'baseline': {'level': base['level']['value'], 'awake_count': base['awake_count']['value'], + 'card_rank': engine.read_rank(image, item)['value']}, + 'new': new}) + used, cards, counter = set(), [], Counter() + for card in row['cards']: + candidates = [(n, d) for n, d in enumerate(detected) if n not in used + and (d['kind'], d['id']) == (card['kind'], card['id'])] + reference = card['bbox'] if truth_box == 'window' else None + found = max(candidates, key=lambda c: iou(c[1]['window'], reference) if reference else 0, default=None) + record = {'kind': card['kind'], 'id': card['id'], + 'truth': {k: card.get(k) for k in ('level', 'awake_count', 'card_rank', 'field_visible')}, + 'localized': found is not None} + counter['cards'] += 1 + counter['localized'] += int(found is not None) + if found: + used.add(found[0]) + d = found[1] + record['window_iou_with_truth_box'] = round(iou(d['window'], card['bbox']), 3) + record['baseline'], record['new'] = d['baseline'], {k: d['new'][k] for k in ('level', 'awake_count', 'card_rank')} + for method in ('baseline', 'new'): + for name in ('level', 'awake_count', 'card_rank'): + if card['kind'] == 'snap' and name == 'awake_count': + continue + score(counter, card.get(name), record[method][name], f'{method}_{name}') + cards.append(record) + counter['extra_detections'] += len(detected)-len(used) + images_out.append({'truth': str(truth_file), 'file': row['file'], 'kind': row.get('kind'), + 'display_mode': row.get('display_mode'), 'counts': dict(counter), 'cards': cards}) + totals.update(counter) + print(json.dumps({'file': row['file'], **dict(counter)}), flush=True) + report = {'schema': 'ournotes-boxlens.kind-field-real/1', 'data': str(data), + 'models': {k: {'path': str(v), 'sha256': hashlib.sha256(Path(v).read_bytes()).hexdigest(), + 'confidence_threshold': thresholds[k]} for k, v in models.items()}, + 'totals': dict(totals), 'images': images_out, + 'scope': 'Real screenshots with manual truth; localization by the existing matcher for every method; ' + 'masterdata level-limit filtering is not applied to either method.'} + write(output, report) + return report + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest='action', required=True) + s = sub.add_parser('scenes') + s.add_argument('--datasets', nargs='+', type=Path, required=True) + s.add_argument('--model', action='append', help='NAME=PATH, NAME in member-fields|snap-fields|member-rank|snap-rank') + s.add_argument('--baseline', type=Path, help='Shared field model ONNX for comparison') + s.add_argument('--data', type=Path, help='Data directory for template-matching rank comparison') + s.add_argument('--output', type=Path, required=True) + s.add_argument('--workers', type=int, default=8) + s.add_argument('--engine-cache', type=Path, help='Reuse/store template-matching results for these scenes') + s.add_argument('--pose-jitter', type=float, default=0., help='Relative pose error applied to truth tiles') + s.add_argument('--tile-key', default='bbox', help='Truth tile field of each card (bbox or bbox_layout)') + s.add_argument('--detections', type=Path, help='Stored detections whose paired tiles replace the truth tiles') + b = sub.add_parser('bank') + b.add_argument('--bank', type=Path, required=True) + b.add_argument('--kind', choices=KINDS, required=True) + b.add_argument('--model', action='append') + b.add_argument('--baseline', type=Path) + b.add_argument('--output', type=Path, required=True) + r = sub.add_parser('real') + r.add_argument('--truth', nargs='+', type=Path, required=True) + r.add_argument('--data', type=Path, required=True) + r.add_argument('--model', action='append') + r.add_argument('--output', type=Path, required=True) + a = p.parse_args() + if a.action == 'scenes': + evaluate_scenes(a.datasets, parse_models(a.model), a.baseline, a.data, a.output, a.workers, a.engine_cache, a.pose_jitter, + a.tile_key, a.detections) + elif a.action == 'bank': + evaluate_bank(a.bank, a.kind, parse_models(a.model), a.baseline, a.output) + else: + evaluate_real(a.truth, a.data, parse_models(a.model), a.output) diff --git a/bdon_vision/kind_fields.py b/bdon_vision/kind_fields.py new file mode 100644 index 0000000..da819ce --- /dev/null +++ b/bdon_vision/kind_fields.py @@ -0,0 +1,347 @@ +"""Per-kind field and card-rank crops, labels and acceptance rules. + +Member cards and Snap cards use separate readers. Every crop is defined +relative to the card tile rectangle ``[x, y, w, h]`` (output-image pixels) and +the tile scale ``s = w / logical_width`` (member 224, Snap 326): + +* parameter field: left ``x - 6s``, top ``y + h - 56s``, size ``144s x 56s``, + resampled to 144x56; +* rank icon: square of side ``96s`` centred on the icon centre + ``(x + cx*s, y + cy*s)`` with ``(cx, cy)`` = member (194.5, 267.2) or + Snap (306.1, 153.8), resampled to 48x48. + +Sampling follows ``cv2.warpAffine`` with ``WARP_INVERSE_MAP``: output pixel +``(i, j)`` reads the bilinear source position ``(left + i*sx, top + j*sy)`` +with integer pixel centres, ``sx = width / out_width`` and +``sy = height / out_height``. Pixels outside the image are zero. + +Crop banks are stored with a context margin (8 logical units on every side) +so training can jitter the crop; the centre of a stored crop is identical to +the deployed crop. +""" +import argparse +import hashlib +import json +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +import cv2 +import numpy as np + +from .assets import read, write +from .corpus import sample + +KINDS = ('member', 'snap') +LOGICAL = {'member': (224, 294), 'snap': (326, 184)} +RANK_CENTER = {'member': (194.5, 267.2), 'snap': (306.1, 153.8)} +FIELD_SIZE = (144, 56) # logical units == model input pixels (W, H) +RANK_SIDE = 96 # logical units of the square rank crop +RANK_INPUT = 48 # model input side in pixels +MARGIN = 8 # stored context margin in logical units +FIELD_STORED = (FIELD_SIZE[0]+2*MARGIN, FIELD_SIZE[1]+2*MARGIN) +RANK_STORED = (RANK_SIDE+2*MARGIN)*RANK_INPUT//RANK_SIDE +FIELD_CLASSES = {'member': 106, 'snap': 101} +RANK_CLASSES = 6 +FIELD_CONFIDENCE = .995 +FIELD_MARGIN = .5 +RANK_CONFIDENCE = .995 +RANK_MARGIN = .5 +LOCALES = ['ja', 'en', 'zh-Hant', 'zh-Hans', 'ko'] +MODES = ['level', 'training', 'total', 'performance', 'technic', 'visual', 'hide'] + + +def tile_scale(kind, bbox): + return bbox[2]/LOGICAL[kind][0] + + +def field_region(kind, bbox, margin=0.): + """Return (left, top, width, height) of the parameter-field crop.""" + x, y, w, h = bbox + s = tile_scale(kind, bbox) + return (x-(6+margin)*s, y+h-(56+margin)*s, (FIELD_SIZE[0]+2*margin)*s, (FIELD_SIZE[1]+2*margin)*s) + + +def rank_region(kind, bbox, margin=0.): + """Return (left, top, width, height) of the square rank-icon crop.""" + x, y, w, h = bbox + s = tile_scale(kind, bbox) + cx, cy = RANK_CENTER[kind] + half = (RANK_SIDE/2+margin)*s + return (x+cx*s-half, y+cy*s-half, 2*half, 2*half) + + +def field_inside(kind, bbox, width, height, border=2.): + left, top, w, h = field_region(kind, bbox) + return left >= border and top >= border and left+w <= width-border and top+h <= height-border + + +def rank_icon_inside(kind, bbox, width, height, border=2.): + """The full 64-unit icon (aspect 102:95) lies inside the image.""" + x, y = bbox[:2] + s = tile_scale(kind, bbox) + cx, cy = RANK_CENTER[kind] + half_w, half_h = 32*s, 32*95/102*s + return (x+cx*s-half_w >= border and y+cy*s-half_h >= border + and x+cx*s+half_w <= width-border and y+cy*s+half_h <= height-border) + + +def crop_field(image, kind, bbox, margin=0.): + """RGB uint8 crop of shape (56+2m, 144+2m, 3) from a BGR image.""" + size = (round(FIELD_SIZE[0]+2*margin), round(FIELD_SIZE[1]+2*margin)) + return sample(image, *field_region(kind, bbox, margin), size) + + +def crop_rank(image, kind, bbox, margin=0.): + """RGB uint8 square rank crop; side 48 at margin 0.""" + side = round((RANK_SIDE+2*margin)*RANK_INPUT/RANK_SIDE) + return sample(image, *rank_region(kind, bbox, margin), (side, side)) + + +def to_input(crops): + """uint8 NHWC RGB -> float32 NCHW in [0, 1].""" + return np.ascontiguousarray(np.asarray(crops, np.float32).transpose(0, 3, 1, 2)/255.) + + +def field_label(kind, card): + """Field class for a crop lying inside the image, or None when unusable. + + 0 other/hidden parameter or an unreadable (occluded) field, 1..100 level, + 101..105 member training count. Values of invisible fields are never + targets: an occluded level field is class 0, not its latent level. + """ + mode = card.get('display_mode', 'level') + if not card.get('field_visible', True): + return 0 + if mode == 'level': + return card.get('level') + if mode == 'training': + if kind != 'member' or card.get('awake_count') is None: + return None + return 100+card['awake_count'] + return 0 + + +def rank_label(card): + """1..5 for a fully visible icon, otherwise 0 (hidden, absent or cut).""" + return card['card_rank'] if card.get('card_rank') is not None else 0 + + +def latent_field(kind, card): + """Value the field would show without occlusion, for error analysis only (-1 unknown). + + Never a training target. It separates a value read through a partial + occlusion that matches the card state from a genuine misread. + """ + latent = card.get('latent_state') + if latent is None: + return -1 + mode = card.get('display_mode', 'level') + if mode == 'level': + return latent['level'] + if mode == 'training': + return 100+latent['awake_count'] if kind == 'member' else 0 + return 0 + + +def latent_rank(card): + """Rank icon the card state would show (0 in the hidden view), for error analysis only (-1 unknown).""" + latent = card.get('latent_state') + if latent is None: + return -1 + return 0 if card.get('display_mode') == 'hide' else latent['card_rank'] + + +def softmax(logits): + logits = np.asarray(logits, np.float64) + p = np.exp(logits-logits.max(axis=-1, keepdims=True)) + return p/p.sum(axis=-1, keepdims=True) + + +def accept(probabilities, confidence, margin): + """Top class, its probability, and whether the acceptance rule holds.""" + p = np.asarray(probabilities) + order = np.sort(p, axis=-1) + top = p.argmax(axis=-1) + best, second = order[..., -1], order[..., -2] + return top, best, (best >= confidence) & (best-second >= margin) + + +def misread_sweep(probabilities, truth, latent=None, limit=None, margin=FIELD_MARGIN, default=FIELD_CONFIDENCE, + grid=(.9, .95, .98, .99, .995, .998, .999, .9995, .9999)): + """Misreads and recall as the confidence threshold rises (margin rule fixed). + + A misread is an emitted value that differs from the visible target, or a + value emitted for a hidden target that differs from the card state + (``latent``; -1 or missing means unknown, so any value is a misread). + ``limit`` is the largest class that emits a value (Snap 100 even for a + 106-class model). Returns the lowest threshold with no misreads on these + crops and the recall of correct values there. + """ + p = np.asarray(probabilities, np.float64) + truth = np.asarray(truth) + latent = np.full(len(truth), -1) if latent is None else np.asarray(latent) + order = np.sort(p, axis=-1) + top, best = p.argmax(axis=-1), order[:, -1] + limit = p.shape[1]-1 if limit is None else limit + value = np.where((best-order[:, -2] >= margin) & (top > 0) & (top <= limit), top, 0) + misread = (value > 0) & (((truth > 0) & (value != truth)) | ((truth == 0) & (value != latent))) + visible = max(1, int((truth > 0).sum())) + + def at(tau): + emitted = (value > 0) & (best >= tau) + return {'misreads': int((emitted & misread).sum()), + 'recall': float((emitted & (truth > 0) & (value == truth)).sum()/visible)} + worst = float(best[misread].max()) if misread.any() else None + zero = 0. if worst is None else float(np.nextafter(worst, 2.)) + return {'visible_targets': int((truth > 0).sum()), 'highest_misread_confidence': worst, + 'zero_misread_threshold': zero, 'zero_misread_achievable': zero <= 1., + 'at_zero_misread_threshold': at(zero), 'at_default_threshold': {'threshold': default, **at(default)}, + 'by_threshold': {str(t): at(t) for t in grid}} + + +def decide_field(kind, probabilities, confidence=FIELD_CONFIDENCE, margin=FIELD_MARGIN): + """Field decision for one crop: dict(level, awake_count, display_mode, confidence, reason).""" + top, best, ok = accept(probabilities, confidence, margin) + top, best, ok = int(top), float(best), bool(ok) + out = {'level': None, 'awake_count': None, 'display_mode': 'unknown', 'confidence': round(best, 4)} + if not ok: + return {**out, 'reason': 'ambiguous_parameter'} + if top == 0: + return {**out, 'display_mode': 'other', 'reason': 'other_or_hidden_parameter'} + if top <= 100: + return {**out, 'level': top, 'display_mode': 'level'} + if kind == 'member' and top <= 105: + return {**out, 'awake_count': top-100, 'display_mode': 'training'} + return {**out, 'reason': 'unsupported_parameter'} + + +def decide_rank(probabilities, confidence=RANK_CONFIDENCE, margin=RANK_MARGIN): + """Rank decision for one crop: dict(value, confidence, reason).""" + top, best, ok = accept(probabilities, confidence, margin) + top, best, ok = int(top), float(best), bool(ok) + if not ok: + return {'value': None, 'confidence': round(best, 4), 'reason': 'ambiguous_icon'} + if top == 0: + return {'value': None, 'confidence': round(best, 4), 'reason': 'no_visible_rank_icon'} + return {'value': top, 'confidence': round(best, 4)} + + +# ---------------------------------------------------------------- crop banks + +def scene_cards(row): + """Catalog cards and foreign-art cards of one screenshot with a foreign flag.""" + for card in row.get('cards', []): + yield card, 0 + for card in row.get('foreign_cards', []): + yield card, 1 + + +def _scene_job(job): + path, row, dataset_index, scene_index = job + raw = (Path(path)/row['file']).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if row.get('sha256') and row['sha256'] != digest: + raise ValueError(f"Image hash mismatch: {row['file']}") + image = cv2.imdecode(np.frombuffer(raw, np.uint8), cv2.IMREAD_COLOR) + if image is None: + raise ValueError(f"Cannot decode {Path(path)/row['file']}") + ih, iw = image.shape[:2] + acquisition = row.get('acquisition', {}) + out = {kind: {'field': [], 'rank': []} for kind in KINDS} + skipped = Counter() + for card_index, (card, foreign) in enumerate(scene_cards(row)): + kind = card['kind'] + bbox = [float(v) for v in card['bbox']] + meta = {'dataset': dataset_index, 'scene': scene_index, 'card': card_index, + 'id': int(card.get('id', -1) if card.get('id') is not None else -1), 'foreign': foreign, + 'locale': LOCALES.index(row.get('locale', 'zh-Hans')), + 'mode': MODES.index(card.get('display_mode', 'level')), + 'width': bbox[2], 'source_resolution': 'x'.join(map(str, acquisition.get('source_resolution', [iw, ih]))), + 'profile': acquisition.get('profile', 'unknown'), + 'occluded': int(not card.get('field_visible', True)), 'latent': latent_field(kind, card)} + label = field_label(kind, card) + left, top, w, h = field_region(kind, bbox) + if label is None: + skipped[kind+':field_no_target'] += 1 + elif left < 0 or top < 0 or left+w > iw or top+h > ih: + skipped[kind+':field_cropped'] += 1 + else: + out[kind]['field'].append((crop_field(image, kind, bbox, MARGIN), label, meta)) + rank = rank_label(card) + meta = {**meta, 'icon_inside': int(rank_icon_inside(kind, bbox, iw, ih)), 'latent': latent_rank(card), + 'occluded': int(card.get('rank_visible') is False and rank_icon_inside(kind, bbox, iw, ih))} + out[kind]['rank'].append((crop_rank(image, kind, bbox, MARGIN), rank, meta)) + return out, skipped, digest + + +META_INT = ['dataset', 'scene', 'card', 'id', 'foreign', 'locale', 'mode', 'icon_inside', 'occluded', 'latent'] + + +def build(datasets, output, workers=8): + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + jobs, fingerprint, names = [], hashlib.sha256(), [] + for d, path in enumerate(map(Path, datasets)): + document = read(path/'truth.json') + if not document.get('complete', False): + raise ValueError(f'Incomplete dataset: {path}') + fingerprint.update((path/'truth.json').read_bytes()) + names.append(str(path)) + jobs += [(str(path), row, d, i) for i, row in enumerate(document['screenshots'])] + banks = {kind: {'field': [], 'rank': []} for kind in KINDS} + skipped = Counter() + with ProcessPoolExecutor(max_workers=workers) as pool: + for done, (out, skip, digest) in enumerate(pool.map(_scene_job, jobs, chunksize=4)): + fingerprint.update(bytes.fromhex(digest)) + skipped.update(skip) + for kind in KINDS: + for part in ('field', 'rank'): + banks[kind][part] += out[kind][part] + if (done+1) % 500 == 0: + print(json.dumps({'scenes': done+1, 'total': len(jobs)}), flush=True) + summary = {} + for kind in KINDS: + for part in ('field', 'rank'): + rows = banks[kind][part] + prefix = output/f'{kind}-{part}' + shape = (FIELD_STORED[1], FIELD_STORED[0], 3) if part == 'field' else (RANK_STORED, RANK_STORED, 3) + images = np.stack([r[0] for r in rows]) if rows else np.zeros((0, *shape), np.uint8) + labels = np.array([r[1] for r in rows], np.int16) + np.save(prefix.with_name(prefix.name+'-images.npy'), images) + np.save(prefix.with_name(prefix.name+'-labels.npy'), labels) + meta = {k: np.array([r[2].get(k, -1) for r in rows], np.int32) for k in META_INT} + meta['width'] = np.array([r[2]['width'] for r in rows], np.float32) + for key in ('source_resolution', 'profile'): + vocabulary = sorted({r[2][key] for r in rows}) + meta[key+'_vocabulary'] = np.array(vocabulary) + meta[key] = np.array([vocabulary.index(r[2][key]) for r in rows], np.int32) + np.savez(prefix.with_name(prefix.name+'-meta.npz'), **meta) + summary[f'{kind}-{part}'] = {'count': len(rows), 'labels': {str(k): int(v) for k, v in sorted(Counter(labels.tolist()).items())}} + manifest = {'schema': 'ournotes-boxlens.kind-crop-bank/1', 'datasets': names, 'scenes': len(jobs), + 'source_fingerprint': fingerprint.hexdigest(), 'banks': summary, 'skipped': dict(skipped), + 'locales': LOCALES, 'modes': MODES, 'margin_logical': MARGIN, + 'field_stored_wh': list(FIELD_STORED), 'rank_stored_side': RANK_STORED, 'complete': True} + write(output/'manifest.json', manifest) + print(json.dumps({'scenes': len(jobs), 'banks': {k: v['count'] for k, v in summary.items()}, 'skipped': dict(skipped)}), flush=True) + return manifest + + +def load_bank(path, kind, part): + """(images uint8 NHWC, labels int64, meta dict) of one stored bank.""" + path = Path(path) + if not read(path/'manifest.json').get('complete'): + raise ValueError(f'Incomplete crop bank: {path}') + images = np.load(path/f'{kind}-{part}-images.npy', mmap_mode='r') + labels = np.load(path/f'{kind}-{part}-labels.npy').astype(np.int64) + with np.load(path/f'{kind}-{part}-meta.npz') as meta: + return images, labels, {k: meta[k] for k in meta.files} + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description='Build per-kind field and rank crop banks from synthetic scene sets') + p.add_argument('--datasets', nargs='+', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--workers', type=int, default=8) + a = p.parse_args() + build(a.datasets, a.output, a.workers) diff --git a/bdon_vision/kind_hardcrops.py b/bdon_vision/kind_hardcrops.py new file mode 100644 index 0000000..4195b5a --- /dev/null +++ b/bdon_vision/kind_hardcrops.py @@ -0,0 +1,259 @@ +"""Acquisition-degraded field and rank crops rendered per card kind. + +Each sample renders one native card tile (artwork, frame, rank icon) with one +localized parameter text layer, composites it on a list-like background, and +then imitates screen capture on the whole tile: it is shrunk to the screen +scale, blurred, padded to a random codec-block phase, compressed one to four +times with JPEG or WebP, and only then sampled with the deployed crop +geometry. Field and rank crops come from the same degraded tile. Optionally +a share of samples gets a UI-like panel over the field or the rank icon; a +value whose glyph or icon pixels are at least 2 % covered becomes class 0, +while panels that only overlap empty parts of the crop keep the value. +""" +import argparse +import hashlib +import io +import json +import random +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from functools import lru_cache +from pathlib import Path + +import numpy as np +from PIL import Image, ImageDraw, ImageEnhance, ImageFilter, ImageFont + +from .assets import native_module, read, write +from .kind_fields import (FIELD_STORED, LOCALES, LOGICAL, MARGIN, META_INT, MODES, RANK_CENTER, RANK_INPUT, RANK_SIDE, + RANK_STORED, crop_field, crop_rank) + +LOCALE_COLUMNS = {'ja': '_japanese', 'en': '_english', 'zh-Hant': '_traditionalChinese', + 'zh-Hans': '_simplifiedChinese', 'ko': '_korean'} +PROFILES = { + 'train': dict(ratios=[.26, .33, .41, .49, .61, .77, .93, 1.09, 1.4], qualities=[34, 46, 58, 73, 87, 95], blur=(.05, .95)), + 'validation': dict(ratios=[.29, .37, .45, .55, .69, .85, 1.01, 1.25], qualities=[38, 50, 64, 79, 91], blur=(.12, 1.10)), +} +PAD = 32 +OCCLUSION_RATE = 0. +OCCLUDED = .02 # a value whose glyph or icon pixels are covered at least this much is unreadable +STATE = {} + + +def _init(data, kind, split, occlusion_rate=OCCLUSION_RATE): + native = native_module(data) + STATE.update(native=native, kind=kind, split=split, occlusion_rate=occlusion_rate, + renderers={loc: native.CardRenderer(col) for loc, col in LOCALE_COLUMNS.items()}) + catalog = [c for c in read(Path(data)/'catalog.json')['cards'] if c['kind'] == kind] + if split == 'train': + catalog = [c for c in catalog if c['id'] % 7 != 0] + STATE['pool'] = catalog + + +@lru_cache(maxsize=1024) +def _base(card_index, icon): + """Card tile without parameter text; icon None hides the rank icon.""" + card = STATE['pool'][card_index] + native = STATE['native'] + state = native.CardState(asset_id=card['asset_id'], rarity=card['rarity'], card_type=card['card_type'], + param='hide' if icon is None else 'level', level=None, rank=icon) + return STATE['renderers']['zh-Hans'].render(state, STATE['kind'], scale=1) + + +@lru_cache(maxsize=4096) +def _text(locale, mode, value): + native = STATE['native'] + kind = STATE['kind'] + power = (value, value+17, value+31) if kind == 'member' else (value, value+7, value+11) + state = native.CardState(asset_id=1, rarity=2, card_type=1, param=mode, + level=value if mode == 'level' else 1, awake_count=value if mode == 'training' else 1, + power=power) + return STATE['renderers'][locale].render(state, kind, scale=1, only='text') + + +@lru_cache(maxsize=1024) +def _icon_mask(card_index, icon): + """Pixels where the rank icon changes the tile.""" + if icon is None: + return None + shown = np.asarray(_base(card_index, icon)).astype(np.int16) + hidden = np.asarray(_base(card_index, None)).astype(np.int16) + return np.abs(shown-hidden).max(axis=2) > 24 + + +def _occlude(canvas, rng, boxes): + """Draw a UI-like panel over part of one target box; return the opaque-enough panel mask.""" + width, height = canvas.size + left, top, w, h = boxes[rng.randrange(len(boxes))] + side = rng.choice(['top', 'bottom', 'left', 'right']) + depth = rng.uniform(.15, .9) + if side in ('top', 'bottom'): + x0, x1 = left-rng.uniform(0, 90), left+w+rng.uniform(0, 90) + if rng.random() < .3: + x0, x1 = (x0, left+w*rng.uniform(.3, .8)) if rng.random() < .5 else (left+w*rng.uniform(.2, .7), x1) + y0, y1 = (top-rng.uniform(5, 60), top+h*depth) if side == 'top' else (top+h*(1-depth), top+h+rng.uniform(5, 60)) + else: + y0, y1 = top-rng.uniform(0, 40), top+h+rng.uniform(0, 40) + x0, x1 = (left-rng.uniform(5, 60), left+w*depth) if side == 'left' else (left+w*(1-depth), left+w+rng.uniform(5, 60)) + dark = rng.random() < .7 + color = tuple(rng.randrange(15, 80) for _ in range(3)) if dark else tuple(rng.randrange(200, 256) for _ in range(3)) + overlay = Image.new('RGBA', (width, height)) + draw = ImageDraw.Draw(overlay) + draw.rounded_rectangle((x0, y0, x1, y1), radius=rng.uniform(0, 18), fill=color+(rng.randrange(175, 250),)) + if rng.random() < .6: + try: + font = ImageFont.load_default(size=rng.randint(14, 30)) + except TypeError: + font = ImageFont.load_default() + text = ''.join(rng.choice('ABCDEFGHJKLMNPRSTUVWXYZ0123456789 ') for _ in range(rng.randint(2, 9))) + draw.text((x0+rng.uniform(0, max(1, x1-x0-40)), y0+rng.uniform(0, max(1, y1-y0-20))), text, font=font, + fill=(255, 255, 255, 255) if dark else (40, 40, 60, 255)) + canvas.alpha_composite(overlay) + return np.asarray(overlay.getchannel('A')) >= 128 + + +def _covered(mask, target): + """Share of target pixels under the panel (0 when the target is empty).""" + if target is None or not target.any(): + return 0. + return float((mask & target).sum())/float(target.sum()) + + +def _sample(rng): + kind = STATE['kind'] + profile = PROFILES[STATE['split']] + lw, lh = LOGICAL[kind] + card_index = rng.randrange(len(STATE['pool'])) + locale = rng.choice(LOCALES) + roll = rng.random() + if kind == 'member' and roll < .2: + mode, value = 'training', rng.randint(1, 5) + label = 100+value + elif roll < .6: + mode, value = 'level', rng.randint(1, 100) + label = value + else: + others = ['hide', 'performance', 'technic', 'visual']+(['total'] if kind == 'member' else []) + mode = rng.choice(others) + value = rng.randrange(64)*293+137 if kind == 'member' else rng.randrange(64)*17+37 + label = 0 + if mode == 'hide': + icon, rank = None, 0 + else: + icon = 0 if rng.random() < .05 else rng.randint(1, 5) + rank = icon + width, height = lw+2*PAD, lh+2*PAD + background = tuple(rng.randrange(90, 240) for _ in range(3))+(255,) + canvas = Image.new('RGBA', (width, height), background) + canvas.alpha_composite(_base(card_index, icon)) + glyphs = None + if mode != 'hide': + text = _text(locale, mode, value) + offset = (rng.randint(-2, 2), rng.randint(-2, 2)) + canvas.alpha_composite(text, offset) + alpha = np.asarray(text.getchannel('A')) >= 128 + ox, oy = offset + glyphs = np.zeros_like(alpha) + source = alpha[max(0, -oy):height-max(0, oy), max(0, -ox):width-max(0, ox)] + glyphs[max(0, oy):height+min(0, oy), max(0, ox):width+min(0, ox)] = source + if label == 0 and rng.random() < .1: + # A partly erased status value is still not a level or training count. + cut = rng.randrange(PAD+20, PAD+130) + canvas.paste(_base(card_index, icon).crop((cut, 0, width, height)), (cut, 0), + _base(card_index, icon).crop((cut, 0, width, height))) + occluded, rank_occluded, field_latent, rank_latent = 0, 0, label, rank + if rng.random() < STATE['occlusion_rate']: + cx, cy = RANK_CENTER[kind] + field_box = (PAD-6, PAD+lh-56, 144, 56) + icon_box = (PAD+cx-32, PAD+cy-32*95/102, 64, 64*95/102) + mask = _occlude(canvas, rng, [field_box, icon_box]) + # The value is unreadable only when the panel covers its own glyphs or icon, + # not when it merely overlaps empty parts of the crop. + if label > 0 and _covered(mask, glyphs) >= OCCLUDED: + label, occluded = 0, 1 + if rank > 0 and _covered(mask, _icon_mask(card_index, icon)) >= OCCLUDED: + rank, rank_occluded = 0, 1 + ratio = rng.choice(profile['ratios'])*rng.uniform(.94, 1.06) + tile = ImageEnhance.Brightness(canvas.convert('RGB')).enhance(rng.uniform(.88, 1.12)) + small_size = (max(8, round(width*ratio)), max(8, round(height*ratio))) + tile = tile.resize(small_size, rng.choice([Image.Resampling.BILINEAR, Image.Resampling.BICUBIC, Image.Resampling.LANCZOS])) + tile = tile.filter(ImageFilter.GaussianBlur(rng.uniform(*profile['blur']))) + px, py = rng.randrange(16), rng.randrange(16) + tile = Image.fromarray(np.pad(np.asarray(tile), ((py, 16-py), (px, 16-px), (0, 0)), mode='edge')) + for _ in range(rng.choice([1, 2, 3, 4])): + buffer = io.BytesIO() + tile.save(buffer, format=rng.choice(['JPEG', 'WEBP']), quality=rng.choice(profile['qualities'])) + tile = Image.open(io.BytesIO(buffer.getvalue())).convert('RGB') + bgr = np.ascontiguousarray(np.asarray(tile)[:, :, ::-1]) + scale_x, scale_y = small_size[0]/width, small_size[1]/height + bbox = [px+PAD*scale_x, py+PAD*scale_y, lw*scale_x, lh*scale_y] + field = crop_field(bgr, kind, bbox, MARGIN) + rank_crop = crop_rank(bgr, kind, bbox, MARGIN) + meta = {'id': STATE['pool'][card_index]['id'], 'foreign': 0, 'locale': LOCALES.index(locale), + 'mode': MODES.index(mode), 'icon_inside': 1, 'occluded': occluded, + 'rank_occluded': rank_occluded, 'width': bbox[2]} + return field, label, rank_crop, rank, meta, field_latent, rank_latent + + +def _chunk(job): + seed, start, count = job + rng = random.Random(seed) + rows = [] + for i in range(count): + field, label, rank_crop, rank, meta, field_latent, rank_latent = _sample(rng) + rows.append((field, label, rank_crop, rank, {**meta, 'dataset': 0, 'scene': start+i, 'card': 0}, + field_latent, rank_latent)) + return rows + + +def generate(data, kind, output, count, seed, split, workers=8, chunk=500, occlusion_rate=OCCLUSION_RATE): + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + jobs = [(seed*1000003+i, i, min(chunk, count-i)) for i in range(0, count, chunk)] + rows = [] + with ProcessPoolExecutor(max_workers=workers, initializer=_init, initargs=(str(data), kind, split, occlusion_rate)) as pool: + for part in pool.map(_chunk, jobs): + rows += part + print(json.dumps({'generated': len(rows), 'total': count, 'kind': kind, 'split': split}), flush=True) + summary = {} + for part, image_index, label_index, shape in [('field', 0, 1, (FIELD_STORED[1], FIELD_STORED[0], 3)), + ('rank', 2, 3, (RANK_STORED, RANK_STORED, 3))]: + images = np.stack([r[image_index] for r in rows]) + assert images.shape[1:] == shape, images.shape + labels = np.array([r[label_index] for r in rows], np.int16) + prefix = f'{kind}-{part}' + np.save(output/f'{prefix}-images.npy', images) + np.save(output/f'{prefix}-labels.npy', labels) + meta = {k: np.array([r[4].get(k, -1) for r in rows], np.int32) for k in META_INT} + meta['latent'] = np.array([r[5 if part == 'field' else 6] for r in rows], np.int32) + if part == 'rank': + meta['occluded'] = np.array([r[4]['rank_occluded'] for r in rows], np.int32) + meta['width'] = np.array([r[4]['width'] for r in rows], np.float32) + meta['source_resolution_vocabulary'] = np.array(['rendered-tile']) + meta['source_resolution'] = np.zeros(len(rows), np.int32) + meta['profile_vocabulary'] = np.array(['hard-'+split]) + meta['profile'] = np.zeros(len(rows), np.int32) + np.savez(output/f'{prefix}-meta.npz', **meta) + summary[prefix] = {'count': len(rows), 'labels': {str(k): int(v) for k, v in sorted(Counter(labels.tolist()).items())}} + write(output/'manifest.json', {'schema': 'ournotes-boxlens.kind-crop-bank/1', 'source': 'rendered-acquisition', + 'kind': kind, 'split': split, 'seed': seed, 'samples': count, 'profile': PROFILES[split], + 'catalog_sha256': hashlib.sha256((Path(data)/'catalog.json').read_bytes()).hexdigest(), + 'banks': summary, 'margin_logical': MARGIN, 'field_stored_wh': list(FIELD_STORED), + 'rank_stored_side': RANK_STORED, 'rank_logical_side': RANK_SIDE, 'rank_input': RANK_INPUT, + 'occlusion_rate': occlusion_rate, 'occluded_threshold': OCCLUDED, + 'complete': True}) + print(json.dumps(summary), flush=True) + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--data', type=Path, required=True) + p.add_argument('--kind', choices=['member', 'snap'], required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--count', type=int, default=60000) + p.add_argument('--seed', type=int, default=130001) + p.add_argument('--split', choices=list(PROFILES), default='train') + p.add_argument('--workers', type=int, default=8) + p.add_argument('--occlusion-rate', type=float, default=OCCLUSION_RATE, + help='Share of samples with a UI-like panel over the field or icon') + a = p.parse_args() + generate(a.data, a.kind, a.output, a.count, a.seed, a.split, a.workers, occlusion_rate=a.occlusion_rate) diff --git a/bdon_vision/kind_pose.py b/bdon_vision/kind_pose.py new file mode 100644 index 0000000..d4d9565 --- /dev/null +++ b/bdon_vision/kind_pose.py @@ -0,0 +1,261 @@ +"""Card-pose error of a card-frame detector and its effect on field and rank crops. + +``detect`` runs a card-frame locator (``bdon_vision.locator.Locator``) over +scene sets and stores the detected tile rectangles per image. ``errors`` pairs +detections with truth tiles (IoU >= 0.5, greedy by IoU) and reports per card +kind: + +* tile centre offset and width / height error in logical units (truth pixels + divided by the truth tile scale ``s``); +* the induced shift of the parameter-field crop centre in model input pixels + (one pixel per logical unit) and of the rank crop centre (one pixel per two + logical units), the relative crop scale error, and the largest displacement + of any crop pixel (centre shift plus scale error times half the crop size). + +Quantiles are reported for absolute values (p50, p90, p99, max) together with +the signed mean. Two tile conventions are measured: ``raw`` uses the detected +rectangle as is (scale from its width, field anchored to its bottom edge); +``aspect`` keeps the detected centre and rebuilds the tile with the logical +aspect from the mean of the width and height scales. +""" +import argparse +import hashlib +import json +from collections import Counter, defaultdict +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +import cv2 +import numpy as np + +from .assets import read, write +from .kind_fields import (FIELD_SIZE, KINDS, LOGICAL, RANK_INPUT, RANK_SIDE, field_region, rank_region, scene_cards, + tile_scale) + +CONVENTIONS = ('raw', 'aspect') +METRICS = ('centre_dx', 'centre_dy', 'width', 'height', 'scale', 'field_dx', 'field_dy', 'field_reach_x', + 'field_reach_y', 'rank_dx', 'rank_dy', 'rank_reach') + + +def iou(a, b): + ix = max(0., min(a[0]+a[2], b[0]+b[2])-max(a[0], b[0])) + iy = max(0., min(a[1]+a[3], b[1]+b[3])-max(a[1], b[1])) + inter = ix*iy + return inter/max(1e-9, a[2]*a[3]+b[2]*b[3]-inter) + + +def pair(truth, detections, min_iou=.5): + """Greedy one-to-one pairing by IoU; returns {truth_index: detection_index}.""" + candidates = sorted(((iou(t, d), i, j) for i, t in enumerate(truth) for j, d in enumerate(detections)), reverse=True) + out, used = {}, set() + for value, i, j in candidates: + if value < min_iou: + break + if i in out or j in used: + continue + out[i] = j + used.add(j) + return out + + +def normalise(kind, tile): + """Tile with the logical aspect, same centre, scale = mean of the width and height scales.""" + x, y, w, h = tile + lw, lh = LOGICAL[kind] + s = (w/lw+h/lh)/2 + return [x+w/2-lw*s/2, y+h/2-lh*s/2, lw*s, lh*s] + + +def convention(kind, tile, name): + if name == 'raw': + return list(tile) + if name == 'aspect': + return normalise(kind, tile) + raise ValueError(f'Unknown tile convention: {name}') + + +def pose_error(kind, truth, tile): + """Errors of ``tile`` against ``truth`` (both [x, y, w, h] tile rectangles) in logical units / input pixels.""" + s = tile_scale(kind, truth) + scale = tile_scale(kind, tile)/s-1 + + def centre(region): + left, top, w, h = region + return left+w/2, top+h/2 + fx, fy = centre(field_region(kind, tile)) + tx, ty = centre(field_region(kind, truth)) + rx, ry = centre(rank_region(kind, tile)) + qx, qy = centre(rank_region(kind, truth)) + per_pixel = RANK_SIDE/RANK_INPUT + field_dx, field_dy = (fx-tx)/s, (fy-ty)/s + rank_dx, rank_dy = (rx-qx)/s/per_pixel, (ry-qy)/s/per_pixel + return {'centre_dx': (tile[0]+tile[2]/2-truth[0]-truth[2]/2)/s, 'centre_dy': (tile[1]+tile[3]/2-truth[1]-truth[3]/2)/s, + 'width': (tile[2]-truth[2])/s, 'height': (tile[3]-truth[3])/s, 'scale': scale, + 'field_dx': field_dx, 'field_dy': field_dy, + 'field_reach_x': abs(field_dx)+abs(scale)*FIELD_SIZE[0]/2, + 'field_reach_y': abs(field_dy)+abs(scale)*FIELD_SIZE[1]/2, + 'rank_dx': rank_dx, 'rank_dy': rank_dy, + 'rank_reach': max(abs(rank_dx), abs(rank_dy))+abs(scale)*RANK_INPUT/2} + + +def summary(values): + v = np.asarray(values, np.float64) + if not len(v): + return {'n': 0} + a = np.abs(v) + return {'n': int(len(v)), 'mean': round(float(v.mean()), 4), 'p50': round(float(np.quantile(a, .5)), 4), + 'p90': round(float(np.quantile(a, .9)), 4), 'p99': round(float(np.quantile(a, .99)), 4), + 'max': round(float(a.max()), 4)} + + +# ---------------------------------------------------------------- detection + +LOCATOR = {} + + +def _init_detector(model, long_edge, threshold): + from .locator import Locator + cv2.setNumThreads(1) + LOCATOR['locator'] = Locator(model, long_edge, threads=1, threshold=threshold) + + +def _detect(job): + path, name = job + image = cv2.imread(str(Path(path)/name), cv2.IMREAD_COLOR) + if image is None: + raise ValueError(f'Cannot decode {Path(path)/name}') + out = LOCATOR['locator'](cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + return [{'kind': d['kind'], 'score': round(d['score'], 5), 'bbox': [round(float(v), 3) for v in d['bbox']]} for d in out] + + +def detect(datasets, model, output, workers=8, long_edge=640, threshold=.4): + """Locator tile rectangles for every image of the given scene sets.""" + report = {'schema': 'ournotes-boxlens.locator-detections/1', + 'locator': {'path': str(model), 'sha256': hashlib.sha256(Path(model).read_bytes()).hexdigest(), + 'long_edge': long_edge, 'threshold': threshold}, 'datasets': {}} + with ProcessPoolExecutor(max_workers=workers, initializer=_init_detector, initargs=(str(model), long_edge, threshold)) as pool: + for path in map(Path, datasets): + document = read(path/'truth.json') + names = [row['file'] for row in document['screenshots']] + images = {} + for done, (name, found) in enumerate(zip(names, pool.map(_detect, [(str(path), n) for n in names], chunksize=4))): + images[name] = found + if (done+1) % 500 == 0: + print(json.dumps({'dataset': str(path), 'images': done+1, 'total': len(names)}), flush=True) + report['datasets'][str(path)] = {'truth_sha256': hashlib.sha256((path/'truth.json').read_bytes()).hexdigest(), + 'images': images} + write(output, report) + return report + + +def detections_for(detections, path): + """Per-image detections of one scene set, checked against its truth file.""" + entry = detections['datasets'].get(str(path)) + if entry is None: + raise ValueError(f'No detections for {path}') + if entry['truth_sha256'] != hashlib.sha256((Path(path)/'truth.json').read_bytes()).hexdigest(): + raise ValueError(f'Detections belong to another truth file: {path}') + return entry['images'] + + +def paired_tiles(row, found, tile_key='bbox', min_iou=.5): + """Detected tile per card of ``scene_cards(row)`` (None when missed or detected as the other kind). + + Also returns counts of missed cards, kind confusions and detections paired + with no card (cards ignored by the truth, e.g. mostly outside the image, + absorb detections without counting). + """ + cards = [card for card, _ in scene_cards(row)] + truth = [[float(v) for v in card[tile_key]] for card in cards] + ignored = [[float(v) for v in card[tile_key]] for card in row.get('ignored_cards', []) if card.get(tile_key)] + boxes = [d['bbox'] for d in found] + matched = pair(truth+ignored, boxes, min_iou) + tiles, counts = [], Counter() + for i, card in enumerate(cards): + j = matched.get(i) + counts[card['kind']+':cards'] += 1 + if j is None: + counts[card['kind']+':missed'] += 1 + tiles.append(None) + elif found[j]['kind'] != card['kind']: + counts[card['kind']+':kind_confused'] += 1 + tiles.append(None) + else: + tiles.append(list(found[j]['bbox'])) + counts['unpaired_detections'] += len(found)-len(matched) + return tiles, counts + + +# ---------------------------------------------------------------- error report + +def width_bin(width): + return 'lt100' if width < 100 else 'lt160' if width < 160 else 'ge160' + + +def errors(datasets, detections, output, tile_key='bbox', min_iou=.5, worst=20): + detections = read(detections) if not isinstance(detections, dict) else detections + report = {'schema': 'ournotes-boxlens.locator-pose-error/1', 'locator': detections['locator'], 'tile_key': tile_key, + 'min_iou': min_iou, 'units': {'centre_dx centre_dy width height': 'logical units (truth tile scale)', + 'scale': 'detected / truth tile scale - 1', + 'field_*': 'field-crop input pixels (1 px = 1 logical unit)', + 'rank_*': 'rank-crop input pixels (1 px = 2 logical units)'}, + 'datasets': {}} + for path in map(Path, datasets): + images = detections_for(detections, path) + document = read(path/'truth.json') + values = {c: {k: defaultdict(list) for k in KINDS} for c in CONVENTIONS} + binned = {k: defaultdict(lambda: defaultdict(list)) for k in KINDS} + counts, examples = Counter(), {k: [] for k in KINDS} + for row in document['screenshots']: + tiles, scene_counts = paired_tiles(row, images[row['file']], tile_key, min_iou) + counts.update(scene_counts) + for index, ((card, _), tile) in enumerate(zip(scene_cards(row), tiles)): + if tile is None: + continue + kind = card['kind'] + truth = [float(v) for v in card[tile_key]] + for name in CONVENTIONS: + error = pose_error(kind, truth, convention(kind, tile, name)) + for metric in METRICS: + values[name][kind][metric].append(error[metric]) + if name == 'raw': + for metric in ('centre_dx', 'centre_dy', 'scale', 'field_reach_x', 'field_reach_y', 'rank_reach'): + binned[kind][width_bin(truth[2])][metric].append(error[metric]) + examples[kind].append((max(error['field_reach_x'], error['field_reach_y']), + {'file': row['file'], 'card': index, 'truth': [round(v, 2) for v in truth], + 'detected': tile, **{m: round(error[m], 3) for m in METRICS}})) + entry = {'images': len(document['screenshots']), 'counts': dict(sorted(counts.items())), 'kinds': {}} + for kind in KINDS: + entry['kinds'][kind] = { + 'pairs': len(values['raw'][kind]['scale']), + 'conventions': {name: {m: summary(values[name][kind][m]) for m in METRICS} for name in CONVENTIONS}, + 'raw_by_truth_width_px': {b: {m: summary(v) for m, v in sorted(d.items())} for b, d in sorted(binned[kind].items())}, + 'largest_field_displacement': [e for _, e in sorted(examples[kind], key=lambda e: -e[0])[:worst]]} + report['datasets'][str(path)] = entry + print(json.dumps({'dataset': str(path), 'counts': entry['counts'], + **{k: {m: entry['kinds'][k]['conventions']['raw'][m].get('p99') for m in METRICS} for k in KINDS}}), flush=True) + write(output, report) + return report + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest='action', required=True) + d = sub.add_parser('detect', help='Run a locator ONNX model over scene sets') + d.add_argument('--datasets', nargs='+', type=Path, required=True) + d.add_argument('--locator', type=Path, required=True) + d.add_argument('--long-edge', type=int, default=640) + d.add_argument('--threshold', type=float, default=.4) + d.add_argument('--workers', type=int, default=8) + d.add_argument('--output', type=Path, required=True) + e = sub.add_parser('errors', help='Pose error of stored detections against truth tiles') + e.add_argument('--datasets', nargs='+', type=Path, required=True) + e.add_argument('--detections', type=Path, required=True) + e.add_argument('--tile-key', default='bbox', help='Truth tile field of each card (bbox or bbox_layout)') + e.add_argument('--min-iou', type=float, default=.5) + e.add_argument('--output', type=Path, required=True) + a = p.parse_args() + if a.action == 'detect': + detect(a.datasets, a.locator, a.output, a.workers, a.long_edge, a.threshold) + else: + errors(a.datasets, a.detections, a.output, a.tile_key, a.min_iou) diff --git a/bdon_vision/select_kind_models.py b/bdon_vision/select_kind_models.py new file mode 100644 index 0000000..3621147 --- /dev/null +++ b/bdon_vision/select_kind_models.py @@ -0,0 +1,100 @@ +"""Checkpoint selection for a finished per-kind training run. + +Every saved ``step-*.pt`` is evaluated on the development crop banks. Two +rules are available: + +* ``zero-misread`` (default): for each checkpoint find the lowest confidence + threshold (margin rule fixed, never below the default threshold) at which + no development crop is misread, then pick the checkpoint with the highest + recall of normal and stress scene targets at that threshold (ties: recall + on degraded crops). The chosen threshold becomes the model's acceptance + threshold. +* ``lexicographic``: the training-time rule of ``train_kind_fields`` at the + default threshold. + +The chosen checkpoint is copied to ``selected.pt`` and exported to ONNX with +a consistency check. +""" +import argparse +import json +import shutil +from pathlib import Path + +import numpy as np +import torch + +from .assets import write +from .kind_fields import FIELD_CONFIDENCE, FIELD_MARGIN, FIELD_SIZE, MARGIN, RANK_CONFIDENCE, RANK_INPUT +from .train_kind_fields import (SELECTION_RULE, ZERO_MISREAD_RULE, GpuBank, center_crop, export_onnx, field_model, field_outcomes, + file_sha256, parse_dev, predict, selection_key, verify_onnx, zero_misread, + zero_misread_key) +from .train_kind_rank import STORED_MARGIN, RankNet, inside_flags, rank_outcomes + + +def reselect(kind, part, run, dev_sets, device='cpu', threads=8, rule='zero-misread'): + torch.set_num_threads(threads) + run = Path(run) + roles = {name: role for name, (_, role) in dev_sets.items()} + dev = {name: GpuBank(paths, kind, 'field' if part == 'fields' else 'rank', device) for name, (paths, _) in dev_sets.items()} + make = (lambda: field_model(kind)) if part == 'fields' else RankNet + out_w, out_h, margin = (*FIELD_SIZE, MARGIN) if part == 'fields' else (RANK_INPUT, RANK_INPUT, STORED_MARGIN) + curve = [] + for path in sorted(run.glob('step-*.pt')): + model = make() + model.load_state_dict(torch.load(path, map_location='cpu', weights_only=True)) + model = model.to(device).eval() + row = {'step': int(path.stem.split('-')[1]), 'sets': {}} + kept = {} + for name, bank in dev.items(): + probabilities = predict(model, bank, out_w, out_h, margin) + labels = bank.y.cpu().numpy() + if part == 'fields': + row['sets'][name] = field_outcomes(kind, probabilities, labels, latent=bank.latent)[1] + keep = np.ones(len(labels), bool) + else: + keep = inside_flags(bank).astype(bool) + row['sets'][name] = rank_outcomes(probabilities, labels, keep, latent=bank.latent)[1] + kept[name] = (probabilities[keep], labels[keep], bank.latent[keep]) + limit = (105 if kind == 'member' else 100) if part == 'fields' else 5 + default = FIELD_CONFIDENCE if part == 'fields' else RANK_CONFIDENCE + row['zero_misread'] = zero_misread({n: v[0] for n, v in kept.items()}, {n: v[1] for n, v in kept.items()}, + {n: v[2] for n, v in kept.items()}, roles, limit, FIELD_MARGIN, default) + curve.append(row) + print(json.dumps({'step': row['step'], **{k: (v['correct'], v['errors'], v['false_visible']) for k, v in row['sets'].items()}, + 'zero_misread': row['zero_misread']}), flush=True) + if rule == 'zero-misread': + best = min(curve, key=zero_misread_key) + else: + best = min(curve, key=lambda row: selection_key(row, roles)) + shutil.copy2(run/f"step-{best['step']:06d}.pt", run/'selected.pt') + model = make() + model.load_state_dict(torch.load(run/'selected.pt', map_location='cpu', weights_only=True)) + onnx_path = run/f'{kind}-{part}.onnx' + export_onnx(model, onnx_path, (1, 3, out_h, out_w)) + probe = next(iter(dev.values())) + samples = center_crop(probe.x[:256].float().div(255), out_w, out_h, margin).cpu().numpy().astype(np.float32) + consistency = verify_onnx(model, onnx_path, samples) + write(run/'learning-curve-reselected.json', {'checkpoints': curve, 'scope': 'development sets only'}) + rule_text = ZERO_MISREAD_RULE if rule == 'zero-misread' else SELECTION_RULE + write(run/'selection.json', {'selected_step': best['step'], 'selected': best, 'roles': roles, 'rule': rule, 'rule_text': rule_text, + 'acceptance': {'confidence': (best['zero_misread']['threshold'] if rule == 'zero-misread' + and best['zero_misread']['achievable'] else default), + 'margin': FIELD_MARGIN}, + 'dev_sets': {k: [str(p) for p in v[0]] for k, v in dev_sets.items()}, + 'onnx': str(onnx_path), 'onnx_sha256': file_sha256(onnx_path), + 'pt_sha256': file_sha256(run/'selected.pt'), 'onnx_consistency': consistency, + 'reselected': True}) + print(json.dumps({'selected_step': best['step'], 'onnx_max_abs_diff': consistency['max_abs_logit_diff']}), flush=True) + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--kind', choices=['member', 'snap'], required=True) + p.add_argument('--part', choices=['fields', 'rank'], required=True) + p.add_argument('--run', type=Path, required=True, help='Training output directory with step-*.pt') + p.add_argument('--dev', nargs='+', required=True, help='NAME:ROLE:PATH[,PATH] development sets (never the final test)') + p.add_argument('--device', default='cpu') + p.add_argument('--threads', type=int, default=8) + p.add_argument('--rule', choices=['zero-misread', 'lexicographic'], default='zero-misread') + a = p.parse_args() + reselect(a.kind, a.part, a.run, parse_dev(a.dev), a.device, a.threads, a.rule) diff --git a/bdon_vision/train_kind_fields.py b/bdon_vision/train_kind_fields.py new file mode 100644 index 0000000..1d131b7 --- /dev/null +++ b/bdon_vision/train_kind_fields.py @@ -0,0 +1,384 @@ +"""Train a per-kind parameter-field reader (member 106 classes, Snap 101 classes). + +Initialised from the shared 106-class field model; the Snap head keeps rows +0..100. Development sets drive checkpoint selection: for every checkpoint the +lowest confidence threshold (not below 0.995, margin 0.5) without any misread +on the development crops is found, and the checkpoint with the highest recall +on normal and stress scenes at its own threshold wins. A misread is a wrong +visible value or a value emitted for a hidden target that differs from the +card state. The chosen threshold is the model's acceptance threshold. +""" +import argparse +import hashlib +import json +import shutil +import time +from pathlib import Path + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from .assets import write +from .kind_fields import (FIELD_CLASSES, FIELD_CONFIDENCE, FIELD_MARGIN, FIELD_SIZE, FIELD_STORED, MARGIN, + accept, load_bank, misread_sweep) +from .train import LevelNet, seed_all + + +def field_model(kind): + model = LevelNet() + model.head = nn.Linear(96*3*8, FIELD_CLASSES[kind]) + return model + + +def initialise_from(model, path): + """Copy every shared layer and the leading rows of the classifier head.""" + state = torch.load(path, map_location='cpu', weights_only=True) + model.load_state_dict({k: v for k, v in state.items() if not k.startswith('head.')}, strict=False) + with torch.no_grad(): + n = min(model.head.out_features, state['head.weight'].shape[0]) + model.head.weight[:n].copy_(state['head.weight'][:n]) + model.head.bias[:n].copy_(state['head.bias'][:n]) + return model + + +class GpuBank: + """uint8 crops resident on the GPU (NCHW) with integer labels.""" + + def __init__(self, paths, kind, part, device='cuda'): + images, labels, metas = [], [], [] + for path in paths: + x, y, m = load_bank(path, kind, part) + images.append(np.asarray(x)) + labels.append(y) + metas.append(m) + self.paths = [str(p) for p in paths] + x = np.concatenate(images) if images else np.zeros((0,), np.uint8) + self.x = torch.from_numpy(x).to(device).permute(0, 3, 1, 2).contiguous() + self.y = torch.from_numpy(np.concatenate(labels)).to(device) + self.meta = metas + self.latent = np.concatenate([m['latent'] if 'latent' in m else np.full(len(y), -1, np.int32) + for m, y in zip(metas, labels)]) if metas else np.zeros(0, np.int32) + + def __len__(self): + return len(self.y) + + def batch(self, indices): + return self.x[indices].float().div_(255), self.y[indices] + + +def jitter_crop(x, out_w, out_h, margin, shift, scale): + """Random shift/scale crop of the stored margin bank; identity is the deployed crop. + + ``shift``: largest centre shift in output pixels, one value or (x, y); + ``scale``: largest relative scale change. Both are drawn uniformly. + """ + n, _, h, w = x.shape + dev = x.device + shift_x, shift_y = (shift, shift) if np.isscalar(shift) else shift + dx = torch.empty(n, 1, 1, device=dev).uniform_(-shift_x, shift_x) + dy = torch.empty(n, 1, 1, device=dev).uniform_(-shift_y, shift_y) + sc = torch.empty(n, 1, 1, device=dev).uniform_(1-scale, 1+scale) + i = torch.arange(out_w, device=dev, dtype=torch.float32).view(1, 1, -1) + j = torch.arange(out_h, device=dev, dtype=torch.float32).view(1, -1, 1) + cx, cy = (out_w-1)/2, (out_h-1)/2 + xs = margin+cx+dx+(i-cx)*sc + ys = margin+cy+dy+(j-cy)*sc + grid = torch.stack([((2*xs+1)/w-1).expand(n, out_h, out_w), ((2*ys+1)/h-1).expand(n, out_h, out_w)], dim=-1) + return F.grid_sample(x, grid, mode='bilinear', padding_mode='zeros', align_corners=False) + + +def jitter_reach(out_w, out_h, shift, scale): + """Largest displacement (x, y) of any output pixel under ``jitter_crop``; compare with the stored margin.""" + shift_x, shift_y = (shift, shift) if np.isscalar(shift) else shift + return shift_x+scale*(out_w-1)/2, shift_y+scale*(out_h-1)/2 + + +def photometric(x, strength=1.): + n = x.shape[0] + dev = x.device + brightness = torch.empty(n, 1, 1, 1, device=dev).uniform_(1-.10*strength, 1+.10*strength) + contrast = torch.empty(n, 1, 1, 1, device=dev).uniform_(1-.12*strength, 1+.12*strength) + gain = torch.empty(n, 3, 1, 1, device=dev).uniform_(1-.04*strength, 1+.04*strength) + mean = x.mean(dim=(1, 2, 3), keepdim=True) + x = ((x-mean)*contrast+mean)*brightness*gain + blur = torch.empty(n, 1, 1, 1, device=dev).uniform_(0, .6*strength)*(torch.rand(n, 1, 1, 1, device=dev) < .4) + x = x*(1-blur)+F.avg_pool2d(x, 3, 1, 1, count_include_pad=False)*blur + x = x+torch.randn_like(x)*.01*strength + return x.clamp_(0, 1) + + +def train_mode(model, freeze_bn=False): + """Training mode; optionally keep the initial BatchNorm statistics fixed.""" + model.train() + if freeze_bn: + for module in model.modules(): + if isinstance(module, nn.BatchNorm2d): + module.eval() + + +def center_crop(x, out_w, out_h, margin): + return x[:, :, margin:margin+out_h, margin:margin+out_w] + + +def predict(model, bank, out_w, out_h, margin, batch=1024): + model.eval() + parts = [] + with torch.inference_mode(): + for start in range(0, len(bank), batch): + x = bank.x[start:start+batch].float().div_(255) + parts.append(model(center_crop(x, out_w, out_h, margin)).float().softmax(1).cpu().numpy()) + return np.concatenate(parts) if parts else np.zeros((0, 1), np.float32) + + +def outcome_counts(value, labels, keep=None, latent=None): + """Counts for emitted values (0 = none) against visible targets (label > 0) and hidden targets (0). + + ``false_visible`` is any value emitted for a hidden target. ``latent`` (-1 + unknown), used only for analysis, splits those into values matching the + card state behind a partial occlusion and genuine misreads. ``errors`` = + wrong visible values + hidden-target values not matching the card state. + """ + labels = np.asarray(labels) + keep = np.ones(len(labels), bool) if keep is None else np.asarray(keep).astype(bool) + latent = np.full(len(labels), -1) if latent is None else np.asarray(latent) + visible = (labels > 0) & keep + hidden = (labels == 0) & keep + read_hidden = hidden & (value > 0) + counts = {'visible_total': int(visible.sum()), 'hidden_total': int(hidden.sum()), + 'correct': int((visible & (value == labels)).sum()), + 'wrong': int((visible & (value > 0) & (value != labels)).sum()), + 'unknown': int((visible & (value == 0)).sum()), + 'false_visible': int(read_hidden.sum()), + 'false_visible_latent_match': int((read_hidden & (value == latent)).sum()), + 'false_visible_latent_mismatch': int((read_hidden & (value != latent)).sum())} + counts['errors'] = counts['wrong']+counts['false_visible_latent_mismatch'] + counts['coverage'] = counts['correct']/counts['visible_total'] if counts['visible_total'] else None + return counts + + +def field_outcomes(kind, probabilities, labels, confidence=FIELD_CONFIDENCE, margin=FIELD_MARGIN, latent=None): + """Per-crop emitted value (0 = no value) and outcome counts. + + For Snap, an accepted 101..105 class (possible only with the shared + 106-class model) is an unsupported parameter and emits no value. + """ + top, _, ok = accept(probabilities, confidence, margin) + limit = 105 if kind == 'member' else 100 + value = np.where(ok & (top > 0) & (top <= limit), top, 0) + return value, outcome_counts(value, labels, latent=latent) + + +SELECTION_RULE = ('min errors on normal sets, then min errors on stress and hard sets, then max correct on ' + 'normal and stress sets, then max correct on hard sets; errors = wrong visible values + values on ' + 'hidden targets that differ from the card state (any hidden-target value when the state is unknown)') + + +def selection_key(row, roles): + def total(key, wanted): + return sum(row['sets'][name][key] for name, role in roles.items() if role in wanted and name in row['sets']) + return (total('errors', ('normal',)), total('errors', ('stress', 'hard')), + -total('correct', ('normal', 'stress')), -total('correct', ('hard',)), row['step']) + + +ZERO_MISREAD_RULE = ('per checkpoint: lowest confidence threshold (not below the default, margin fixed) with no misread on ' + 'any development crop; choose the highest recall of normal and stress scene targets at that ' + 'threshold, then the highest recall on degraded crops; checkpoints where no threshold removes ' + 'every misread rank after all others, by fewest misreads at the default threshold, then recall') + + +def zero_misread(probabilities, truth, latent, roles, limit, margin, default): + """Joint zero-misread threshold over all sets, with recall per role group there and at the default.""" + names = list(probabilities) + + def joined(chosen, key): + return np.concatenate([key[n] for n in chosen]) + sweep = misread_sweep(joined(names, probabilities), joined(names, truth), joined(names, latent), limit, margin, default) + threshold = max(default, sweep['zero_misread_threshold']) + out = {'threshold': threshold, 'achievable': threshold <= 1., + 'highest_misread_confidence': sweep['highest_misread_confidence'], + 'misreads_at_default': sweep['at_default_threshold']['misreads'], 'recall': {}, 'recall_at_default': {}, + 'per_set': {}} + for group, wanted in [('scenes', ('normal', 'stress')), ('hard', ('hard',))]: + chosen = [n for n in names if roles[n] in wanted] + if not chosen: + continue + part = misread_sweep(joined(chosen, probabilities), joined(chosen, truth), joined(chosen, latent), limit, margin, + default, grid=(threshold,) if threshold <= 1. else ()) + out['recall_at_default'][group] = part['at_default_threshold']['recall'] + out['recall'][group] = part['by_threshold'][str(threshold)]['recall'] if threshold <= 1. else 0. + for name in names: + alone = misread_sweep(probabilities[name], truth[name], latent[name], limit, margin, default, grid=()) + out['per_set'][name] = {'zero_misread_threshold': max(default, alone['zero_misread_threshold']), + 'achievable': alone['zero_misread_achievable'], + 'recall_at_zero_misread_threshold': alone['at_zero_misread_threshold']['recall'] + if alone['zero_misread_threshold'] > default else alone['at_default_threshold']['recall'], + 'misreads_at_default': alone['at_default_threshold']['misreads']} + return out + + +def zero_misread_key(row): + z = row['zero_misread'] + if z['achievable']: + return (0, -z['recall'].get('scenes', 0.), -z['recall'].get('hard', 0.), row['step']) + return (1, z['misreads_at_default'], -z['recall_at_default'].get('scenes', 0.), row['step']) + + +def export_onnx(model, path, shape): + model = model.eval().cpu() + torch.onnx.export(model, torch.zeros(*shape), str(path), input_names=['image'], output_names=['logits'], + dynamic_axes={'image': {0: 'batch'}, 'logits': {0: 'batch'}}, opset_version=17, dynamo=False) + + +def verify_onnx(model, path, samples): + """Compare ONNX Runtime CPU against PyTorch on real crops and several batch sizes.""" + import onnxruntime as ort + session = ort.InferenceSession(str(path), providers=['CPUExecutionProvider']) + model = model.eval().cpu() + report = {'batches': {}} + worst = 0. + for size in (1, 7, 64, len(samples)): + x = samples[:size] + with torch.inference_mode(): + reference = model(torch.from_numpy(x)).numpy() + got = session.run(None, {'image': x})[0] + diff = float(np.abs(reference-got).max()) + worst = max(worst, diff) + report['batches'][str(len(x))] = {'max_abs_logit_diff': diff, + 'argmax_agreement': float((reference.argmax(1) == got.argmax(1)).mean())} + report['max_abs_logit_diff'] = worst + report['opset'] = 17 + report['inputs'] = [(i.name, i.shape) for i in session.get_inputs()] + report['outputs'] = [(o.name, o.shape) for o in session.get_outputs()] + return report + + +def file_sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def train(kind, train_banks, hard_banks, dev_sets, output, initial, steps, batch, eval_every, lr, hard_fraction, device='cuda', threads=8, + freeze_bn=False, jitter_shift=(2.5, 2.5), jitter_scale=.04): + if device.startswith('cuda') and not torch.cuda.is_available(): + raise RuntimeError('GPU not available') + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + seed_all(91019 if kind == 'member' else 91031) + torch.set_num_threads(threads) + out_w, out_h = FIELD_SIZE + scenes = GpuBank(train_banks, kind, 'field', device) + hard = GpuBank(hard_banks, kind, 'field', device) if hard_banks else None + dev = {name: GpuBank(paths, kind, 'field', device) for name, (paths, _) in dev_sets.items()} + roles = {name: role for name, (_, role) in dev_sets.items()} + model = field_model(kind) + if initial: + initialise_from(model, initial) + model = model.to(device) + optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4) + schedule = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, total_steps=steps, pct_start=.05, + div_factor=10, final_div_factor=100) + started = time.perf_counter() + curve = [] + config = {'kind': kind, 'train_banks': [str(p) for p in train_banks], 'hard_banks': [str(p) for p in hard_banks], + 'dev_sets': {k: {'banks': [str(p) for p in v[0]], 'role': v[1]} for k, v in dev_sets.items()}, + 'initial': str(initial) if initial else None, 'initial_sha256': file_sha256(initial) if initial else None, + 'steps': steps, 'batch': batch, 'eval_every': eval_every, 'lr': lr, 'hard_fraction': hard_fraction, 'freeze_bn': freeze_bn, + 'jitter': {'shift_px': list(jitter_shift), 'scale': jitter_scale, 'stored_margin_px': MARGIN, + 'reach_px': [round(v, 3) for v in jitter_reach(out_w, out_h, jitter_shift, jitter_scale)]}, + 'train_counts': {'scenes': len(scenes), 'hard': len(hard) if hard else 0}, + 'dev_counts': {k: len(v) for k, v in dev.items()}, + 'acceptance': {'confidence': FIELD_CONFIDENCE, 'margin': FIELD_MARGIN}, + 'schedule': 'linear warmup 5% then cosine decay', 'selection_rule': ZERO_MISREAD_RULE} + write(output/'config.json', config) + + def checkpoint(step, loss): + row = {'step': step, 'train_loss': loss, 'elapsed_s': round(time.perf_counter()-started, 1), 'sets': {}} + kept = {} + for name, bank in dev.items(): + probabilities = predict(model, bank, out_w, out_h, MARGIN) + labels = bank.y.cpu().numpy() + row['sets'][name] = field_outcomes(kind, probabilities, labels, latent=bank.latent)[1] + kept[name] = (probabilities, labels, bank.latent) + row['zero_misread'] = zero_misread({n: v[0] for n, v in kept.items()}, {n: v[1] for n, v in kept.items()}, + {n: v[2] for n, v in kept.items()}, roles, 105 if kind == 'member' else 100, + FIELD_MARGIN, FIELD_CONFIDENCE) + curve.append(row) + torch.save({k: v.detach().cpu() for k, v in model.state_dict().items()}, output/f'step-{step:06d}.pt') + write(output/'learning-curve.json', {'checkpoints': curve, 'scope': 'development sets only'}) + print(json.dumps({'checkpoint': step, **{k: (v['correct'], v['errors'], v['visible_total']) for k, v in row['sets'].items()}, + 'zero_misread': [row['zero_misread']['threshold'], row['zero_misread']['recall']]}), flush=True) + train_mode(model, freeze_bn) + + checkpoint(0, None) + train_mode(model, freeze_bn) + for step in range(steps): + n_hard = int(batch*hard_fraction) if hard else 0 + parts = [scenes.batch(torch.randint(len(scenes), (batch-n_hard,), device=device))] + if n_hard: + parts.append(hard.batch(torch.randint(len(hard), (n_hard,), device=device))) + x = torch.cat([p[0] for p in parts]) + y = torch.cat([p[1] for p in parts]) + x = photometric(jitter_crop(x, out_w, out_h, MARGIN, jitter_shift, jitter_scale)) + loss = F.cross_entropy(model(x), y) + optimizer.zero_grad(set_to_none=True) + loss.backward() + optimizer.step() + schedule.step() + if step == 0 or (step+1) % 100 == 0: + write(output/'progress.json', {'step': step+1, 'steps': steps, 'loss': float(loss.detach()), + 'lr': schedule.get_last_lr()[0], 'elapsed_s': round(time.perf_counter()-started, 1), + 'gpu_allocated_gib': round(torch.cuda.memory_allocated()/2**30, 2) if device.startswith('cuda') else None}) + if (step+1) % eval_every == 0 or step+1 == steps: + checkpoint(step+1, float(loss.detach())) + best = min(curve, key=zero_misread_key) + shutil.copy2(output/f"step-{best['step']:06d}.pt", output/'selected.pt') + model = field_model(kind) + model.load_state_dict(torch.load(output/'selected.pt', map_location='cpu', weights_only=True)) + onnx_path = output/f'{kind}-fields.onnx' + export_onnx(model, onnx_path, (1, 3, out_h, out_w)) + probe = next(iter(dev.values())) + samples = center_crop(probe.x[:256].float().div(255), out_w, out_h, MARGIN).cpu().numpy().astype(np.float32) + consistency = verify_onnx(model, onnx_path, samples) + write(output/'selection.json', {'selected_step': best['step'], 'selected': best, 'roles': roles, + 'rule': ZERO_MISREAD_RULE, + 'acceptance': {'confidence': (best['zero_misread']['threshold'] if best['zero_misread']['achievable'] + else FIELD_CONFIDENCE), 'margin': FIELD_MARGIN}, 'onnx': str(onnx_path), 'onnx_sha256': file_sha256(onnx_path), + 'pt_sha256': file_sha256(output/'selected.pt'), 'onnx_consistency': consistency, + 'elapsed_s': round(time.perf_counter()-started, 1)}) + print(json.dumps({'selected_step': best['step'], 'onnx_max_abs_diff': consistency['max_abs_logit_diff']}), flush=True) + + +def parse_dev(values): + """NAME:ROLE:PATH[,PATH...] with ROLE normal|stress|hard|monitor (monitor sets do not affect selection).""" + sets = {} + for value in values or []: + name, role, paths = value.split(':', 2) + if role not in ('normal', 'stress', 'hard', 'monitor'): + raise ValueError(f'Unknown development role: {role}') + sets[name] = ([Path(p) for p in paths.split(',')], role) + return sets + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--kind', choices=['member', 'snap'], required=True) + p.add_argument('--train', nargs='+', type=Path, required=True, help='Scene crop banks') + p.add_argument('--hard', nargs='*', type=Path, default=[], help='Acquisition-degraded crop banks') + p.add_argument('--dev', nargs='+', required=True, help='NAME:ROLE:PATH[,PATH] development sets (never the final test)') + p.add_argument('--initial', type=Path) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--steps', type=int, default=6000) + p.add_argument('--batch', type=int, default=512) + p.add_argument('--eval-every', type=int, default=500) + p.add_argument('--lr', type=float, default=1e-4) + p.add_argument('--hard-fraction', type=float, default=.4) + p.add_argument('--device', default='cuda') + p.add_argument('--threads', type=int, default=8) + p.add_argument('--freeze-bn', action='store_true', help='Keep the initial BatchNorm running statistics') + p.add_argument('--jitter-shift', nargs=2, type=float, default=[2.5, 2.5], metavar=('X', 'Y'), + help='Largest training crop shift in input pixels (uniform)') + p.add_argument('--jitter-scale', type=float, default=.04, help='Largest relative training crop scale change (uniform)') + a = p.parse_args() + train(a.kind, a.train, a.hard, parse_dev(a.dev), a.output, a.initial, a.steps, a.batch, a.eval_every, a.lr, a.hard_fraction, a.device, a.threads, + a.freeze_bn, tuple(a.jitter_shift), a.jitter_scale) diff --git a/bdon_vision/train_kind_rank.py b/bdon_vision/train_kind_rank.py new file mode 100644 index 0000000..65be4df --- /dev/null +++ b/bdon_vision/train_kind_rank.py @@ -0,0 +1,171 @@ +"""Train a per-kind card-rank icon classifier. + +Six classes: 0 = no readable owned-rank icon (hidden view, absent or cut), +1..5 = card rank. A prediction is accepted only when its softmax probability +and the gap to the runner-up both pass the thresholds in ``kind_fields``; +otherwise the rank is unknown. Checkpoint selection uses the same +zero-misread rule as the field reader, counting only crops whose icon lies +fully inside the image because deployed inference rejects the others by +geometry before the model runs. +""" +import argparse +import json +import shutil +import time +from pathlib import Path + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from .assets import write +from .kind_fields import MARGIN, RANK_CLASSES, RANK_CONFIDENCE, RANK_INPUT, RANK_MARGIN, RANK_SIDE, accept +from .train import seed_all +from .train_kind_fields import (ZERO_MISREAD_RULE, GpuBank, center_crop, export_onnx, file_sha256, jitter_crop, jitter_reach, + outcome_counts, parse_dev, photometric, predict, verify_onnx, zero_misread, + zero_misread_key) + +STORED_MARGIN = MARGIN*RANK_INPUT//RANK_SIDE + + +class RankNet(nn.Module): + """Compact CNN, input 3x48x48 RGB in [0, 1], output 6 logits.""" + + def __init__(self): + super().__init__() + + def block(a, b): + return [nn.Conv2d(a, b, 3, padding=1), nn.BatchNorm2d(b), nn.ReLU()] + self.layers = nn.Sequential(*block(3, 32), *block(32, 32), nn.MaxPool2d(2), + *block(32, 64), nn.MaxPool2d(2), + *block(64, 96), nn.MaxPool2d(2), + nn.Conv2d(96, 128, 3, padding=1), nn.ReLU(), nn.AvgPool2d(2)) + self.dropout = nn.Dropout(.2) + self.head = nn.Linear(128*3*3, RANK_CLASSES) + + def forward(self, x): + return self.head(self.dropout(self.layers(x).flatten(1))) + + +def rank_outcomes(probabilities, labels, inside=None, confidence=RANK_CONFIDENCE, margin=RANK_MARGIN, latent=None): + """Emitted rank (0 = none) and counts over crops whose icon is inside the image.""" + top, _, ok = accept(probabilities, confidence, margin) + value = np.where(ok & (top > 0), top, 0) + counts = outcome_counts(value, labels, inside, latent) + keep = np.ones(len(value), bool) if inside is None else np.asarray(inside).astype(bool) + counts['outside_emitted'] = int((~keep & (value > 0)).sum()) + return value, counts + + +def inside_flags(bank): + return np.concatenate([m['icon_inside'] for m in bank.meta]) if bank.meta else np.zeros(0, int) + + +def train(kind, train_banks, hard_banks, dev_sets, output, steps, batch, eval_every, lr, hard_fraction, device='cuda', threads=8, + jitter_shift=(2.5, 2.5), jitter_scale=.07): + if device.startswith('cuda') and not torch.cuda.is_available(): + raise RuntimeError('GPU not available') + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + seed_all(92017 if kind == 'member' else 92033) + torch.set_num_threads(threads) + side = RANK_INPUT + scenes = GpuBank(train_banks, kind, 'rank', device) + hard = GpuBank(hard_banks, kind, 'rank', device) if hard_banks else None + dev = {name: GpuBank(paths, kind, 'rank', device) for name, (paths, _) in dev_sets.items()} + inside = {name: inside_flags(bank) for name, bank in dev.items()} + roles = {name: role for name, (_, role) in dev_sets.items()} + model = RankNet().to(device) + optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4) + schedule = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, total_steps=steps, pct_start=.05) + started = time.perf_counter() + curve = [] + config = {'kind': kind, 'train_banks': [str(p) for p in train_banks], 'hard_banks': [str(p) for p in hard_banks], + 'dev_sets': {k: {'banks': [str(p) for p in v[0]], 'role': v[1]} for k, v in dev_sets.items()}, + 'steps': steps, 'batch': batch, 'eval_every': eval_every, 'lr': lr, 'hard_fraction': hard_fraction, + 'jitter': {'shift_px': list(jitter_shift), 'scale': jitter_scale, 'stored_margin_px': STORED_MARGIN, + 'reach_px': [round(v, 3) for v in jitter_reach(side, side, jitter_shift, jitter_scale)]}, + 'train_counts': {'scenes': len(scenes), 'hard': len(hard) if hard else 0}, + 'dev_counts': {k: len(v) for k, v in dev.items()}, + 'input': {'side': side, 'logical_side': RANK_SIDE}, + 'acceptance': {'confidence': RANK_CONFIDENCE, 'margin': RANK_MARGIN}, + 'selection_rule': ZERO_MISREAD_RULE+'; icon-inside crops only'} + write(output/'config.json', config) + + def checkpoint(step, loss): + row = {'step': step, 'train_loss': loss, 'elapsed_s': round(time.perf_counter()-started, 1), 'sets': {}} + kept = {} + for name, bank in dev.items(): + probabilities = predict(model, bank, side, side, STORED_MARGIN) + labels = bank.y.cpu().numpy() + row['sets'][name] = rank_outcomes(probabilities, labels, inside[name], latent=bank.latent)[1] + keep = inside[name].astype(bool) + kept[name] = (probabilities[keep], labels[keep], bank.latent[keep]) + row['zero_misread'] = zero_misread({n: v[0] for n, v in kept.items()}, {n: v[1] for n, v in kept.items()}, + {n: v[2] for n, v in kept.items()}, roles, 5, RANK_MARGIN, RANK_CONFIDENCE) + curve.append(row) + torch.save({k: v.detach().cpu() for k, v in model.state_dict().items()}, output/f'step-{step:06d}.pt') + write(output/'learning-curve.json', {'checkpoints': curve, 'scope': 'development sets only'}) + print(json.dumps({'checkpoint': step, **{k: (v['correct'], v['errors'], v['visible_total']) for k, v in row['sets'].items()}, + 'zero_misread': [row['zero_misread']['threshold'], row['zero_misread']['recall']]}), flush=True) + model.train() + + model.train() + for step in range(steps): + n_hard = int(batch*hard_fraction) if hard else 0 + parts = [scenes.batch(torch.randint(len(scenes), (batch-n_hard,), device=device))] + if n_hard: + parts.append(hard.batch(torch.randint(len(hard), (n_hard,), device=device))) + x = torch.cat([p[0] for p in parts]) + y = torch.cat([p[1] for p in parts]) + x = photometric(jitter_crop(x, side, side, STORED_MARGIN, jitter_shift, jitter_scale)) + loss = F.cross_entropy(model(x), y) + optimizer.zero_grad(set_to_none=True) + loss.backward() + optimizer.step() + schedule.step() + if step == 0 or (step+1) % 100 == 0: + write(output/'progress.json', {'step': step+1, 'steps': steps, 'loss': float(loss.detach()), + 'lr': schedule.get_last_lr()[0], 'elapsed_s': round(time.perf_counter()-started, 1), + 'gpu_allocated_gib': round(torch.cuda.memory_allocated()/2**30, 2) if device.startswith('cuda') else None}) + if (step+1) % eval_every == 0 or step+1 == steps: + checkpoint(step+1, float(loss.detach())) + best = min(curve, key=zero_misread_key) + shutil.copy2(output/f"step-{best['step']:06d}.pt", output/'selected.pt') + model = RankNet() + model.load_state_dict(torch.load(output/'selected.pt', map_location='cpu', weights_only=True)) + onnx_path = output/f'{kind}-rank.onnx' + export_onnx(model, onnx_path, (1, 3, side, side)) + probe = next(iter(dev.values())) + samples = center_crop(probe.x[:256].float().div(255), side, side, STORED_MARGIN).cpu().numpy().astype(np.float32) + consistency = verify_onnx(model, onnx_path, samples) + write(output/'selection.json', {'selected_step': best['step'], 'selected': best, 'roles': roles, + 'rule': config['selection_rule'], + 'acceptance': {'confidence': (best['zero_misread']['threshold'] if best['zero_misread']['achievable'] + else RANK_CONFIDENCE), 'margin': RANK_MARGIN}, 'onnx': str(onnx_path), 'onnx_sha256': file_sha256(onnx_path), + 'pt_sha256': file_sha256(output/'selected.pt'), 'onnx_consistency': consistency, + 'elapsed_s': round(time.perf_counter()-started, 1)}) + print(json.dumps({'selected_step': best['step'], 'onnx_max_abs_diff': consistency['max_abs_logit_diff']}), flush=True) + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--kind', choices=['member', 'snap'], required=True) + p.add_argument('--train', nargs='+', type=Path, required=True) + p.add_argument('--hard', nargs='*', type=Path, default=[]) + p.add_argument('--dev', nargs='+', required=True, help='NAME:ROLE:PATH[,PATH] development sets (never the final test)') + p.add_argument('--output', type=Path, required=True) + p.add_argument('--steps', type=int, default=6000) + p.add_argument('--batch', type=int, default=256) + p.add_argument('--eval-every', type=int, default=500) + p.add_argument('--lr', type=float, default=2e-3) + p.add_argument('--hard-fraction', type=float, default=.4) + p.add_argument('--device', default='cuda') + p.add_argument('--threads', type=int, default=8) + p.add_argument('--jitter-shift', nargs=2, type=float, default=[2.5, 2.5], metavar=('X', 'Y'), + help='Largest training crop shift in input pixels (uniform)') + p.add_argument('--jitter-scale', type=float, default=.07, help='Largest relative training crop scale change (uniform)') + a = p.parse_args() + train(a.kind, a.train, a.hard, parse_dev(a.dev), a.output, a.steps, a.batch, a.eval_every, a.lr, a.hard_fraction, a.device, a.threads, + tuple(a.jitter_shift), a.jitter_scale) diff --git a/tests/test_kind_fields.py b/tests/test_kind_fields.py new file mode 100644 index 0000000..250d664 --- /dev/null +++ b/tests/test_kind_fields.py @@ -0,0 +1,211 @@ +"""Per-kind field/rank crop geometry, labels and acceptance rules on synthetic inputs.""" +import unittest + +import numpy as np + +from bdon_vision.kind_fields import (FIELD_STORED, MARGIN, RANK_INPUT, RANK_SIDE, RANK_STORED, crop_field, crop_rank, + decide_field, decide_rank, field_inside, field_label, field_region, + latent_field, misread_sweep, rank_icon_inside, rank_label, rank_region) + + +def one_hot(classes, index, confidence=.999): + p = np.full(classes, (1-confidence)/(classes-1)) + p[index] = confidence + return p + + +class GeometryTests(unittest.TestCase): + def test_field_region_follows_tile_scale(self): + left, top, w, h = field_region('member', [100., 50., 112., 147.]) + s = 112/224 + self.assertAlmostEqual(left, 100-6*s) + self.assertAlmostEqual(top, 50+147-56*s) + self.assertAlmostEqual(w, 144*s) + self.assertAlmostEqual(h, 56*s) + left, top, w, h = field_region('snap', [10., 20., 652., 368.]) + self.assertEqual((left, top, w, h), (10-12., 20+368-112., 288., 112.)) + + def test_rank_region_is_centred_on_icon(self): + left, top, w, h = rank_region('member', [0., 0., 224., 294.]) + self.assertEqual((left+w/2, top+h/2, w, h), (194.5, 267.2, 96., 96.)) + left, top, w, h = rank_region('snap', [0., 0., 163., 92.]) + self.assertAlmostEqual(left+w/2, 306.1/2) + self.assertAlmostEqual(top+h/2, 153.8/2) + self.assertAlmostEqual(w, 48.) + + def test_crop_reads_the_expected_pixels(self): + image = np.zeros((400, 600, 3), np.uint8) + tile = [100., 40., 224., 294.] + left, top, w, h = (round(v) for v in field_region('member', tile)) + image[top:top+h, left:left+w] = (10, 20, 30) + crop = crop_field(image, 'member', tile) + self.assertEqual(crop.shape, (56, 144, 3)) + # BGR input, RGB output. + self.assertTrue((crop[1:-1, 1:-1] == (30, 20, 10)).all()) + rank = crop_rank(image, 'member', tile) + self.assertEqual(rank.shape, (RANK_INPUT, RANK_INPUT, 3)) + + def test_stored_margin_centre_equals_deployed_crop(self): + # Smooth content: warpAffine quantizes source positions to 1/32 pixel. + yy, xx = np.mgrid[:500, :700] + image = np.stack([127+100*np.sin(xx/7.)*np.cos(yy/9.), 127+90*np.cos(xx/11.), 127+80*np.sin(yy/5.)], -1) + image = image.astype(np.uint8) + for kind, tile in [('member', [123.4, 56.7, 151.3, 198.6]), ('snap', [80.2, 90.1, 260.8, 147.2])]: + stored = crop_field(image, kind, tile, MARGIN) + self.assertEqual(stored.shape, (FIELD_STORED[1], FIELD_STORED[0], 3)) + direct = crop_field(image, kind, tile) + centre = stored[MARGIN:MARGIN+56, MARGIN:MARGIN+144] + self.assertLessEqual(np.abs(centre.astype(int)-direct.astype(int)).max(), 2) + stored = crop_rank(image, kind, tile, MARGIN) + self.assertEqual(stored.shape, (RANK_STORED, RANK_STORED, 3)) + m = MARGIN*RANK_INPUT//RANK_SIDE + direct = crop_rank(image, kind, tile) + centre = stored[m:m+RANK_INPUT, m:m+RANK_INPUT] + self.assertLessEqual(np.abs(centre.astype(int)-direct.astype(int)).max(), 2) + + def test_outside_checks(self): + self.assertTrue(field_inside('member', [100., 40., 224., 294.], 600, 400)) + self.assertFalse(field_inside('member', [100., 120., 224., 294.], 600, 400)) + self.assertFalse(field_inside('member', [5., 40., 224., 294.], 600, 400)) + self.assertTrue(rank_icon_inside('snap', [10., 10., 326., 184.], 400, 300)) + self.assertFalse(rank_icon_inside('snap', [80., 10., 326., 184.], 400, 300)) + + +class LabelTests(unittest.TestCase): + def test_field_labels(self): + self.assertEqual(field_label('member', {'display_mode': 'level', 'level': 37}), 37) + self.assertIsNone(field_label('member', {'display_mode': 'level', 'level': None})) + self.assertEqual(field_label('member', {'display_mode': 'training', 'awake_count': 4}), 104) + self.assertIsNone(field_label('snap', {'display_mode': 'training', 'awake_count': 4})) + self.assertEqual(field_label('snap', {'display_mode': 'visual', 'field_visible': True}), 0) + occluded = {'display_mode': 'level', 'level': None, 'field_visible': False, 'latent_state': {'level': 37}} + self.assertEqual(field_label('member', occluded), 0) + + def test_rank_labels_never_use_latent_state(self): + self.assertEqual(rank_label({'card_rank': 3, 'display_mode': 'level'}), 3) + hidden = {'card_rank': None, 'display_mode': 'hide', 'latent_state': {'card_rank': 4}} + self.assertEqual(rank_label(hidden), 0) + + +class MisreadSweepTests(unittest.TestCase): + def test_zero_misread_threshold_and_recall(self): + p = np.array([one_hot(6, 3, .9999), one_hot(6, 2, .997), one_hot(6, 4, .996), one_hot(6, 5, .999)]) + truth = np.array([3, 2, 1, 0]) + latent = np.array([3, 2, 1, 5]) + sweep = misread_sweep(p, truth, latent, margin=.5, default=.995) + # Only the 0.996 read of a rank-1 icon is wrong; the hidden read matches the card state. + self.assertAlmostEqual(sweep['highest_misread_confidence'], .996) + self.assertGreater(sweep['zero_misread_threshold'], .996) + self.assertEqual(sweep['at_zero_misread_threshold']['misreads'], 0) + self.assertAlmostEqual(sweep['at_zero_misread_threshold']['recall'], 2/3) + self.assertEqual(sweep['at_default_threshold']['misreads'], 1) + unknown_state = misread_sweep(p, truth, None, margin=.5) + self.assertAlmostEqual(unknown_state['highest_misread_confidence'], .999) + + def test_latent_value_is_analysis_only(self): + card = {'display_mode': 'level', 'level': None, 'field_visible': False, 'latent_state': {'level': 12}} + self.assertEqual(field_label('member', card), 0) + self.assertEqual(latent_field('member', card), 12) + + +class DecisionTests(unittest.TestCase): + def test_member_field_decisions(self): + self.assertEqual(decide_field('member', one_hot(106, 42))['level'], 42) + training = decide_field('member', one_hot(106, 103)) + self.assertEqual((training['awake_count'], training['display_mode']), (3, 'training')) + other = decide_field('member', one_hot(106, 0)) + self.assertEqual((other['level'], other['display_mode']), (None, 'other')) + self.assertIsNone(decide_field('member', one_hot(106, 42, .99))['level']) + + def test_snap_field_decisions(self): + self.assertEqual(decide_field('snap', one_hot(101, 100))['level'], 100) + # A shared 106-class model may emit a training class; Snap has none. + unsupported = decide_field('snap', one_hot(106, 102)) + self.assertIsNone(unsupported['level']) + self.assertIsNone(unsupported['awake_count']) + + def test_margin_rule(self): + p = np.zeros(101) + p[5], p[6] = .996, .004 + self.assertEqual(decide_field('snap', p)['level'], 5) + p = np.zeros(6) + p[2], p[3] = .7, .3 + self.assertIsNone(decide_rank(p)['value']) + + def test_rank_decisions(self): + self.assertEqual(decide_rank(one_hot(6, 4))['value'], 4) + none = decide_rank(one_hot(6, 0)) + self.assertIsNone(none['value']) + self.assertEqual(none['reason'], 'no_visible_rank_icon') + + +try: + import torch +except ImportError: + torch = None + + +@unittest.skipIf(torch is None, 'PyTorch not installed') +class ModelTests(unittest.TestCase): + def test_shapes(self): + from bdon_vision.train_kind_fields import field_model + from bdon_vision.train_kind_rank import RankNet + self.assertEqual(tuple(field_model('member')(torch.zeros(2, 3, 56, 144)).shape), (2, 106)) + self.assertEqual(tuple(field_model('snap')(torch.zeros(2, 3, 56, 144)).shape), (2, 101)) + self.assertEqual(tuple(RankNet().eval()(torch.zeros(3, 3, 48, 48)).shape), (3, 6)) + + def test_snap_head_initialised_from_leading_rows(self): + import tempfile + from pathlib import Path + from bdon_vision.train_kind_fields import field_model, initialise_from + member = field_model('member') + with tempfile.TemporaryDirectory() as folder: + path = Path(folder)/'shared.pt' + torch.save(member.state_dict(), path) + snap = initialise_from(field_model('snap'), path) + self.assertTrue(torch.equal(snap.head.weight, member.head.weight[:101])) + self.assertTrue(torch.equal(snap.layers[0].weight, member.layers[0].weight)) + + def test_unjittered_crop_is_the_stored_centre(self): + from bdon_vision.train_kind_fields import center_crop, jitter_crop + x = torch.rand(2, 3, FIELD_STORED[1], FIELD_STORED[0]) + same = jitter_crop(x, 144, 56, MARGIN, 0., 0.) + self.assertLess(float((same-center_crop(x, 144, 56, MARGIN)).abs().max()), 1e-5) + + def test_anisotropic_jitter_shift(self): + from bdon_vision.train_kind_fields import center_crop, jitter_crop, jitter_reach + h, w = FIELD_STORED[1], FIELD_STORED[0] + columns = torch.arange(w, dtype=torch.float32).view(1, 1, 1, w).expand(64, 1, h, w) + rows = torch.arange(h, dtype=torch.float32).view(1, 1, h, 1).expand(64, 1, h, w) + x = torch.cat([columns, rows], 1) + torch.manual_seed(0) + moved = jitter_crop(x, 144, 56, MARGIN, (3., 0.), 0.)-center_crop(x, 144, 56, MARGIN) + dx = moved[:, 0, 20:36, 20:124] + self.assertLess(float((dx-dx[:, :1, :1]).abs().max()), 1e-3) + self.assertLessEqual(float(dx.abs().max()), 3.+1e-4) + self.assertGreater(float(dx.abs().max()), 1.) + self.assertLess(float(moved[:, 1].abs().max()), 1e-3) + reach = jitter_reach(144, 56, (4., 7.), .035) + self.assertAlmostEqual(reach[0], 4+.035*71.5) + self.assertAlmostEqual(reach[1], 7+.035*27.5) + + def test_onnx_export_has_variable_batch(self): + try: + import onnxruntime # noqa: F401 + except ImportError: + self.skipTest('ONNX Runtime not installed') + import tempfile + from pathlib import Path + from bdon_vision.train_kind_fields import export_onnx, verify_onnx + from bdon_vision.train_kind_rank import RankNet + with tempfile.TemporaryDirectory() as folder: + path = Path(folder)/'rank.onnx' + model = RankNet() + export_onnx(model, path, (1, 3, 48, 48)) + report = verify_onnx(model, path, np.random.default_rng(0).random((70, 3, 48, 48), dtype=np.float32)) + self.assertLess(report['max_abs_logit_diff'], 1e-4) + self.assertIn('70', report['batches']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_kind_pose.py b/tests/test_kind_pose.py new file mode 100644 index 0000000..b63538a --- /dev/null +++ b/tests/test_kind_pose.py @@ -0,0 +1,90 @@ +"""Detector pose error against truth tiles and its effect on field and rank crops, on synthetic rectangles.""" +import unittest + +import numpy as np + +from bdon_vision.kind_fields import LOGICAL, misread_sweep +from bdon_vision.kind_pose import normalise, pair, paired_tiles, pose_error, summary + + +class PairTests(unittest.TestCase): + def test_greedy_one_to_one_pairing(self): + truth = [[0, 0, 100, 100], [200, 0, 100, 100]] + detections = [[205, 0, 100, 100], [2, 0, 100, 100], [100, 0, 100, 100], [600, 0, 50, 50]] + self.assertEqual(pair(truth, detections), {0: 1, 1: 0}) + self.assertEqual(pair(truth, [[60, 0, 100, 100]]), {}) + + def test_paired_tiles_counts_misses_kind_confusions_and_ignored_cards(self): + row = {'cards': [{'kind': 'member', 'bbox': [0, 0, 224, 294]}, + {'kind': 'snap', 'bbox': [300, 0, 326, 184]}, + {'kind': 'member', 'bbox': [0, 400, 224, 294]}], + 'ignored_cards': [{'kind': 'member', 'bbox': [700, 0, 224, 294]}]} + found = [{'kind': 'member', 'bbox': [2, 1, 224, 294]}, {'kind': 'member', 'bbox': [301, 0, 326, 184]}, + {'kind': 'member', 'bbox': [702, 0, 224, 294]}, {'kind': 'snap', 'bbox': [1500, 900, 326, 184]}] + tiles, counts = paired_tiles(row, found) + self.assertEqual(tiles, [[2, 1, 224, 294], None, None]) + self.assertEqual(counts['member:cards'], 2) + self.assertEqual(counts['member:missed'], 1) + self.assertEqual(counts['snap:kind_confused'], 1) + self.assertEqual(counts['unpaired_detections'], 1) + + def test_layout_tile_key(self): + row = {'cards': [{'kind': 'member', 'bbox': [0, 0, 10, 10], 'bbox_layout': [0, 0, 224, 294]}]} + tiles, _ = paired_tiles(row, [{'kind': 'member', 'bbox': [1, 0, 224, 294]}], tile_key='bbox_layout') + self.assertEqual(tiles, [[1, 0, 224, 294]]) + + +class PoseErrorTests(unittest.TestCase): + def test_identity_has_no_error(self): + error = pose_error('snap', [10., 20., 163., 92.], [10., 20., 163., 92.]) + self.assertTrue(all(abs(v) < 1e-12 for v in error.values())) + + def test_shift_in_logical_units_and_crop_pixels(self): + s = .5 + truth = [100., 50., 224*s, 294*s] + error = pose_error('member', truth, [100.+2*s, 50.-3*s, 224*s, 294*s]) + for key, value in {'centre_dx': 2, 'centre_dy': -3, 'width': 0, 'height': 0, 'scale': 0, 'field_dx': 2, + 'field_dy': -3, 'field_reach_x': 2, 'field_reach_y': 3, 'rank_dx': 1, 'rank_dy': -1.5, + 'rank_reach': 1.5}.items(): + self.assertAlmostEqual(error[key], value, msg=key) + + def test_scale_about_the_tile_origin_moves_the_crops(self): + error = pose_error('member', [0., 0., 224., 294.], [0., 0., 224*1.1, 294*1.1]) + self.assertAlmostEqual(error['scale'], .1) + self.assertAlmostEqual(error['field_dx'], 66*.1) + self.assertAlmostEqual(error['field_dy'], (294-28)*.1) + self.assertAlmostEqual(error['field_reach_x'], 66*.1+72*.1) + self.assertAlmostEqual(error['rank_dx'], 194.5*.1/2) + self.assertAlmostEqual(error['rank_dy'], 267.2*.1/2) + + def test_normalise_keeps_centre_and_logical_aspect(self): + tile = [10., 20., 230., 280.] + x, y, w, h = normalise('member', tile) + self.assertAlmostEqual(x+w/2, 10+115) + self.assertAlmostEqual(y+h/2, 20+140) + self.assertAlmostEqual(w/h, LOGICAL['member'][0]/LOGICAL['member'][1]) + self.assertAlmostEqual(w/224, (230/224+280/294)/2) + + def test_summary_uses_absolute_quantiles_and_signed_mean(self): + report = summary([-4., 1., 2., 3.]) + self.assertEqual(report['n'], 4) + self.assertEqual(report['mean'], .5) + self.assertEqual(report['max'], 4.) + self.assertEqual(report['p50'], 2.5) + self.assertEqual(summary([]), {'n': 0}) + + +class UnreadCardTests(unittest.TestCase): + def test_unread_cards_never_emit_a_value(self): + from bdon_vision.evaluate_kind_fields import scatter + read = np.array([[.001, .999, 0.], [.999, .001, 0.]]) + full = scatter(read, [0, 2], 3) + self.assertEqual(full.shape, (3, 3)) + self.assertTrue(np.array_equal(full[1], [1., 0., 0.])) + sweep = misread_sweep(full, np.array([1, 2, 0]), limit=2) + self.assertEqual(sweep['at_zero_misread_threshold'], {'misreads': 0, 'recall': .5}) + self.assertIsNone(scatter(None, [], 3)) + + +if __name__ == '__main__': + unittest.main() From 168e0318176c85093f8152eff034f70df0d18e8b Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:45:40 +0800 Subject: [PATCH 5/7] feat: scan with the neural locator, per-kind encoders and field readers Engine.scan now runs the card-frame locator on the whole screenshot, identifies each tile by nearest-neighbour search in a per-kind artwork gallery and reads level, training count and card rank with the per-kind field and rank models. All seven models and their acceptance thresholds are pinned by models/recognition.json and checked by SHA-256 at start-up. - tiles whose identity is not accepted are returned in a new `unidentified` list with the nearest gallery card as a review candidate; they never enter the merged box - galleries are cached per kind in models/-gallery.npz, keyed by the encoder hash, catalog order and reference artwork bytes, and rebuilt by `prepare`; new cards need no model change - fields and rank icons that are not fully inside the image stay unknown - observations schema v2 and scan-result/2 describe the new card fields, the `unidentified` list and recognition provenance v2 - evaluate_engine scores any engine through the scan contract: one-to-one tile pairing, identity, foreign-art rejection, visible and hidden fields, stratified by kind, locale, view, resolution, card width and occlusion - verify_gallery_update checks that removed cards come back unidentified and are identified again after re-adding them, with unchanged models - the template, RootSIFT and OCR inference paths and their diagnostics are removed, together with training modules kept only in the frozen reproduction stages --- bdon_vision/__init__.py | 2 + bdon_vision/assets.py | 53 +--- bdon_vision/benchmark.py | 6 +- bdon_vision/deck/adapter.py | 3 +- bdon_vision/deck/visual_data.py | 6 +- bdon_vision/diagnose_fields.py | 70 ----- bdon_vision/engine.py | 441 +++++++++++++--------------- bdon_vision/evaluate_engine.py | 243 +++++++++++++++ bdon_vision/evaluate_kind_fields.py | 217 ++------------ bdon_vision/evaluate_locator.py | 58 +--- bdon_vision/evaluate_open_set.py | 50 ---- bdon_vision/fielddata.py | 105 ------- bdon_vision/inference.py | 42 --- bdon_vision/ocr.py | 102 ------- bdon_vision/server.py | 2 +- bdon_vision/train_fields.py | 117 -------- bdon_vision/train_scene_encoder.py | 111 ------- bdon_vision/web.js | 2 + docs/DECK_PIPELINE.md | 4 +- schemas/box.schema.json | 27 +- schemas/examples/scan-response.json | 3 +- schemas/observations.schema.json | 259 ++++++++++++++-- schemas/scan-response.schema.json | 286 ++++++++++++++++-- scripts/diagnose_occlusion.py | 127 -------- scripts/publish_metrics.py | 66 ----- scripts/verify_gallery_update.py | 106 ++++--- scripts/verify_weights.py | 41 --- tests/test_deck_inputs.py | 8 +- tests/test_integrity.py | 95 +++++- tests/test_ui_geometry.py | 32 +- 30 files changed, 1170 insertions(+), 1514 deletions(-) delete mode 100644 bdon_vision/diagnose_fields.py create mode 100644 bdon_vision/evaluate_engine.py delete mode 100644 bdon_vision/evaluate_open_set.py delete mode 100644 bdon_vision/fielddata.py delete mode 100644 bdon_vision/inference.py delete mode 100644 bdon_vision/ocr.py delete mode 100644 bdon_vision/train_fields.py delete mode 100644 bdon_vision/train_scene_encoder.py delete mode 100644 scripts/diagnose_occlusion.py delete mode 100644 scripts/publish_metrics.py delete mode 100644 scripts/verify_weights.py diff --git a/bdon_vision/__init__.py b/bdon_vision/__init__.py index d6645c5..ea65f15 100644 --- a/bdon_vision/__init__.py +++ b/bdon_vision/__init__.py @@ -1 +1,3 @@ """CPU-only BDON member / Snap screenshot importer.""" + +__version__ = '0.2.0' diff --git a/bdon_vision/assets.py b/bdon_vision/assets.py index b2645a4..955155e 100644 --- a/bdon_vision/assets.py +++ b/bdon_vision/assets.py @@ -8,10 +8,12 @@ from functools import lru_cache from pathlib import Path -import cv2 -import numpy as np from PIL import Image, ImageOps +from . import __version__ + +USER_AGENT = f'ournotes-boxlens/{__version__}' + def read(path): return json.loads(Path(path).read_text(encoding='utf-8')) @@ -118,7 +120,7 @@ def fetch(row): url = f'https://assets.bdon.moe/zh-Hans/{directory}/{asset}/{stem}/{stem}.webp' if not f.exists(): if not allow_download:raise FileNotFoundError(f'Missing local artwork: {f.name}') - request = urllib.request.Request(url, headers={'Referer': 'https://bdon.moe/', 'User-Agent': 'BDON-box-vision/0.1'}) + request = urllib.request.Request(url, headers={'Referer': 'https://bdon.moe/', 'User-Agent': USER_AGENT}) with urllib.request.urlopen(request, timeout=45) as response: body = response.read() f.write_bytes(body) @@ -131,7 +133,7 @@ def fetch(row): square_url = f'https://assets.bdon.moe/zh-Hans/MemberCard/{asset}/member_thumbnail/square.webp' if not square.exists(): if not allow_download:raise FileNotFoundError(f'Missing local artwork: {square.name}') - req = urllib.request.Request(square_url, headers={'Referer':'https://bdon.moe/', 'User-Agent':'BDON-box-vision/0.1'}) + req = urllib.request.Request(square_url, headers={'Referer':'https://bdon.moe/', 'User-Agent':USER_AGENT}) with urllib.request.urlopen(req, timeout=45) as response: square.write_bytes(response.read()) with Image.open(square) as im: @@ -160,42 +162,7 @@ def fetch(row): print(json.dumps({'catalog': len(completed), 'unavailable': failures}), flush=True) if not completed: raise RuntimeError('No artwork available') - build_index(data) - if (data/'models/encoder.onnx').exists(): - from .inference import Gallery - Gallery(data,read(data/'catalog.json')['cards']) - print(json.dumps({'embedding_gallery':'updated'}),flush=True) - - -def build_index(data): - data = Path(data) - cv2.setNumThreads(2) - sift = cv2.SIFT_create(nfeatures=260, contrastThreshold=.018, edgeThreshold=12) - descriptors, points, owners = [], [], [] - cards = read(data / 'catalog.json')['cards'] - for i, card in enumerate(cards): - im = cv2.imread(str(data / card.get('match_file', card['file']))) - # Match at list-thumbnail resolution; original coordinates are retained. - h, w = im.shape[:2] - scales = [min(1., 220 / w)] - for scale in scales: - gray = cv2.cvtColor(cv2.resize(im, None, fx=scale, fy=scale), cv2.COLOR_BGR2GRAY) - mask = np.full(gray.shape, 255, np.uint8) - mh, mw = mask.shape - mask[int(mh*.82):, :int(mw*.4)] = 0 - mask[int(mh*.72):, int(mw*.76):] = 0 - mask[:int(mh*.18), :int(mw*.15)] = 0 - kp, desc = sift.detectAndCompute(gray, mask) - if desc is None: - continue - # RootSIFT improves matching under screenshot compression. - desc = np.sqrt(desc / (desc.sum(axis=1, keepdims=True) + 1e-8)) - descriptors.append(desc) - points.extend([(k.pt[0] / scale, k.pt[1] / scale) for k in kp]) - owners.extend([i] * len(kp)) - card['width'], card['height'] = w, h - np.savez_compressed(data / 'index.npz', descriptors=np.vstack(descriptors), - points=np.array(points, np.float32), owners=np.array(owners, np.int32)) - write(data / 'catalog.json', {'schema': 'bdon-box-catalog/1', 'cards': cards, - 'unavailable': read(data / 'catalog.json').get('unavailable', [])}) - print(json.dumps({'index_descriptors': len(points)}), flush=True) + if (data/'models/recognition.json').exists(): + from .engine import Engine + engine=Engine(data) + print(json.dumps({'galleries':{kind:len(g.indices) for kind,g in engine.galleries.items()}}),flush=True) diff --git a/bdon_vision/benchmark.py b/bdon_vision/benchmark.py index 06afb47..9b77e8c 100644 --- a/bdon_vision/benchmark.py +++ b/bdon_vision/benchmark.py @@ -66,18 +66,18 @@ def benchmark(data,images,output,cpus=2,repeats=3): cpu_start=time.process_time() result=engine.scan(image,path.name) records.append({'file':path.name,'repeat':repeat,'pixels':[image.shape[1],image.shape[0]], - 'cards':len(result['cards']),'elapsed_ms':result['elapsed_ms'], + 'cards':len(result['cards']),'unidentified':len(result['unidentified']),'elapsed_ms':result['elapsed_ms'], 'process_cpu_ms':round((time.process_time()-cpu_start)*1000,2)}) times=[r['elapsed_ms'] for r in records] report={'logical_cpu_affinity':affinity,'threads':cpus,'platform':platform.platform(), 'processor':platform.processor(),'host_logical_cpus':os.cpu_count(), - 'model_sha256':{n:hashlib.sha256((Path(data)/'models'/n).read_bytes()).hexdigest() for n in ['encoder.onnx','fields.onnx']}, + 'model_manifest_sha256':hashlib.sha256((Path(data)/'models/recognition.json').read_bytes()).hexdigest(), 'startup_ms':round(startup,2),'startup_memory':startup_memory,'final_memory':memory_usage(), 'input_sha256':{p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in images}, 'warmup':'one pass of all images','repeats':repeats, 'median_ms':float(np.median(times)),'p95_ms':float(np.percentile(times,95)), 'min_ms':min(times),'max_ms':max(times),'measurements':records, - 'scope':'Same process constrained to listed logical CPUs. Scan includes localization and field reading; excludes file decoding and HTTP. Other host workloads are not controlled.'} + 'scope':'Same process constrained to listed logical CPUs. Scan includes localization, identity retrieval and field and rank reading; excludes file decoding and HTTP. Other host workloads are not controlled.'} write(output,report);print({k:v for k,v in report.items() if k!='measurements'}) diff --git a/bdon_vision/deck/adapter.py b/bdon_vision/deck/adapter.py index 0442363..a1fbf97 100644 --- a/bdon_vision/deck/adapter.py +++ b/bdon_vision/deck/adapter.py @@ -1,5 +1,6 @@ """Strict evidence gate in front of the EXISTING Rust Roster, no scoring model.""" from copy import deepcopy +from .. import __version__ from .common import read, digest, rows, load_deck FIELDS = {'member': {'level': 'level', 'awake': 'awake_count', 'rank': 'card_rank'}, @@ -23,7 +24,7 @@ def inventory(box, deck_path, *, mock=False, region=None, fixture_manifest=None, return {'schema': 'ournotes.player-inventory/1', 'source': 'boxlens', 'mock': mock, 'region': region, 'deckDataSha256': deck_sha, 'masterVersion': deck['provenance']['master']['version'], - 'boxlensVersion': '0.1.0', 'boxlensCommit':'2ea25d8e2fdcbd3d1514b101a13b34d3e70d2ff1', + 'boxlensVersion': __version__, 'modelScope':'Offline native model 1.0.1-25; JP client 1.0.4 equivalence not inferred', 'box': deepcopy(box), 'recognitionProvenance': box.get('recognition_provenance', []), 'recognitionDataset':deepcopy(recognition_dataset),'fixtureManifest': fixture_manifest, 'coverage': 'observed_only'} diff --git a/bdon_vision/deck/visual_data.py b/bdon_vision/deck/visual_data.py index 16edc98..52073cc 100644 --- a/bdon_vision/deck/visual_data.py +++ b/bdon_vision/deck/visual_data.py @@ -6,7 +6,7 @@ def binding(data,deck_path): data=Path(data).resolve() deck,deck_sha=load_deck(deck_path) - required=['catalog.json','index.npz','models/encoder.onnx','models/fields.onnx'] + required=['catalog.json','models/recognition.json'] absent=[name for name in required if not (data/name).is_file()] if absent:raise ValueError('Visual dataset missing required files: '+', '.join(absent)) cat=read(data/'catalog.json') @@ -24,7 +24,7 @@ def binding(data,deck_path): catalog=deck['provenance'].get('catalog',{}) for key,deck_key in [('resourceVersion','resourceVersion'),('resourceHash','resourceHash'),('officialCatalogSha256','sha256')]: if key in manifest and manifest[key]!=catalog.get(deck_key):raise ValueError('Visual bundle '+key+' does not match DeckData') - files={name:digest(data/name) for name in ['catalog.json','index.npz','models/encoder.onnx','models/fields.onnx','models/gallery.npz','master/MasterMemberCard.json','master/MasterSupportCard.json'] if (data/name).exists()} + files={name:digest(data/name) for name in ['catalog.json','models/recognition.json','models/member-gallery.npz','models/snap-gallery.npz','master/MasterMemberCard.json','master/MasterSupportCard.json'] if (data/name).exists()} if manifest: claimed=manifest.get('files',{}) if not isinstance(claimed,dict):raise ValueError('Visual manifest files must be a hash map') @@ -33,6 +33,6 @@ def binding(data,deck_path): if data not in path.parents or not path.is_file():raise ValueError('Visual manifest file absent or outside dataset: '+name) expected=value.get('sha256') if isinstance(value,dict) else value if not isinstance(expected,str) or digest(path)!=expected:raise ValueError('Visual dataset manifest hash mismatch: '+name) - for key,name in [('catalogSha256','catalog.json'),('indexSha256','index.npz'),('encoderSha256','models/encoder.onnx'),('fieldModelSha256','models/fields.onnx'),('gallerySha256','models/gallery.npz')]: + for key,name in [('catalogSha256','catalog.json'),('recognitionModelsSha256','models/recognition.json')]: if key in manifest and manifest[key]!=files.get(name):raise ValueError('Visual dataset manifest hash mismatch: '+name) return {'schema':'ournotes.loaded-visual-data/1','dataPath':str(data),'boundDeckDataSha256':deck_sha,'boundDeckRegion':deck['provenance']['region'],'boundDeckMasterVersion':deck['provenance']['master']['version'],'identityCounts':dict(Counter(k for k,_ in present)),'identities':[{'kind':k,'id':i} for k,i in sorted(present)],'missingBoundIdentities':[{'kind':k,'id':i} for k,i in missing],'extraIdentities':[{'kind':k,'id':i} for k,i in extra],'coversBoundMasterIdentities':not missing and not extra,'files':files,'datasetManifest':manifest,'datasetManifestSha256':digest(manifest_path) if manifest else None,'scope':'Visual recognition capability only; not player ownership, runtime equivalence, or OCR field accuracy'} diff --git a/bdon_vision/diagnose_fields.py b/bdon_vision/diagnose_fields.py deleted file mode 100644 index b6b559f..0000000 --- a/bdon_vision/diagnose_fields.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Separate field-reader errors from missing or inaccurate localization, using development data.""" -import argparse -from collections import Counter,defaultdict -from pathlib import Path - -import cv2 - -from .assets import read,write -from .ocr import NumberReader,FIELD_CONFIDENCE -from .synthetic import bbox_iou - - -def encoded(values): - if values['awake_count']['value'] is not None:return 100+values['awake_count']['value'] - return values['level']['value'] - - -def count_result(counts,prefix,truth,value): - if truth==0: - counts[prefix+'_hidden_total']+=1;counts[prefix+'_false_visible']+=int(value is not None) - else: - counts[prefix+'_visible_total']+=1 - counts[prefix+'_correct']+=int(value==truth) - counts[prefix+'_unknown']+=int(value is None) - counts[prefix+'_wrong']+=int(value is not None and value!=truth) - - -def diagnose(data,dataset,observations,output): - reader=NumberReader(data,2);truth=read(dataset/'truth.json') - scans={r['source']:r for r in read(observations)};totals=Counter();groups=defaultdict(Counter);examples=[] - for row in truth['screenshots']: - image=cv2.imread(str(dataset/row['file']));items=[];targets=[];actual=[] - for card in row['cards']: - mode=card.get('display_mode','level') - if mode=='level':target=card.get('level') - elif mode=='training':target=100+card['awake_count'] if card.get('awake_count') is not None else None - else:target=0 if card.get('field_visible') else None - if target is None:continue - x,y,w,h=card['bbox'];member=card['kind']=='member';sx=w/(224 if member else 326);sy=h/(294 if member else 184) - items.append({'kind':card['kind'],'bbox':[x+6*sx,y+6*sy,w-12*sx,h-12*sy]});targets.append((card,target)) - candidates=[c for c in scans[row['file']]['cards'] if c['kind']==card['kind'] and c['id']==card['id'] and bbox_iou(c['bbox'],card['bbox'])>=.5] - actual.append(max(candidates,key=lambda c:bbox_iou(c['bbox'],card['bbox'])) if candidates else None) - oracle=reader.read_fields(image,items) - predicted=iter(reader.read_fields(image,[c for c in actual if c is not None])) - for (card,target),ideal,found in zip(targets,oracle,actual): - value=encoded(next(predicted)) if found else None;oracle_value=encoded(ideal) - keys=['overall','kind:'+card['kind'],'locale:'+row.get('locale','zh-Hans'), - 'card_width_bin:'+str(int(card['bbox'][2]//30)*30)] - for key in keys: - counts=totals if key=='overall' else groups[key] - count_result(counts,'oracle',target,oracle_value) - if found: - count_result(counts,'detected_pose',target,value) - counts['oracle_correct_but_pose_lost']+=int(target>0 and oracle_value==target and value!=target) - else: - counts['localization_missing']+=1 - counts['oracle_correct_on_missing_card']+=int(target>0 and oracle_value==target) - if target>0 and oracle_value==target and value!=target: - examples.append({'file':row['file'],'kind':card['kind'],'id':card['id'],'truth':target, - 'oracle':oracle_value,'detected_pose':value,'bbox':found['bbox'] if found else None}) - report={'schema':'ournotes-boxlens.field-diagnosis/1','threshold':FIELD_CONFIDENCE,'counts':dict(totals), - 'strata':{k:dict(v) for k,v in groups.items()},'localization_loss_examples':examples, - 'scope':'Development screenshots only. Oracle artwork rectangle derived from native tile truth; detected poses come from saved matching identity+IoU results. Conditional pose counts exclude cards not localized.'} - write(output,report);print(report['counts']) - - -if __name__=='__main__': - p=argparse.ArgumentParser() - for name in ['data','dataset','observations','output']:p.add_argument('--'+name,type=Path,required=True) - a=p.parse_args();diagnose(a.data,a.dataset,a.observations,a.output) diff --git a/bdon_vision/engine.py b/bdon_vision/engine.py index 8414a5a..21f8b68 100644 --- a/bdon_vision/engine.py +++ b/bdon_vision/engine.py @@ -1,15 +1,40 @@ +"""Screenshot recognition: card-frame locator, per-kind identity and field readers. + +One scan runs three stages: + +1. ``locator.onnx`` finds member and Snap card tiles (and their kind) anywhere + in the screenshot, including cards that are not in the local gallery. +2. The per-kind artwork encoder embeds each tile's artwork window and compares + it with the gallery of the same kind. A tile is identified only when the + best cosine similarity and its gap to the runner-up both reach the model's + thresholds; otherwise it is reported as unidentified. +3. Per-kind field and card-rank classifiers read the parameter field (level or + member training count) and the card-rank icon of every located tile. + +Every model file, its SHA-256 and its acceptance thresholds come from +``models/recognition.json``. The gallery is computed from the user's local +catalog artwork and cached in ``models/-gallery.npz``; adding a card only +adds a gallery vector. +""" import hashlib -import math -import time -import json import importlib.metadata +import json +import os +import time from collections import defaultdict from pathlib import Path import cv2 import numpy as np +from . import kind_fields from .assets import read +from .kind_geometry import KINDS, query_crop, to_chw +from .locator import Locator, visible_fraction + +MODELS_SCHEMA = 'ournotes-boxlens.recognition-models/2' +PROVENANCE_SCHEMA = 'ournotes-boxlens.recognition-provenance/2' +GALLERY_SCHEMA = 'ournotes-boxlens.gallery-cache/2' def field(value=None, confidence=0., reason=None): @@ -19,252 +44,194 @@ def field(value=None, confidence=0., reason=None): return out -def overlap(a, b): - x = max(0., min(a[0]+a[2], b[0]+b[2])-max(a[0], b[0])) - y = max(0., min(a[1]+a[3], b[1]+b[3])-max(a[1], b[1])) - return x*y / max(1., min(a[2]*a[3], b[2]*b[3])) +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def load_models(data): + """Read ``models/recognition.json`` and check every referenced file's hash.""" + folder = Path(data) / 'models' + path = folder / 'recognition.json' + if not path.is_file(): + raise FileNotFoundError(f'Missing model manifest: {path}. Unpack the released weights into {folder}.') + manifest = read(path) + if manifest.get('schema') != MODELS_SCHEMA: + raise ValueError(f'Unsupported model manifest schema in {path}: {manifest.get("schema")!r}') + entries = [('locator', manifest['locator'])] + entries += [(f'{kind}-{role}', manifest[role][kind]) for role in ('encoders', 'fields', 'ranks') for kind in KINDS] + for name, entry in entries: + file = folder / entry['file'] + if not file.is_file(): + raise FileNotFoundError(f'Missing model file for {name}: {file}') + if sha256(file) != entry['sha256']: + raise ValueError(f'Model file hash mismatch for {name}: {file}') + return manifest + + +def session(path, threads): + import onnxruntime as ort + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + return ort.InferenceSession(str(path), sess_options=options, providers=['CPUExecutionProvider']) + + +def run(model, batch, chunk=64): + if not len(batch): + return np.zeros((0, model.get_outputs()[0].shape[-1]), np.float32) + return np.concatenate([model.run(None, {'image': np.ascontiguousarray(batch[i:i+chunk], np.float32)})[0] + for i in range(0, len(batch), chunk)]) + + +class Gallery: + """Reference embeddings of one kind, rebuilt when the model or artwork changes.""" + + def __init__(self, data, cards, kind, model, model_sha256, use_cache=True): + from .kind_encoder import reference_array, reference_source + data = Path(data) + self.kind = kind + self.indices = [i for i, c in enumerate(cards) if c['kind'] == kind] + sources = [reference_source(data, cards[i]) for i in self.indices] + fingerprint = hashlib.sha256(f'{GALLERY_SCHEMA}:{kind}:{model_sha256}'.encode()) + for i, source in zip(self.indices, sources): + fingerprint.update(f":{cards[i]['id']}:".encode()) + fingerprint.update(source.read_bytes()) + digest = fingerprint.hexdigest() + cache = data / f'models/{kind}-gallery.npz' + if use_cache and cache.exists(): + with np.load(cache, allow_pickle=False) as saved: + if str(saved['fingerprint']) == digest: + self.vectors = saved['vectors'].copy() + self.fingerprint = digest + return + references = np.stack([reference_array(kind, source) for source in sources]) if sources else np.zeros((0, 3, 1, 1), np.float32) + self.vectors = run(model, references) if sources else np.zeros((0, 128), np.float32) + self.fingerprint = digest + if use_cache: + temporary = cache.with_name(f'{cache.stem}.{os.getpid()}.tmp.npz') + np.savez_compressed(temporary, vectors=self.vectors, fingerprint=digest) + os.replace(temporary, cache) + + def retrieve(self, embeddings): + """Per query: (catalog index or None, similarity, gap to the runner-up).""" + if not len(self.indices): + return [(None, 0., 0.) for _ in embeddings] + scores = embeddings @ self.vectors.T + out = [] + for row in scores: + order = np.argsort(row) + best = float(row[order[-1]]) + second = float(row[order[-2]]) if len(order) > 1 else -1. + out.append((self.indices[int(order[-1])], best, best - second)) + return out class Engine: - def __init__(self, data, threads=2,excluded_identities=None): + def __init__(self, data, threads=2, excluded_identities=None): self.data = Path(data) - from .ocr import FIELD_CONFIDENCE - core=hashlib.sha256() - for name in ['engine.py','ocr.py','inference.py']: - core.update(name.encode());core.update((Path(__file__).parent/name).read_bytes()) - self.provenance={'schema':'ournotes-boxlens.recognition-provenance/1', - 'files':{name:hashlib.sha256((self.data/name).read_bytes()).hexdigest() - for name in ['catalog.json','index.npz','models/encoder.onnx','models/fields.onnx'] if (self.data/name).exists()}, - 'inference_core_sha256':core.hexdigest(),'field_confidence_threshold':FIELD_CONFIDENCE, - 'onnxruntime_version':importlib.metadata.version('onnxruntime')} + self.models = load_models(self.data) + folder = self.data / 'models' cv2.setNumThreads(threads) - cv2.setRNGSeed(12345) self.cards = read(self.data / 'catalog.json')['cards'] - index = np.load(self.data / 'index.npz') - self.points = index['points'] - self.owners = index['owners'] - descriptors=index['descriptors'].astype(np.float32) if excluded_identities: - allowed=np.array([(c['kind'],c['id']) not in excluded_identities for c in self.cards]) - mapping=np.cumsum(allowed)-1;selected=allowed[self.owners] - self.points=self.points[selected];self.owners=mapping[self.owners[selected]] - descriptors=descriptors[selected];self.cards=[c for c,keep in zip(self.cards,allowed) if keep] - self.matcher = cv2.FlannBasedMatcher(dict(algorithm=1, trees=4), dict(checks=64)) - self.matcher.add([descriptors]) - self.matcher.train() - self.sift = cv2.SIFT_create(nfeatures=8000, contrastThreshold=.018, edgeThreshold=12) - self.art = [cv2.imread(str(self.data / c.get('match_file',c['file']))) for c in self.cards] - self.ranks = {} - for kind, prefix in [('member', 'CardRank'), ('snap', 'snaplimit_')]: - self.ranks[kind] = [cv2.imread(str(self.data / f'native/game-ui/sprites/{prefix}{i}.png'), cv2.IMREAD_UNCHANGED) for i in range(6)] - from .ocr import NumberReader - self.numbers = NumberReader(self.data,threads) - self.gallery=None - if (self.data/'models/encoder.onnx').exists(): - from .inference import Gallery - self.gallery=Gallery(self.data,self.cards,threads,use_cache=not bool(excluded_identities)) + self.cards = [c for c in self.cards if (c['kind'], c['id']) not in excluded_identities] + locator = self.models['locator'] + self.locator = Locator(folder / locator['file'], long_edge=locator['long_edge'], threads=threads, + threshold=locator['threshold']) + self.encoders = {k: session(folder / self.models['encoders'][k]['file'], threads) for k in KINDS} + self.fields = {k: session(folder / self.models['fields'][k]['file'], threads) for k in KINDS} + self.ranks = {k: session(folder / self.models['ranks'][k]['file'], threads) for k in KINDS} + self.galleries = {k: Gallery(self.data, self.cards, k, self.encoders[k], self.models['encoders'][k]['sha256'], + use_cache=not excluded_identities) for k in KINDS} + core = hashlib.sha256() + for name in ['engine.py', 'locator.py', 'kind_geometry.py', 'kind_encoder.py', 'kind_fields.py']: + core.update(name.encode()) + core.update((Path(__file__).parent / name).read_bytes()) + self.provenance = { + 'schema': PROVENANCE_SCHEMA, + 'files': {name: sha256(self.data / name) for name in ['catalog.json', 'models/recognition.json']}, + 'galleries': {k: self.galleries[k].fingerprint for k in KINDS}, + 'inference_core_sha256': core.hexdigest(), + 'onnxruntime_version': importlib.metadata.version('onnxruntime')} - def fill_grid(self,image,items): - """Recover weak-feature cards only at grid positions supported by matches.""" - def clusters(values,tolerance): - groups=[] - for value in sorted(values): - if groups and abs(value-np.median(groups[-1]))1: - pitch=float(np.min(gaps)) - if 1.02*size.08:continue - xs=axis([c['bbox'][0] for c in seed],w);ys=axis([c['bbox'][1] for c in seed],h) - proposals=[];patches=[] - for y in ys: - for x in xs: - bbox=[x,y,float(w),float(h)] - if any(overlap(bbox,c['bbox'])>.4 for c in items):continue - if min(x,y)<0 or x+w>image.shape[1] or y+h>image.shape[0]:continue - patch=cv2.getRectSubPix(image,(round(w),round(h)),(x+w/2,y+h/2)) - proposals.append(bbox);patches.append(patch) - if not proposals:continue - candidates=[i for i,c in enumerate(self.cards) if c['kind']==kind] - mask=np.ones((48,48),bool);mask[37:]=False;mask[:9,:10]=False - art=np.stack([cv2.resize(self.art[i],(48,48))[mask].ravel() for i in candidates]).astype(float) - art-=art.mean(axis=1,keepdims=True);art/=np.maximum(np.linalg.norm(art,axis=1,keepdims=True),1e-8) - learned=self.gallery.retrieve(patches,kind) if self.gallery else [None]*len(patches) - for bbox,patch,neural in zip(proposals,patches,learned): - query=cv2.resize(patch,(48,48))[mask].ravel().astype(float) - query-=query.mean();query/=max(np.linalg.norm(query),1e-8) - scores=art@query;order=np.argsort(scores);i=candidates[int(order[-1])] - score=float(scores[order[-1]]);margin=float(scores[order[-1]]-scores[order[-2]]) - if neural: - ni,ns,ngap=neural - if ni!=i or ns<.72 or ngap<.06:continue - strong_neural=neural is not None and neural[1]>=.72 and neural[2]>=.25 - if score<(.60 if strong_neural else .78) or margin<.08:continue - card=self.cards[i] - items.append({'kind':kind,'id':card['id'],'name':card['name'],'rarity':card['rarity'], - 'card_type':card['card_type'],'bbox':[round(v,2) for v in bbox], - 'identity_confidence':round(score,4),'identity_method':'grid_gallery', - 'embedding_similarity':round(neural[1],4) if neural else None, - 'inliers':0,'visible_fraction':1.,'_index':i}) - return sorted(items,key=lambda c:(round(c['bbox'][1]/20),c['bbox'][0])) - - def identify(self, im): - h, w = im.shape[:2] - factor = min(1., 1500 / max(h, w)) - small = cv2.resize(im, None, fx=factor, fy=factor) if factor < 1 else im - gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY) - kp, desc = self.sift.detectAndCompute(gray, None) - if desc is None: - return [] - desc = np.sqrt(desc / (desc.sum(axis=1, keepdims=True) + 1e-8)) - grouped = defaultdict(list) - for matches in self.matcher.knnMatch(desc, k=2): - if len(matches) == 2 and matches[0].distance < .76 * matches[1].distance: - m = matches[0] - grouped[int(self.owners[m.trainIdx])].append((m.trainIdx, m.queryIdx, m.distance)) - found = [] - for i, matches in grouped.items(): - if len(matches) < 5: - continue - src = np.float32([self.points[t] for t, q, d in matches]) - dst = np.float32([kp[q].pt for t, q, d in matches]) / factor - transform, inliers = cv2.estimateAffine2D(src, dst, method=cv2.RANSAC, - ransacReprojThreshold=2.5/factor, maxIters=1500) - if transform is None or int(inliers.sum()) < 5: - continue - scale = math.hypot(transform[0, 0], transform[1, 0]) - scale_y = math.hypot(transform[0, 1], transform[1, 1]) - angle = math.degrees(math.atan2(transform[1, 0], transform[0, 0])) - c = self.cards[i] - if abs(angle) > 4 or abs(transform[0,1]) > .08*scale_y or not .55 < scale/scale_y < 1.8 or not 55 < c['width']*scale < w*.9: + def identify(self, image, detections): + """Gallery decision for each detection: (catalog index or None, similarity, gap, candidate index).""" + out = [None] * len(detections) + for kind in KINDS: + chosen = [i for i, d in enumerate(detections) if d['kind'] == kind] + if not chosen: continue - x, y = transform[:, 2] - bw, bh = c['width']*scale, c['height']*scale_y - if x+bw < 0 or y+bh < 0 or x > w or y > h: - continue - # Check the actual artwork, not only local-feature correspondence. - recovered = cv2.warpAffine(im, cv2.invertAffineTransform(transform), - (c['width'], c['height']), flags=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT) - a = cv2.resize(self.art[i], (48, 64 if c['kind']=='member' else 27)).astype(float) - b = cv2.resize(recovered, (a.shape[1], a.shape[0])).astype(float) - valid = (b.max(axis=2) > 0) - valid[int(len(valid)*.80):] = False - valid[:max(1,int(len(valid)*.1)), :8] = False - if valid.sum() < 150: - continue - aa, bb = a[valid].ravel(), b[valid].ravel() - similarity = float(np.corrcoef(aa, bb)[0, 1]) - if not np.isfinite(similarity) or similarity < .68: - continue - visible = max(0,min(w,x+bw)-max(0,x))*max(0,min(h,y+bh)-max(0,y))/(bw*bh) - found.append({'kind': c['kind'], 'id': c['id'], 'name': c['name'], - 'rarity': c['rarity'], 'card_type': c['card_type'], - 'bbox': [round(float(v),2) for v in (x,y,bw,bh)], - 'identity_confidence': round(similarity,4), 'inliers': int(inliers.sum()), - 'visible_fraction': round(float(visible),4), '_index': i}) - accepted=[] - for item in sorted(found, key=lambda x:(x['identity_confidence'],x['inliers']), reverse=True): - if not any(overlap(item['bbox'], other['bbox']) > .5 for other in accepted): - accepted.append(item) - return sorted(accepted, key=lambda x:(round(x['bbox'][1]/20),x['bbox'][0])) + crops = np.stack([to_chw(query_crop(image, kind, detections[i]['bbox'])) for i in chosen]) + accept = self.models['encoders'][kind]['accept'] + for i, (index, similarity, gap) in zip(chosen, self.galleries[kind].retrieve(run(self.encoders[kind], crops))): + identified = index is not None and similarity >= accept['similarity'] and gap >= accept['margin'] + out[i] = (index if identified else None, similarity, gap, index) + return out - def ui_bbox(self, item): - """Convert the matched full artwork box to its aspect-fill UI window. - - Member references are already canonical 212x282 crops. Snap references - retain the full artwork; their 314x172 UI window crops its center. - """ - x,y,w,h=item['bbox'] - if item['kind']!='snap':return [x,y,w,h] - ah,aw=self.art[item['_index']].shape[:2] - art_ratio=aw/ah;frame_ratio=314/172 - if art_ratio image.shape[0]-2: - return field(reason='cropped') - member=item['kind']=='member' - scale=w/(212 if member else 314) - cx=x+(188.5 if member else 300.1)*scale - cy=y+(261.2 if member else 147.8)*scale - # The icon extends beyond the artwork. remap's default black padding - # can resemble dark lobes and falsely identify a clipped right edge. - half_width=32*scale*1.03+.8 - half_height=32*scale*1.03*95/102+.8 - if (cx-half_width<2 or cy-half_height<2 or - cx+half_width>image.shape[1]-2 or cy+half_height>image.shape[0]-2): - return field(reason='cropped_rank_icon') - sprites=np.stack(self.ranks[item['kind']]).astype(np.float32) - alpha=sprites[:,:,:,3:4]/255. - mask=alpha.max(axis=0)[:,:,0]>.45 - different=(np.ptp(sprites,axis=0).max(axis=2)>35)&mask - # Rank 0 has translucent dark lobes. An opaque-only mask incorrectly - # ignores the colored lobes and systematically prefers rank 0. - weights=(mask.astype(np.float32)+different*3)[:,:,None] - art=self.art[item['_index']];ah,aw=art.shape[:2] - art_x,art_y,art_w,art_h=item['bbox'] - scores=[(-1.,i) for i in range(6)] - for ratio in [.97,1.,1.03]: - sw=64*scale*ratio;sh=sw*95/102 - for dx,dy in [(0,0),(-.8,0),(.8,0),(0,-.8),(0,.8)]: - gx=np.linspace(cx+dx-sw/2,cx+dx+sw/2,102,dtype=np.float32) - gy=np.linspace(cy+dy-sh/2,cy+dy+sh/2,95,dtype=np.float32) - mx,my=np.meshgrid(gx,gy) - observed=cv2.remap(image,mx,my,cv2.INTER_LINEAR).astype(np.float32) - bg=cv2.remap(art,(mx-art_x)*aw/art_w,(my-art_y)*ah/art_h,cv2.INTER_LINEAR, - borderMode=cv2.BORDER_REPLICATE).astype(np.float32) - expected=sprites[:,:,:,:3]*alpha+bg[None]*(1-alpha) - error=(np.abs(expected-observed[None])*weights).sum(axis=(1,2,3))/(weights.sum()*3*255) - for rank,e in enumerate(error): - scores[rank]=(max(scores[rank][0],1-float(e)),rank) - scores.sort(reverse=True) - if len(scores)<2: - return field(reason='unreadable') - best,rank=scores[0] - gap=best-scores[1][0] - if rank==0:return field(confidence=best,reason='no_owned_rank_icon') - if best < .78 or gap < .007: - return field(confidence=best,reason='ambiguous_icon') - return field(rank,best) + def read_fields(self, image, detections): + """Parameter-field and card-rank readings for each detection.""" + height, width = image.shape[:2] + out = [{'level': field(reason='cropped'), 'awake_count': field(reason='cropped'), 'display_mode': 'unknown', + 'card_rank': field(reason='cropped_rank_icon')} for _ in detections] + for kind in KINDS: + chosen = [i for i, d in enumerate(detections) if d['kind'] == kind] + readable = [i for i in chosen if kind_fields.field_inside(kind, detections[i]['bbox'], width, height)] + if readable: + crops = kind_fields.to_input([kind_fields.crop_field(image, kind, detections[i]['bbox']) for i in readable]) + accept = self.models['fields'][kind]['accept'] + for i, p in zip(readable, kind_fields.softmax(run(self.fields[kind], crops))): + decision = kind_fields.decide_field(kind, p, accept['confidence'], accept['margin']) + confidence, reason = decision['confidence'], decision.get('reason') + if decision['level'] is not None: + level, awake = field(decision['level'], confidence), field(reason='not_visible_in_level_view') + elif decision['awake_count'] is not None: + level, awake = field(reason='not_visible_in_training_view'), field(decision['awake_count'], confidence) + else: + level = awake = field(confidence=confidence, reason=reason) + out[i].update(level=level, awake_count=awake, display_mode=decision['display_mode']) + readable = [i for i in chosen if kind_fields.rank_icon_inside(kind, detections[i]['bbox'], width, height)] + if readable: + crops = kind_fields.to_input([kind_fields.crop_rank(image, kind, detections[i]['bbox']) for i in readable]) + accept = self.models['ranks'][kind]['accept'] + for i, p in zip(readable, kind_fields.softmax(run(self.ranks[kind], crops))): + decision = kind_fields.decide_rank(p, accept['confidence'], accept['margin']) + out[i]['card_rank'] = field(decision['value'], decision['confidence'], decision.get('reason')) + return out def scan(self, image, source='image'): - started=time.perf_counter() - items=self.fill_grid(image,self.identify(image)) - field_items=[{**item,'bbox':self.ui_bbox(item)} for item in items] - fields=self.numbers.read_fields(image,field_items) - for item,values in zip(items,fields): - item['card_rank']=self.read_rank(image,item) - item.update(values) - catalog=self.cards[item['_index']] - maximum=catalog.get('max_level') - rank=item['card_rank']['value'] - if rank is not None:maximum=catalog.get('rank_level_limits',{}).get(str(rank),maximum) - if maximum and item['level']['value'] is not None and item['level']['value']>maximum: - item['level']=field(reason='outside_masterdata_level_limit') - item['review']=any(item[k]['value'] is None for k in ('level','card_rank')) - item.pop('_index',None) - return {'source':source,'width':image.shape[1],'height':image.shape[0], - 'recognition_provenance':self.provenance, - 'source_id':hashlib.sha256(str(image.shape).encode()+image.tobytes()).hexdigest(), - 'cards':items,'elapsed_ms':round((time.perf_counter()-started)*1000,2), - 'coverage':'observed_only'} + started = time.perf_counter() + height, width = image.shape[:2] + detections = self.locator(np.ascontiguousarray(image[:, :, ::-1])) + identities = self.identify(image, detections) + readings = self.read_fields(image, detections) + cards, unidentified = [], [] + for detection, (index, similarity, gap, candidate), values in zip(detections, identities, readings): + item = {'kind': detection['kind'], 'bbox': [round(float(v), 2) for v in detection['bbox']], + 'locator_score': round(detection['score'], 4), + 'visible_fraction': round(visible_fraction(detection['bbox'], (width, height)), 4), + 'identity_similarity': round(similarity, 4), 'identity_margin': round(gap, 4), **values} + if index is None: + item['candidate'] = None if candidate is None else {'id': self.cards[candidate]['id'], 'name': self.cards[candidate]['name']} + item['review'] = True + unidentified.append(item) + continue + catalog = self.cards[index] + maximum = catalog.get('max_level') + rank = item['card_rank']['value'] + if rank is not None: + maximum = catalog.get('rank_level_limits', {}).get(str(rank), maximum) + if maximum and item['level']['value'] is not None and item['level']['value'] > maximum: + item['level'] = field(reason='outside_masterdata_level_limit') + cards.append({'kind': item['kind'], 'id': catalog['id'], 'name': catalog['name'], 'rarity': catalog['rarity'], + 'card_type': catalog['card_type'], **{k: v for k, v in item.items() if k != 'kind'}, + 'review': any(item[k]['value'] is None for k in ('level', 'card_rank'))}) + order = lambda c: (round(c['bbox'][1] / 20), c['bbox'][0]) + return {'source': source, 'width': width, 'height': height, + 'recognition_provenance': self.provenance, + 'source_id': hashlib.sha256(str(image.shape).encode() + image.tobytes()).hexdigest(), + 'cards': sorted(cards, key=order), 'unidentified': sorted(unidentified, key=order), + 'elapsed_ms': round((time.perf_counter() - started) * 1000, 2), + 'coverage': 'observed_only'} def merge(scans, player='local'): diff --git a/bdon_vision/evaluate_engine.py b/bdon_vision/evaluate_engine.py new file mode 100644 index 0000000..401ccc5 --- /dev/null +++ b/bdon_vision/evaluate_engine.py @@ -0,0 +1,243 @@ +"""End-to-end evaluation of ``Engine.scan`` on scene sets with tile truth. + +Every truth card (``cards`` and ``foreign_cards`` of each screenshot in +``truth.json``) is paired one-to-one with the scan output tile of the highest +IoU at or above 0.5; ``cards`` and ``unidentified`` outputs both take part. +Outputs that pair with no truth card and overlap no ignored tile count as +extra detections. + +Per paired card the report scores: + +* identity: a gallery card must come back with its own ID (correct), another + ID (wrong) or unidentified (unknown); a foreign card (artwork outside the + gallery) must come back unidentified, otherwise it is a false accept; +* level, member training count and card rank: a visible target is correct, + wrong or unknown; a value emitted for a hidden target is a misread unless it + equals the card state. A target is visible when the list shows it and its + crop lies fully inside the image (the field box, or the whole rank icon). + +The module needs only the scan output contract, so the same file scores any +engine version that implements ``Engine(data).scan(image)``. +""" +import argparse +import hashlib +import json +import os +import platform +from collections import Counter, defaultdict +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +import cv2 +import numpy as np + +from .assets import read, write + +LOGICAL = {'member': (224, 294), 'snap': (326, 184)} +RANK_CENTER = {'member': (194.5, 267.2), 'snap': (306.1, 153.8)} +FIELDS = ('level', 'awake_count', 'card_rank') +WORKER = {} + + +def iou(a, b): + ix = max(0., min(a[0]+a[2], b[0]+b[2])-max(a[0], b[0])) + iy = max(0., min(a[1]+a[3], b[1]+b[3])-max(a[1], b[1])) + inter = ix*iy + return inter/max(1e-9, a[2]*a[3]+b[2]*b[3]-inter) + + +def field_inside(kind, tile, width, height, border=2.): + x, y, w, h = tile + s = w/LOGICAL[kind][0] + left, top = x-6*s, y+h-56*s + return left >= border and top >= border and left+144*s <= width-border and top+56*s <= height-border + + +def rank_inside(kind, tile, width, height, border=2.): + x, y, w, _ = tile + s = w/LOGICAL[kind][0] + cx, cy = x+RANK_CENTER[kind][0]*s, y+RANK_CENTER[kind][1]*s + hw, hh = 32*s, 32*95/102*s + return cx-hw >= border and cy-hh >= border and cx+hw <= width-border and cy+hh <= height-border + + +def targets(card, width, height): + """{field: (visible value or None, card-state value or None)} for one truth card.""" + kind, mode = card['kind'], card.get('display_mode', 'level') + latent = card.get('latent_state') or {} + shown = card.get('field_visible', True) and field_inside(kind, card['bbox'], width, height) + out = {'level': (card.get('level') if shown and mode == 'level' else None, latent.get('level')), + 'card_rank': (card.get('card_rank') if rank_inside(kind, card['bbox'], width, height) else None, + 0 if mode == 'hide' else latent.get('card_rank'))} + if kind == 'member': + out['awake_count'] = (card.get('awake_count') if shown and mode == 'training' else None, latent.get('awake_count')) + return out + + +def pair(truth, outputs): + """Greedy one-to-one pairing by IoU >= 0.5; returns {truth index: output index}.""" + candidates = sorted(((iou(t['bbox'], o['bbox']), i, j) for i, t in enumerate(truth) for j, o in enumerate(outputs)), + reverse=True) + used_t, used_o, out = set(), set(), {} + for score, i, j in candidates: + if score < .5: + break + if i in used_t or j in used_o: + continue + used_t.add(i) + used_o.add(j) + out[i] = j + return out + + +def strata(row, card, foreign, width): + acquisition = row.get('acquisition', {}) + keys = ['overall', 'kind:'+card['kind'], 'profile:'+acquisition.get('profile', 'unknown'), + 'locale:'+row.get('locale', 'unknown'), 'mode:'+card.get('display_mode', 'level'), + 'source_resolution:'+'x'.join(map(str, acquisition.get('source_resolution', ['unknown']))), + 'card_width_bin:'+str(int(card['bbox'][2]//60)*60), 'foreign:'+str(foreign)] + if not foreign: + keys.append('identity:'+('heldout' if card['id'] % 7 == 0 else 'seen')) + if card.get('occluded_fraction', 0) > 0: + keys.append('occluded') + if card.get('visible_fraction', 1.) < 1: + keys.append('partly_outside') + return keys + + +def score_screen(row, scan): + height, width = scan['height'], scan['width'] + truth = [(c, False) for c in row.get('cards', [])] + [(c, True) for c in row.get('foreign_cards', [])] + ignored = row.get('ignored_cards', []) + row.get('ignored_foreign_cards', []) + outputs = scan['cards'] + scan.get('unidentified', []) + pairs = pair([c for c, _ in truth], outputs) + counts = defaultdict(Counter) + errors = [] + for i, (card, foreign) in enumerate(truth): + keys = strata(row, card, foreign, width) + found = outputs[pairs[i]] if i in pairs else None + result = Counter({'cards': 1, 'located': int(found is not None)}) + if found is not None: + result['kind_correct'] += int(found['kind'] == card['kind']) + predicted = found.get('id') + if foreign: + result['foreign_rejected' if predicted is None else 'foreign_false_accept'] += 1 + elif predicted is None: + result['identity_unknown'] += 1 + else: + correct = predicted == card['id'] and found['kind'] == card['kind'] + result['identity_correct' if correct else 'identity_wrong'] += 1 + if not correct: + errors.append({'file': row['file'], 'card': i, 'error': 'identity', 'expected': [card['kind'], card['id']], + 'actual': [found['kind'], predicted]}) + for name, (visible, latent) in targets(card, width, height).items(): + value = found[name]['value'] if name in found else None + if visible is not None: + result[name+'_visible'] += 1 + result[name+'_correct'] += int(value == visible) + result[name+'_unknown'] += int(value is None) + if value is not None and value != visible: + result[name+'_wrong'] += 1 + errors.append({'file': row['file'], 'card': i, 'error': name, 'expected': visible, 'actual': value}) + elif value is not None: + match = value == latent + result[name+'_hidden_emitted'] += 1 + result[name+('_hidden_latent_match' if match else '_hidden_misread')] += 1 + if not match: + errors.append({'file': row['file'], 'card': i, 'error': name+'_hidden', 'expected': None, + 'latent': latent, 'actual': value}) + elif not foreign: + for name, (visible, _) in targets(card, width, height).items(): + result[name+'_visible'] += int(visible is not None) + result[name+'_unknown'] += int(visible is not None) + for key in keys: + counts[key].update(result) + paired = set(pairs.values()) + extra = [o for j, o in enumerate(outputs) if j not in paired and not any(iou(o['bbox'], g['bbox']) >= .5 for g in ignored)] + screen = Counter({'screens': 1, 'negative_screens': int(bool(row.get('negative'))), + 'extra_identified': sum(o.get('id') is not None for o in extra), + 'extra_unidentified': sum(o.get('id') is None for o in extra)}) + for key in ['overall', 'profile:'+row.get('acquisition', {}).get('profile', 'unknown'), 'locale:'+row.get('locale', 'unknown')]: + counts[key].update(screen) + return counts, errors + + +def summarize(counter): + out = dict(counter) + known = counter['cards']-counter['foreign_rejected']-counter['foreign_false_accept'] + accepted = counter['identity_correct']+counter['identity_wrong'] + out['identity_recall'] = round(counter['identity_correct']/known, 6) if known else None + out['identity_precision'] = round(counter['identity_correct']/accepted, 6) if accepted else None + for name in FIELDS: + total = counter[name+'_visible'] + out[name+'_coverage'] = round(counter[name+'_correct']/total, 6) if total else None + out[name+'_misreads'] = counter[name+'_wrong']+counter[name+'_hidden_misread'] + return out + + +def _init(data, threads): + cv2.setNumThreads(threads) + from .engine import Engine + WORKER['engine'] = Engine(data, threads=threads) + + +def _scan(job): + path, row = job + image = cv2.imread(str(Path(path)/row['file']), cv2.IMREAD_COLOR) + if image is None: + raise ValueError(f"Cannot decode {Path(path)/row['file']}") + scan = WORKER['engine'].scan(image, row['file']) + counts, errors = score_screen(row, scan) + return {k: dict(v) for k, v in counts.items()}, errors, scan['elapsed_ms'], scan + + +def evaluate(data, sets, output, workers=8, threads=1, keep_scans=False): + report = {'schema': 'ournotes-boxlens.engine-evaluation/1', 'data': str(data), 'sets': {}, + 'platform': platform.platform(), 'workers': workers, 'threads_per_worker': threads, + 'matching': 'one-to-one tile pairing at IoU >= 0.5; cards and unidentified outputs both pair', + 'scope': __doc__.strip().split('\n\n')[0]} + with ProcessPoolExecutor(max_workers=workers, initializer=_init, initargs=(str(data), threads)) as pool: + report['provenance'] = None + for name, path in sets: + path = Path(path) + document = read(path/'truth.json') + if not document.get('complete', False): + raise ValueError(f'Incomplete dataset: {path}') + jobs = [(str(path), row) for row in document['screenshots']] + counts, errors, timings, scans = defaultdict(Counter), [], [], [] + for done, (screen, screen_errors, elapsed, scan) in enumerate(pool.map(_scan, jobs, chunksize=2)): + for key, value in screen.items(): + counts[key].update(value) + errors += screen_errors + timings.append(elapsed) + report['provenance'] = scan.get('recognition_provenance', report['provenance']) + if keep_scans: + scans.append(scan) + if (done+1) % 250 == 0: + print(json.dumps({'set': name, 'screens': done+1, 'total': len(jobs)}), flush=True) + report['sets'][name] = { + 'path': str(path), 'truth_sha256': hashlib.sha256((path/'truth.json').read_bytes()).hexdigest(), + 'screens': len(jobs), 'latency_ms': {'median': float(np.median(timings)), 'p95': float(np.percentile(timings, 95))}, + 'strata': {key: summarize(value) for key, value in sorted(counts.items())}, + 'errors': errors[:500], 'error_count': len(errors)} + if keep_scans: + write(Path(output).with_name(Path(output).stem+f'-{name}-scans.json'), scans) + overall = report['sets'][name]['strata'].get('overall', {}) + print(json.dumps({'set': name, **{k: overall.get(k) for k in ('cards', 'located', 'identity_recall', 'identity_precision', + 'foreign_false_accept', 'level_coverage', 'level_misreads', 'awake_count_coverage', + 'awake_count_misreads', 'card_rank_coverage', 'card_rank_misreads', 'extra_identified', + 'extra_unidentified')}}), flush=True) + write(output, report) + return report + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + p.add_argument('--data', type=Path, required=True) + p.add_argument('--set', action='append', required=True, metavar='NAME=DIR') + p.add_argument('--output', type=Path, required=True) + p.add_argument('--workers', type=int, default=min(8, os.cpu_count() or 1)) + p.add_argument('--threads', type=int, default=1) + p.add_argument('--keep-scans', action='store_true') + a = p.parse_args() + evaluate(a.data, [s.split('=', 1) for s in a.set], a.output, a.workers, a.threads, a.keep_scans) diff --git a/bdon_vision/evaluate_kind_fields.py b/bdon_vision/evaluate_kind_fields.py index 84154bc..b2d6248 100644 --- a/bdon_vision/evaluate_kind_fields.py +++ b/bdon_vision/evaluate_kind_fields.py @@ -1,10 +1,8 @@ """Development evaluation of the per-kind field and rank readers. -Compares, on identical crops and identical acceptance rules: - -* new per-kind field readers against the shared field model; -* new per-kind rank classifiers against native-sprite template matching - (``Engine.read_rank``) given the same truth-derived card pose. +Compares the per-kind field readers with a shared baseline field model on +identical crops and identical acceptance rules, and reports the per-kind rank +classifiers. Scenes are read directly from ``truth.json`` and images; crops use the deployed geometry in ``kind_fields``. The card pose is the truth tile, a @@ -25,7 +23,7 @@ import numpy as np from .assets import read, write -from .kind_fields import (FIELD_CONFIDENCE, KINDS, RANK_CONFIDENCE, LOGICAL, MARGIN, RANK_INPUT, RANK_SIDE, crop_field, crop_rank, decide_field, +from .kind_fields import (FIELD_CONFIDENCE, KINDS, RANK_CONFIDENCE, MARGIN, RANK_INPUT, RANK_SIDE, crop_field, crop_rank, decide_field, decide_rank, field_inside, field_label, latent_field, latent_rank, load_bank, misread_sweep, rank_icon_inside, rank_label, scene_cards, softmax, to_input) from .kind_pose import detections_for, paired_tiles @@ -84,50 +82,14 @@ def finish(counter): return out -# ---------------------------------------------------------------- engine rank - -ENGINE = {} - +WORKER = {} -def light_engine(data): - """Engine instance carrying only what ``read_rank`` needs.""" - from .engine import Engine - engine = Engine.__new__(Engine) - engine.data = Path(data) - engine.cards = read(engine.data/'catalog.json')['cards'] - engine.art = [cv2.imread(str(engine.data/c.get('match_file', c['file']))) for c in engine.cards] - engine.ranks = {} - for kind, prefix in [('member', 'CardRank'), ('snap', 'snaplimit_')]: - engine.ranks[kind] = [cv2.imread(str(engine.data/f'native/game-ui/sprites/{prefix}{i}.png'), cv2.IMREAD_UNCHANGED) - for i in range(6)] - engine.lookup = {(c['kind'], c['id']): i for i, c in enumerate(engine.cards)} - return engine - -def art_item(engine, kind, card_id, tile): - """Matched-artwork item for ``Engine.read_rank`` from a tile rectangle.""" - index = engine.lookup.get((kind, card_id)) - if index is None: - return None - x, y, w, h = tile - s = w/LOGICAL[kind][0] - if kind == 'member': - return {'kind': kind, '_index': index, 'bbox': [x+6*s, y+6*s, 212*s, 282*s]} - wx, wy, ww, wh = x+6*s, y+6*s, 314*s, 172*s - ah, aw = engine.art[index].shape[:2] - if aw/ah < 314/172: - full_h = ww*ah/aw - return {'kind': kind, '_index': index, 'bbox': [wx, wy-(full_h-wh)/2, ww, full_h]} - full_w = wh*aw/ah - return {'kind': kind, '_index': index, 'bbox': [wx-(full_w-ww)/2, wy, full_w, wh]} - - -def _init_worker(data, jitter=0., tile_key='bbox', detections=None): +def _init_worker(jitter=0., tile_key='bbox', detections=None): cv2.setNumThreads(1) - ENGINE['engine'] = light_engine(data) if data else None - ENGINE['jitter'] = jitter - ENGINE['tile_key'] = tile_key - ENGINE['detections'] = detections + WORKER['jitter'] = jitter + WORKER['tile_key'] = tile_key + WORKER['detections'] = detections def perturb(tile, jitter, key): @@ -146,15 +108,14 @@ def _scene(job): if image is None: raise ValueError(f"Cannot decode {Path(path)/row['file']}") ih, iw = image.shape[:2] - engine = ENGINE.get('engine') - tile_key = ENGINE.get('tile_key', 'bbox') - detections = ENGINE.get('detections') + tile_key = WORKER.get('tile_key', 'bbox') + detections = WORKER.get('detections') located = paired_tiles(row, detections[dataset][row['file']], tile_key)[0] if detections is not None else None acquisition = row.get('acquisition', {}) rows = [] for index, (card, foreign) in enumerate(scene_cards(row)): kind = card['kind'] - reference = perturb([float(v) for v in card[tile_key]], ENGINE.get('jitter', 0.), f"{dataset}/{row['file']}/{index}") + reference = perturb([float(v) for v in card[tile_key]], WORKER.get('jitter', 0.), f"{dataset}/{row['file']}/{index}") tile = reference if located is None else located[index] f_label = field_label(kind, card) f_inside = field_inside(kind, reference, iw, ih) @@ -170,14 +131,7 @@ def _scene(job): 'field_occluded': not card.get('field_visible', True), 'rank_occluded': card.get('rank_visible') is False, 'rank_label': rank_label(card), 'rank_inside': r_inside, 'field_crop': crop_field(image, kind, tile) if f_read else None, - 'rank_crop': crop_rank(image, kind, tile) if r_read else None, 'engine_rank': None} - if engine is not None and r_inside and not foreign: - if not r_read: - item['engine_rank'] = {'value': None, 'reason': 'not_located'} - else: - art = art_item(engine, kind, card.get('id'), tile) - if art is not None: - item['engine_rank'] = engine.read_rank(image, art) + 'rank_crop': crop_rank(image, kind, tile) if r_read else None} rows.append(item) return rows @@ -203,10 +157,8 @@ def scatter(probabilities, index, total): return out -def evaluate_scenes(datasets, models, baseline, data, output, workers=8, engine_cache=None, jitter=0., - tile_key='bbox', detections=None): - """``engine_cache``: JSON file reusing template-matching results for the same truth fingerprint and - pose. ``jitter``: deterministic relative pose error applied to every truth tile (0 = exact). +def evaluate_scenes(datasets, models, baseline, output, workers=8, jitter=0., tile_key='bbox', detections=None): + """``jitter``: deterministic relative pose error applied to every truth tile (0 = exact). ``tile_key``: truth tile field of each card. ``detections``: stored detections (``kind_pose detect``) whose paired tiles replace the truth tiles; the evaluated population stays defined by the truth tiles.""" jobs = [] @@ -225,28 +177,12 @@ def evaluate_scenes(datasets, models, baseline, data, output, workers=8, engine_ 'detections': {'path': str(detections), 'sha256': hashlib.sha256(Path(detections).read_bytes()).hexdigest(), 'locator': stored['locator']} if stored else None} items = [] - cached = None - if engine_cache and Path(engine_cache).exists(): - cached = read(engine_cache) - same_pose = (cached.get('jitter', 0.) == jitter and cached.get('tile_key', 'bbox') == tile_key - and cached.get('detections') == (pose['detections'] or {}).get('sha256')) - if cached['truth_fingerprint'] != fingerprint.hexdigest() or cached['data'] != str(data) or not same_pose: - raise ValueError('Engine cache belongs to other scenes, data or pose') - run_engine = str(data) if data and cached is None else None with ProcessPoolExecutor(max_workers=workers, initializer=_init_worker, - initargs=(run_engine, jitter, tile_key, located)) as pool: + initargs=(jitter, tile_key, located)) as pool: for done, rows in enumerate(pool.map(_scene, jobs, chunksize=4)): items += rows if (done+1) % 500 == 0: print(json.dumps({'scenes': done+1, 'total': len(jobs)}), flush=True) - key = lambda item: f"{item['dataset']}/{item['file']}/{item['card']}" - if cached is not None: - for item in items: - item['engine_rank'] = cached['results'].get(key(item)) - elif engine_cache and data: - write(engine_cache, {'truth_fingerprint': fingerprint.hexdigest(), 'data': str(data), 'jitter': jitter, - 'tile_key': tile_key, 'detections': (pose['detections'] or {}).get('sha256'), - 'results': {key(i): i['engine_rank'] for i in items if i['engine_rank'] is not None}}) sessions = {name: session(path) for name, path in models.items()} thresholds = acceptance(models) base = session(baseline) if baseline else None @@ -255,7 +191,7 @@ def evaluate_scenes(datasets, models, baseline, data, output, workers=8, engine_ 'models': {k: {'path': str(v), 'sha256': hashlib.sha256(Path(v).read_bytes()).hexdigest(), 'confidence_threshold': thresholds[k]} for k, v in models.items()}, 'baseline_fields': {'path': str(baseline), 'sha256': hashlib.sha256(Path(baseline).read_bytes()).hexdigest()} if baseline else None, - 'engine_rank_data': str(data) if data else None, 'pose_jitter': jitter, 'pose': pose, 'kinds': {}} + 'pose_jitter': jitter, 'pose': pose, 'kinds': {}} for kind in KINDS: fields = [i for i in items if i['kind'] == kind and i['field_label'] is not None and i['field_inside']] ranks = [i for i in items if i['kind'] == kind and i['rank_inside']] @@ -282,15 +218,10 @@ def evaluate_scenes(datasets, models, baseline, data, output, workers=8, engine_ for n, item in enumerate(ranks): truth = item['rank_label'] new_value = (decide_rank(rank_p[n], thresholds[f'{kind}-rank'])['value'] or 0) if rank_p is not None else None - engine_value = (item['engine_rank']['value'] or 0) if item['engine_rank'] is not None else None for key in strata_keys(item): latent = item['rank_latent'] if new_value is not None: tally(strata[key], truth, new_value, 'rank_new_', latent) - if engine_value is not None: - tally(strata[key], truth, engine_value, 'rank_engine_', latent) - if new_value is not None: - tally(strata[key], truth, new_value, 'rank_new_on_engine_cards_', latent) if new_value is not None and new_value > 0 and new_value != truth and len(examples) < 50: examples.append({k: item[k] for k in ('dataset', 'file', 'card', 'id', 'mode', 'width', 'rank_label')} | {'predicted': new_value}) limit = 105 if kind == 'member' else 100 @@ -377,107 +308,6 @@ def parse_models(values): return models -def tile_from_window(kind, window): - """Card tile rectangle from the artwork window (member 212x282, Snap 314x172 logical).""" - x, y, w, h = window - s = w/(212 if kind == 'member' else 314) - return [x-6*s, y-6*s, LOGICAL[kind][0]*s, LOGICAL[kind][1]*s] - - -def iou(a, b): - ix = max(0., min(a[0]+a[2], b[0]+b[2])-max(a[0], b[0])) - iy = max(0., min(a[1]+a[3], b[1]+b[3])-max(a[1], b[1])) - inter = ix*iy - return inter/max(1e-9, a[2]*a[3]+b[2]*b[3]-inter) - - -def score(counter, truth, value, name): - if truth is None: - counter[name+'_hidden_predictions'] += int(value is not None) - return - counter[name+'_total'] += 1 - counter[name+'_correct'] += int(value == truth) - counter[name+'_wrong'] += int(value is not None and value != truth) - counter[name+'_unknown'] += int(value is None) - - -def evaluate_real(truth_files, data, models, output, truth_box='window'): - """Real screenshots: production localization, baseline readers and new readers per card. - - Cards are localized by the existing matcher; the new readers use the tile - implied by the matched artwork window, the baseline readers use the - production code path. ``truth_box`` only serves to pair duplicates. - """ - from .engine import Engine - engine = Engine(data, threads=4) - sessions = {name: session(path, 4) for name, path in models.items()} - thresholds = acceptance(models) - images_out, totals = [], Counter() - for truth_file in map(Path, truth_files): - document = read(truth_file) - for row in document['screenshots']: - image = cv2.imread(str(truth_file.parent/row['file'])) - if image is None: - raise ValueError(f"Cannot decode {truth_file.parent/row['file']}") - ih, iw = image.shape[:2] - items = engine.fill_grid(image, engine.identify(image)) - windows = [engine.ui_bbox(item) for item in items] - baseline_fields = engine.numbers.read_fields(image, [{'kind': i['kind'], 'bbox': w} for i, w in zip(items, windows)]) - detected = [] - for item, window, base in zip(items, windows, baseline_fields): - kind = item['kind'] - tile = tile_from_window(kind, window) - new = {'level': None, 'awake_count': None, 'card_rank': None} - if f'{kind}-fields' in sessions and field_inside(kind, tile, iw, ih): - probabilities = run(sessions[f'{kind}-fields'], crop_field(image, kind, tile)[None])[0] - decision = decide_field(kind, probabilities, thresholds[f'{kind}-fields']) - new['level'], new['awake_count'] = decision['level'], decision['awake_count'] - new['field_confidence'] = decision['confidence'] - if f'{kind}-rank' in sessions and rank_icon_inside(kind, tile, iw, ih): - decision = decide_rank(run(sessions[f'{kind}-rank'], crop_rank(image, kind, tile)[None])[0], - thresholds[f'{kind}-rank']) - new['card_rank'], new['rank_confidence'] = decision['value'], decision['confidence'] - detected.append({'kind': kind, 'id': item['id'], 'window': [round(v, 2) for v in window], 'tile': tile, - 'baseline': {'level': base['level']['value'], 'awake_count': base['awake_count']['value'], - 'card_rank': engine.read_rank(image, item)['value']}, - 'new': new}) - used, cards, counter = set(), [], Counter() - for card in row['cards']: - candidates = [(n, d) for n, d in enumerate(detected) if n not in used - and (d['kind'], d['id']) == (card['kind'], card['id'])] - reference = card['bbox'] if truth_box == 'window' else None - found = max(candidates, key=lambda c: iou(c[1]['window'], reference) if reference else 0, default=None) - record = {'kind': card['kind'], 'id': card['id'], - 'truth': {k: card.get(k) for k in ('level', 'awake_count', 'card_rank', 'field_visible')}, - 'localized': found is not None} - counter['cards'] += 1 - counter['localized'] += int(found is not None) - if found: - used.add(found[0]) - d = found[1] - record['window_iou_with_truth_box'] = round(iou(d['window'], card['bbox']), 3) - record['baseline'], record['new'] = d['baseline'], {k: d['new'][k] for k in ('level', 'awake_count', 'card_rank')} - for method in ('baseline', 'new'): - for name in ('level', 'awake_count', 'card_rank'): - if card['kind'] == 'snap' and name == 'awake_count': - continue - score(counter, card.get(name), record[method][name], f'{method}_{name}') - cards.append(record) - counter['extra_detections'] += len(detected)-len(used) - images_out.append({'truth': str(truth_file), 'file': row['file'], 'kind': row.get('kind'), - 'display_mode': row.get('display_mode'), 'counts': dict(counter), 'cards': cards}) - totals.update(counter) - print(json.dumps({'file': row['file'], **dict(counter)}), flush=True) - report = {'schema': 'ournotes-boxlens.kind-field-real/1', 'data': str(data), - 'models': {k: {'path': str(v), 'sha256': hashlib.sha256(Path(v).read_bytes()).hexdigest(), - 'confidence_threshold': thresholds[k]} for k, v in models.items()}, - 'totals': dict(totals), 'images': images_out, - 'scope': 'Real screenshots with manual truth; localization by the existing matcher for every method; ' - 'masterdata level-limit filtering is not applied to either method.'} - write(output, report) - return report - - if __name__ == '__main__': p = argparse.ArgumentParser(description=__doc__) sub = p.add_subparsers(dest='action', required=True) @@ -485,10 +315,8 @@ def evaluate_real(truth_files, data, models, output, truth_box='window'): s.add_argument('--datasets', nargs='+', type=Path, required=True) s.add_argument('--model', action='append', help='NAME=PATH, NAME in member-fields|snap-fields|member-rank|snap-rank') s.add_argument('--baseline', type=Path, help='Shared field model ONNX for comparison') - s.add_argument('--data', type=Path, help='Data directory for template-matching rank comparison') s.add_argument('--output', type=Path, required=True) s.add_argument('--workers', type=int, default=8) - s.add_argument('--engine-cache', type=Path, help='Reuse/store template-matching results for these scenes') s.add_argument('--pose-jitter', type=float, default=0., help='Relative pose error applied to truth tiles') s.add_argument('--tile-key', default='bbox', help='Truth tile field of each card (bbox or bbox_layout)') s.add_argument('--detections', type=Path, help='Stored detections whose paired tiles replace the truth tiles') @@ -498,16 +326,9 @@ def evaluate_real(truth_files, data, models, output, truth_box='window'): b.add_argument('--model', action='append') b.add_argument('--baseline', type=Path) b.add_argument('--output', type=Path, required=True) - r = sub.add_parser('real') - r.add_argument('--truth', nargs='+', type=Path, required=True) - r.add_argument('--data', type=Path, required=True) - r.add_argument('--model', action='append') - r.add_argument('--output', type=Path, required=True) a = p.parse_args() if a.action == 'scenes': - evaluate_scenes(a.datasets, parse_models(a.model), a.baseline, a.data, a.output, a.workers, a.engine_cache, a.pose_jitter, + evaluate_scenes(a.datasets, parse_models(a.model), a.baseline, a.output, a.workers, a.pose_jitter, a.tile_key, a.detections) - elif a.action == 'bank': - evaluate_bank(a.bank, a.kind, parse_models(a.model), a.baseline, a.output) else: - evaluate_real(a.truth, a.data, parse_models(a.model), a.output) + evaluate_bank(a.bank, a.kind, parse_models(a.model), a.baseline, a.output) diff --git a/bdon_vision/evaluate_locator.py b/bdon_vision/evaluate_locator.py index 425599e..5d43897 100644 --- a/bdon_vision/evaluate_locator.py +++ b/bdon_vision/evaluate_locator.py @@ -1,4 +1,4 @@ -"""Evaluate card-frame locators and the RootSIFT baseline on scene and real sets. +"""Evaluate card-frame locators on scene and real sets. Writes one JSON report with recall / precision at IoU 0.5 and 0.75, card kind accuracy, strata by card width and resolution, and per-image results for @@ -8,7 +8,6 @@ import argparse import json import time -from multiprocessing import Pool from pathlib import Path import cv2 @@ -148,53 +147,6 @@ def real_row(scene, image, detections, overlay=None, tag=''): 'false_positive_boxes': false_positives, 'cards': cards} -_ENGINE = None - - -def _baseline_worker(args): - data, path, grid, excluded = args - global _ENGINE - if _ENGINE is None: - from .engine import Engine - _ENGINE = Engine(data, threads=1, excluded_identities=excluded or None) - image = cv2.imread(path, cv2.IMREAD_COLOR) - started = time.perf_counter() - items = _ENGINE.identify(image) - if grid: - items = _ENGINE.fill_grid(image, items) - elapsed = time.perf_counter() - started - out = [] - for item in items: - x, y, w, h = _ENGINE.ui_bbox(item) - s = w / (212 if item['kind'] == 'member' else 314) - out.append({'kind': item['kind'], 'score': float(item['identity_confidence']), 'id': item['id'], - 'bbox': [x - 6 * s, y - 6 * s, w + 12 * s, h + 12 * s]}) - return out, elapsed - - -def evaluate_baseline(data, sets, real=None, workers=8, grid=False, overlay=None, excluded=()): - excluded = {(kind, int(i)) for kind, i in excluded} - report = {'method': 'RootSIFT + FLANN gallery matching' + (' + grid fill' if grid else ''), - 'excluded_identities': sorted(excluded), - 'tile_box': 'matched artwork window expanded by 6 logical pixels per side', 'sets': {}} - with Pool(workers) as pool: - for name, scenes in sets: - results = pool.map(_baseline_worker, [(data, s['path'], grid, excluded) for s in scenes], chunksize=2) - acc = Accumulator() - for scene, (detections, _) in zip(scenes, results): - image = cv2.imread(scene['path'], cv2.IMREAD_UNCHANGED) - h, w = image.shape[:2] - acc.add(detections, scene, w, h, 640 / max(w, h)) - report['sets'][name] = {'strata': acc.summary(-1.), - 'median_s': float(np.median([e for _, e in results]))} - print(json.dumps({'baseline': name, 'overall': report['sets'][name]['strata'].get('overall')}), flush=True) - if real: - results = pool.map(_baseline_worker, [(data, s['path'], grid, excluded) for s in real], chunksize=1) - report['real'] = [real_row(scene, read_rgb(scene['path']), detections, overlay, 'baseline') - for scene, (detections, _) in zip(real, results)] - return report - - def main(): p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) p.add_argument('--model', help='Locator state dict (.pt, GPU) or exported .onnx (CPU)') @@ -205,11 +157,6 @@ def main(): p.add_argument('--sizes', type=int, nargs='+', default=[640]) p.add_argument('--threshold', type=float) p.add_argument('--aspect-tolerance', type=float, help='Drop tiles whose aspect differs from the class by more') - p.add_argument('--baseline-data', help='Data directory for the RootSIFT baseline') - p.add_argument('--baseline-grid', action='store_true') - p.add_argument('--baseline-exclude', nargs='*', default=[], metavar='KIND:ID', - help='Remove identities from the baseline gallery (simulates cards added after it was built)') - p.add_argument('--workers', type=int, default=8, help='Baseline worker processes') p.add_argument('--threads', type=int, default=4, help='onnxruntime CPU threads') p.add_argument('--overlay', help='Directory for annotated real screenshots') p.add_argument('--output', required=True) @@ -223,9 +170,6 @@ def main(): if a.model: predictor = OnnxPredictor(a.model, a.threads) if a.model.endswith('.onnx') else TorchPredictor(a.model, a.width) report['locator'] = evaluate_sets(predictor, sets, a.sizes, a.threshold, real, a.overlay, a.aspect_tolerance) - if a.baseline_data: - excluded = [item.split(':') for item in a.baseline_exclude] - report['baseline'] = evaluate_baseline(a.baseline_data, sets, real, a.workers, a.baseline_grid, a.overlay, excluded) write_json(a.output, report) diff --git a/bdon_vision/evaluate_open_set.py b/bdon_vision/evaluate_open_set.py deleted file mode 100644 index 2e6aae5..0000000 --- a/bdon_vision/evaluate_open_set.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Remove holdout cards from the gallery and measure false known-card claims.""" -import argparse -import json -from pathlib import Path -import cv2 -import numpy as np -from .assets import read,write -from .engine import Engine -from .synthetic import bbox_iou - - -def evaluate(data,dataset,output): - catalog=read(Path(data)/'catalog.json')['cards'] - excluded={(c['kind'],c['id']) for c in catalog if c['id']%7==0} - engine=Engine(data,2,excluded_identities=excluded) - truth=read(Path(dataset)/'truth.json');unknown_total=0;false_known=[];background_claims=[];known_total=known_correct=0 - for n,row in enumerate(truth['screenshots']): - image=cv2.imread(str(Path(dataset)/row['file']));found=engine.identify(image);found=engine.fill_grid(image,found) - used=set() - for card in row['cards']: - candidates=[(i,p) for i,p in enumerate(found) if bbox_iou(card['bbox'],p['bbox'])>=.5] - identity=(card['kind'],card['id']) - if identity in excluded:unknown_total+=1 - else:known_total+=1 - if not candidates:continue - i,p=max(candidates,key=lambda v:bbox_iou(card['bbox'],v[1]['bbox']));used.add(i) - actual=(p['kind'],p['id']) - if identity in excluded:false_known.append({'file':row['file'],'unknown':identity,'predicted':actual,'score':p['identity_confidence']}) - elif actual==identity:known_correct+=1 - for i,p in enumerate(found): - if i not in used:background_claims.append({'file':row['file'],'predicted':[p['kind'],p['id']]}) - if (n+1)%32==0:print(json.dumps({'evaluated':n+1,'unknown_total':unknown_total,'false_known':len(false_known)}),flush=True) - blank_results=[] - rng=np.random.default_rng(8901) - for value in [0,128,255]: - image=np.full((720,1280,3),value,np.uint8);blank_results.append(len(engine.identify(image))) - for _ in range(5): - image=rng.integers(0,256,(40,70,3),dtype=np.uint8);image=cv2.resize(image,(1280,720),interpolation=cv2.INTER_CUBIC) - blank_results.append(len(engine.identify(image))) - report={'excluded_ids':sorted(excluded),'unknown_visible_instances':unknown_total, - 'false_known_claims':len(false_known),'false_known_examples':false_known, - 'known_visible_instances':known_total,'known_correct':known_correct, - 'unmatched_predictions':background_claims,'blank_scene_predictions':blank_results, - 'scope':'same-artwork holdout removed from gallery; identity+IoU matching; no false-field claims inferred from this test'} - write(Path(output)/'open-set.json',report);print(json.dumps(report),flush=True) - - -if __name__=='__main__': - p=argparse.ArgumentParser();p.add_argument('--data',type=Path,required=True);p.add_argument('--dataset',type=Path,required=True);p.add_argument('--output',type=Path,required=True) - a=p.parse_args();evaluate(a.data,a.dataset,a.output) diff --git a/bdon_vision/fielddata.py b/bdon_vision/fielddata.py deleted file mode 100644 index c096c40..0000000 --- a/bdon_vision/fielddata.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Native text-layer composites for multilingual level/cultivation/mode training.""" -import argparse -import io -import random -import hashlib -from collections import Counter -from functools import lru_cache -from pathlib import Path - -import numpy as np -from numpy.lib.format import open_memmap -from PIL import Image,ImageFilter,ImageEnhance - -from .assets import read,write,native_module - -LOCALES={'ja':'_japanese','en':'_english','zh-Hant':'_traditionalChinese','zh-Hans':'_simplifiedChinese','ko':'_korean'} - - -def generate(data,output,count=106000,seed=81101,split='train',acquisition='legacy'): - data,output=Path(data),Path(output);output.mkdir(parents=True,exist_ok=True) - if not (data/'native/card-fonts/manifest.json').exists():raise ValueError('Actual localized font assets required') - native=native_module(data);renderers={loc:native.CardRenderer(col) for loc,col in LOCALES.items()} - catalog=read(data/'catalog.json')['cards'];rng=random.Random(seed) - names=list(LOCALES);pools={kind:[c for c in catalog if c['kind']==kind] for kind in ('member','snap')} - backgrounds={} - for card in catalog: - kind=card['kind'];height=294 if kind=='member' else 184 - tile=renderers['zh-Hans'].render(native.CardState(asset_id=card['asset_id'],rarity=card['rarity'], - card_type=card['card_type'],param='hide',rank=None),kind,scale=1) - backgrounds[(kind,card['id'])]=tile.crop((26,32+height-56,170,32+height)) - - @lru_cache(maxsize=4096) - def layer(locale,kind,mode,value): - height=294 if kind=='member' else 184 - power=(value,value+17,value+31) if kind=='member' else (value,value+7,value+11) - tile=renderers[locale].render(native.CardState(asset_id=1,rarity=2,card_type=1, - param=mode,level=value if mode=='level' else 1,awake_count=value if mode=='training' else 1, - power=power),kind,scale=1,only='text') - return tile.crop((26,32+height-56,170,32+height)) - - images=open_memmap(output/'fields.npy',mode='w+',dtype='uint8',shape=(count,56,144,3)) - labels=open_memmap(output/'field-labels.npy',mode='w+',dtype='int32',shape=(count,)) - locales=open_memmap(output/'locales.npy',mode='w+',dtype='uint8',shape=(count,)) - values=Counter();mode_counts=Counter();locale_counts=Counter() - for j in range(count): - locale=rng.choice(names);kind=rng.choice(['member','snap']) - if j%5==0: - modes=['hide','performance','technic','visual']+(['total'] if kind=='member' else []) - mode=rng.choice(modes);value=(rng.randrange(64)*293+137) if kind=='member' else (rng.randrange(64)*17+37);label=0 - elif j%5==1: - kind='member';mode='training';value=(j//5)%5+1;label=100+value - else:mode='level';value=rng.randint(1,100);label=value - card=rng.choice(pools[kind]);patch=Image.new('RGBA',(144,56),tuple(rng.randrange(100,230) for _ in range(3))+(255,)) - patch.alpha_composite(backgrounds[(kind,card['id'])]) - text=layer(locale,kind,mode,value) - # Registration errors from card localization, not arbitrary font swapping. - patch.alpha_composite(text,(rng.randint(-3,3),rng.randint(-3,3))) - if label==0 and j%11==0: - # Partial glyphs are not license to infer hidden values. - cut=rng.randrange(35,110);patch.paste(backgrounds[(kind,card['id'])].crop((cut,0,144,56)),(cut,0)) - if acquisition=='screen': - # Compression happens on the received low-resolution pixels BEFORE - # the recognizer enlarges the crop. Legacy preprocessing compressed - # an already enlarged patch, which understates hard screenshot loss. - ratios=[.26,.33,.41,.49,.61,.77,.93,1.09] if split=='train' else [.29,.37,.45,.55,.69,.85,1.01] - qualities=[34,46,58,73,87,95] if split=='train' else [38,50,64,79,91] - small=rng.choice(ratios) - patch=ImageEnhance.Brightness(patch.convert('RGB')).enhance(rng.uniform(.88,1.12)) - patch=patch.resize((round(144*small),round(56*small)),rng.choice([Image.Resampling.BILINEAR,Image.Resampling.BICUBIC,Image.Resampling.LANCZOS])) - patch=patch.filter(ImageFilter.GaussianBlur(rng.uniform(.05,.95) if split=='train' else rng.uniform(.12,1.10))) - width,height=patch.size;px=rng.randrange(16);py=rng.randrange(16) - # Random codec-block alignment approximates cropping from a larger - # screenshot instead of always aligning the text crop to block zero. - patch=Image.fromarray(np.pad(np.asarray(patch),((py,16-py),(px,16-px),(0,0)),mode='edge')) - for _ in range(rng.choice([1,2,3,4])): - buf=io.BytesIO();patch.save(buf,format=rng.choice(['JPEG','WEBP']),quality=rng.choice(qualities)) - patch=Image.open(io.BytesIO(buf.getvalue())).convert('RGB') - patch=patch.crop((px,py,px+width,py+height)).resize((144,56),Image.Resampling.BILINEAR) - else: - small=rng.choice([.40,.52,.65,.80,1.,1.18] if split=='train' else [.34,.46,.59,.73,.91,1.08]) - patch=patch.resize((round(144*small),round(56*small)),Image.Resampling.BILINEAR).resize((144,56),Image.Resampling.BILINEAR) - if rng.random()<.35:patch=patch.filter(ImageFilter.GaussianBlur(rng.uniform(.1,.5 if split=='train' else .75))) - patch=ImageEnhance.Brightness(patch.convert('RGB')).enhance(rng.uniform(.88,1.12)) - for _ in range(rng.choice([1,2] if split=='train' else [2,3])): - buf=io.BytesIO();patch.save(buf,format=rng.choice(['JPEG','WEBP']), - quality=rng.choice([48,62,76,89,97] if split=='train' else [40,56,70,83,93])) - patch=Image.open(io.BytesIO(buf.getvalue())).convert('RGB') - images[j]=np.asarray(patch);labels[j]=label;locales[j]=names.index(locale) - values[str(label)]+=1;mode_counts[mode]+=1;locale_counts[locale]+=1 - if (j+1)%5000==0:print({'generated':j+1,'total':count,'split':split},flush=True) - for a in (images,labels,locales):a.flush() - metadata={'schema':'bdon-native-field-corpus/1','field_count':count,'field_counts':dict(values), - 'modes':dict(mode_counts),'locales':dict(locale_counts),'locale_names':names,'split':split, - 'seed':seed,'acquisition':acquisition,'complete':True,'label_encoding':{'0':'no level/cultivation value','1..100':'level','101..105':'awake_count1..5'}, - 'font_manifest_sha256':hashlib.sha256((data/'native/card-fonts/manifest.json').read_bytes()).hexdigest(), - 'provenance':'native prefab text geometry; actual localized glyphs, material and master text; raster AA approximate'} - write(output/'manifest.json',metadata);print(metadata,flush=True) - - -if __name__=='__main__': - p=argparse.ArgumentParser();p.add_argument('--data',type=Path,required=True);p.add_argument('--output',type=Path,required=True) - p.add_argument('--count',type=int,default=106000);p.add_argument('--seed',type=int,default=81101) - p.add_argument('--split',choices=['train','validation'],default='train') - p.add_argument('--acquisition',choices=['legacy','screen'],default='legacy') - a=p.parse_args();generate(a.data,a.output,a.count,a.seed,a.split,a.acquisition) diff --git a/bdon_vision/inference.py b/bdon_vision/inference.py deleted file mode 100644 index f01beb4..0000000 --- a/bdon_vision/inference.py +++ /dev/null @@ -1,42 +0,0 @@ -"""CPU ONNX artwork embeddings with a versioned, incrementally rebuilt gallery.""" -import hashlib -from pathlib import Path -import cv2 -import numpy as np - - -class Gallery: - def __init__(self,data,cards,threads=2,use_cache=True): - import onnxruntime as ort - data=Path(data);model=data/'models/encoder.onnx' - options=ort.SessionOptions();options.intra_op_num_threads=threads;options.inter_op_num_threads=1 - self.session=ort.InferenceSession(str(model),sess_options=options,providers=['CPUExecutionProvider']) - self.cards=cards - fingerprint=hashlib.sha256(model.read_bytes()) - images=[] - for card in cards: - path=data/card.get('match_file',card['file']) - fingerprint.update(f"{card['kind']}:{card['id']}".encode()) - fingerprint.update(path.read_bytes()) - images.append(cv2.imread(str(path))) - digest=fingerprint.hexdigest();cache=data/'models/gallery.npz' - saved=np.load(cache,allow_pickle=False) if use_cache and cache.exists() else None - if saved is not None and str(saved['fingerprint'])==digest: - self.vectors=saved['vectors'].copy() - else: - self.vectors=np.concatenate([self.encode(images[i:i+32]) for i in range(0,len(images),32)]) - if use_cache:np.savez_compressed(cache,vectors=self.vectors,fingerprint=digest) - if saved is not None:saved.close() - - def encode(self,images): - xs=np.stack([cv2.resize(im,(128,160))[:,:,::-1].transpose(2,0,1) for im in images]).astype(np.float32)/255 - return self.session.run(None,{'image':xs})[0] - - def retrieve(self,images,kind): - ids=[i for i,c in enumerate(self.cards) if c['kind']==kind] - sims=self.encode(images)@self.vectors[ids].T - result=[] - for row in sims: - order=np.argsort(row) - result.append((ids[int(order[-1])],float(row[order[-1]]),float(row[order[-1]]-row[order[-2]]))) - return result diff --git a/bdon_vision/ocr.py b/bdon_vision/ocr.py deleted file mode 100644 index df8e1a9..0000000 --- a/bdon_vision/ocr.py +++ /dev/null @@ -1,102 +0,0 @@ -import cv2 -import numpy as np -from pathlib import Path - -from .assets import native_module -from .engine import field - -FIELD_CONFIDENCE=.995 - -class NumberReader: - def __init__(self,data,threads=2): - self.session=None - path=Path(data)/'models/fields.onnx' - self.multifield=path.exists() - if not self.multifield: - raise FileNotFoundError(f'Missing validated parameter-mode model: {path}. Level-only models cannot safely read status views.') - if path.exists(): - import onnxruntime as ort - options=ort.SessionOptions() - options.intra_op_num_threads=threads;options.inter_op_num_threads=1 - self.session=ort.InferenceSession(str(path),sess_options=options, - providers=['CPUExecutionProvider']) - self.templates=[] - return - native=native_module(data) - font=native.GameText() - self.templates=[] - for value in range(1,101): - tile=np.array(font.text('Lv.'+str(value),36)) - self.templates.append((value,cv2.cvtColor(tile,cv2.COLOR_RGBA2BGRA))) - - def read_many(self,image,items): - return [r['level'] for r in self.read_fields(image,items)] - - def read_fields(self,image,items): - def empty(reason): - return {'level':field(reason=reason),'awake_count':field(reason=reason),'display_mode':'unknown'} - if self.session is None: - return [{**empty('mode_model_missing'),'level':self.read(image,item)} for item in items] - patches=[];indices=[];results=[empty('cropped') for _ in items] - for i,item in enumerate(items): - x,y,w,h=item['bbox'];scale=w/(212 if item['kind']=='member' else 314) - left=x-12*scale;top=y+h-50*scale - right=left+144*scale;bottom=top+56*scale - if left<2 or top<2 or right>image.shape[1]-2 or bottom>image.shape[0]-2: - continue - matrix=np.array([[scale,0,left],[0,scale,top]],np.float32) - patch=cv2.warpAffine(image,matrix,(144,56),flags=cv2.INTER_LINEAR|cv2.WARP_INVERSE_MAP) - patches.append(patch[:,:,::-1].transpose(2,0,1).astype(np.float32)/255) - indices.append(i) - if patches: - logits=self.session.run(None,{'image':np.stack(patches)})[0] - probs=np.exp(logits-logits.max(axis=1,keepdims=True));probs/=probs.sum(axis=1,keepdims=True) - for i,p in zip(indices,probs): - order=np.argsort(p);value=int(order[-1]);confidence=float(p[value]) - if value==0: - certain=confidence>=FIELD_CONFIDENCE and confidence-float(p[order[-2]])>=.5 - results[i]=empty('other_or_hidden_parameter' if certain else 'ambiguous_parameter') - results[i]['display_mode']='other' if certain else 'unknown' - elif confidence100: - if items[i]['kind']=='member': - results[i]=empty('not_visible_in_training_view') - results[i]['awake_count']=field(value-100,confidence) - results[i]['display_mode']='training' - else:results[i]=empty('unsupported_parameter') - else: - results[i]=empty('not_visible_in_level_view') - results[i]['level']=field(value,confidence) - results[i]['display_mode']='level' - return results - - def read(self,image,item): - x,y,w,h=item['bbox'] - if y+h>image.shape[0]-2: - return field(reason='cropped') - patch=image[max(0,int(y+h*.78)):min(image.shape[0],int(y+h*1.07)), - max(0,int(x-w*.06)):min(image.shape[1],int(x+w*.52))] - if patch.size==0: - return field(reason='cropped') - scores=[] - target=(w/216 if item['kind']=='member' else w/318) - for value,full in self.templates: - best=-1 - for factor in [.90,1.,1.10]: - scale=target*factor - templ=cv2.resize(full,None,fx=scale,fy=scale,interpolation=cv2.INTER_AREA) - th,tw=templ.shape[:2] - if th>patch.shape[0] or tw>patch.shape[1] or min(th,tw)<3: - continue - mask=(templ[:,:,3]>180).astype(np.uint8)*255 - match=cv2.matchTemplate(patch,templ[:,:,:3],cv2.TM_CCORR_NORMED,mask=mask) - match=np.nan_to_num(match,nan=-1,posinf=-1,neginf=-1) - best=max(best,float(match.max())) - scores.append((best,value)) - scores.sort(reverse=True) - best,value=scores[0] - if best<.90 or best-scores[1][0]<.003: - return field(confidence=best,reason='unreadable_level') - return field(value,best) diff --git a/bdon_vision/server.py b/bdon_vision/server.py index 798ab4a..5d3aeec 100644 --- a/bdon_vision/server.py +++ b/bdon_vision/server.py @@ -78,7 +78,7 @@ def do_POST(self): if image is None or image.shape[0]*image.shape[1]>20_000_000: raise ValueError('无法读取图片，或分辨率超过 2000 万像素') scan=engine.scan(image,source);cache[digest]=scan;scans.append(scan) - self.send({'schema':'ournotes-boxlens.scan-result/1','box':merge(scans,player=player),'scans':scans}) + self.send({'schema':'ournotes-boxlens.scan-result/2','box':merge(scans,player=player),'scans':scans}) except (ValueError,KeyError,TypeError) as error: self.send({'error':str(error)},400) diff --git a/bdon_vision/train_fields.py b/bdon_vision/train_fields.py deleted file mode 100644 index 32a1204..0000000 --- a/bdon_vision/train_fields.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Train a compact CPU-exportable field reader from native multilingual crops.""" -import argparse -import json -import time -import hashlib -from pathlib import Path - -import numpy as np -import torch -from torch import nn -from torch.nn import functional as F - -from .assets import read,write -from .train import LevelNet,seed_all,export - - -def load(path): - meta=read(path/'manifest.json') - if not meta['complete']:raise ValueError(f'Incomplete corpus: {path}') - count=meta['field_count'] - images=np.load(path/'fields.npy',mmap_mode='r')[:count] - labels=np.load(path/'field-labels.npy',mmap_mode='r')[:count] - # Copy avoids read-only array aliasing; all samples remain resident on GPU. - x=torch.from_numpy(np.array(images)).permute(0,3,1,2).cuda().float().div_(255) - y=torch.from_numpy(np.array(labels)).long().cuda() - return x,y,meta - - -def validation_metrics(model,x,y,batch): - was_training=model.training;model.eval();loss=0.;parts=[] - with torch.inference_mode(): - for start in range(0,len(x),batch): - logits=model(x[start:start+batch]);loss+=float(F.cross_entropy(logits,y[start:start+batch],reduction='sum')) - parts.append(logits.softmax(1).cpu().numpy()) - p=np.concatenate(parts);truth=y.cpu().numpy();pred=p.argmax(1);ordered=np.sort(p,axis=1) - accepted=(p.max(1)>=.995)&(ordered[:,-1]-ordered[:,-2]>=.5)&(pred>0) - correct=int(((pred==truth)&accepted).sum());wrong=int(((pred!=truth)&(truth>0)&accepted).sum()) - false_visible=int(((truth==0)&accepted).sum()) - model.train(was_training) - return {'samples':len(truth),'cross_entropy':loss/len(truth),'raw_correct':int((pred==truth).sum()), - 'visible_truth':int((truth>0).sum()),'numeric_correct':correct,'numeric_wrong':wrong, - 'false_visible':false_visible,'accepted_numeric':int(accepted.sum()), - 'numeric_precision':correct/(correct+wrong+false_visible) if correct+wrong+false_visible else None, - 'numeric_coverage':correct/int((truth>0).sum()) if (truth>0).any() else None} - - -def train(train_dir,valid_dir,output,initial,steps=8000,batch=512,replay=None,eval_every=0,stress=None): - if not torch.cuda.is_available():raise RuntimeError('GPU required') - output.mkdir(parents=True,exist_ok=True);seed_all(90011);torch.set_num_threads(8) - xs,ys,tm=load(train_dir);vx,vy,vm=load(valid_dir) - if stress:sx,sy,sm=load(stress) - if replay:rx,ry,rm=load(replay) - model=LevelNet();model.head=nn.Linear(96*3*8,106) - if initial: - state=torch.load(initial,map_location='cpu',weights_only=True) - model.load_state_dict({k:v for k,v in state.items() if not k.startswith('head.')},strict=False) - with torch.no_grad(): - n=min(model.head.out_features,state['head.weight'].shape[0]) - model.head.weight[:n].copy_(state['head.weight'][:n]);model.head.bias[:n].copy_(state['head.bias'][:n]) - model=model.cuda();optimizer=torch.optim.AdamW(model.parameters(),lr=.0003,weight_decay=.0001) - started=time.perf_counter();model.train() - curve=[] - def checkpoint(step,train_loss=None): - row={'step':step,'train_loss':train_loss,'elapsed_s':time.perf_counter()-started, - 'validation':validation_metrics(model,vx,vy,batch)} - if stress:row['development_stress']=validation_metrics(model,sx,sy,batch) - curve.append(row);write(output/'learning-curve.json',{'checkpoints':curve, - 'initial_sha256':hashlib.sha256(initial.read_bytes()).hexdigest() if initial else None, - 'scope':'Development corpus only; additional fine-tuning with fresh AdamW state, not an uninterrupted optimizer resume'}) - torch.save({k:v.detach().cpu() for k,v in model.state_dict().items()},output/f'fields-step-{step:06d}.pt') - print(json.dumps({'validation_checkpoint':row}),flush=True) - if eval_every:checkpoint(0) - for step in range(steps): - ix=torch.randint(len(xs),(batch,),device='cuda') - x=xs[ix];target=ys[ix] - if replay: - count=batch//3;ri=torch.randint(len(rx),(count,),device='cuda') - x=torch.cat([x[:-count],rx[ri]]);target=torch.cat([target[:-count],ry[ri]]) - x=(x*torch.empty(batch,1,1,1,device='cuda').uniform_(.93,1.07)).clamp(0,1) - loss=F.cross_entropy(model(x),target) - optimizer.zero_grad(set_to_none=True);loss.backward();optimizer.step() - if step==0 or (step+1)%200==0: - status={'step':step+1,'steps':steps,'loss':float(loss.detach()), - 'elapsed_s':round(time.perf_counter()-started,1),'gpu_allocated_gib':round(torch.cuda.memory_allocated()/2**30,2)} - write(output/'fields-progress.json',status);print(json.dumps(status),flush=True) - if eval_every and ((step+1)%eval_every==0 or step+1==steps):checkpoint(step+1,float(loss.detach())) - model.eval();chunks=[] - with torch.no_grad(): - for start in range(0,len(vx),batch):chunks.append(model(vx[start:start+batch]).softmax(1).cpu().numpy()) - probabilities=np.concatenate(chunks);truth=vy.cpu().numpy();pred=probabilities.argmax(1) - confidence=probabilities.max(1);order=np.sort(probabilities,axis=1);margin=order[:,-1]-order[:,-2] - locales=np.load(valid_dir/'locales.npy')[:len(truth)] if (valid_dir/'locales.npy').exists() else np.zeros(len(truth),int) - reports={} - for threshold in [.90,.95,.98,.995]: - accepted=(confidence>=threshold)&(margin>=.5) - reports[str(threshold)]={'correct':int(((pred==truth)&accepted).sum()),'accepted':int(accepted.sum()), - 'wrong':int(((pred!=truth)&accepted).sum()), - 'false_visible':int(((truth==0)&(pred!=0)&accepted).sum()),'total':len(truth)} - per_locale={} - for i,name in enumerate(vm.get('locale_names',['unknown'])): - mask=locales==i;per_locale[name]={'correct':int((pred[mask]==truth[mask]).sum()),'total':int(mask.sum())} - np.savez_compressed(output/'fields-validation.npz',truth=truth,pred=pred,confidence=confidence,margin=margin,locales=locales) - torch.save(model.cpu().state_dict(),output/'fields.pt');export(model,output/'fields.onnx',(1,3,56,144)) - report={'training_samples':len(xs),'validation_samples':len(vx),'steps':steps,'batch':batch, - 'raw_correct':int((pred==truth).sum()),'thresholds':reports,'per_locale':per_locale, - 'train_corpus':tm,'validation_corpus':vm,'replay_corpus':rm if replay else None,'elapsed_s':time.perf_counter()-started} - write(output/'fields-evaluation.json',report) - print(json.dumps({k:v for k,v in report.items() if k not in ('train_corpus','validation_corpus')}),flush=True) - - -if __name__=='__main__': - p=argparse.ArgumentParser();p.add_argument('--train',type=Path,required=True);p.add_argument('--validation',type=Path,required=True) - p.add_argument('--output',type=Path,required=True);p.add_argument('--initial',type=Path) - p.add_argument('--steps',type=int,default=8000);p.add_argument('--batch',type=int,default=512) - p.add_argument('--replay',type=Path) - p.add_argument('--eval-every',type=int,default=0);p.add_argument('--stress',type=Path,help='Development stress corpus, never the final test set') - a=p.parse_args();train(a.train,a.validation,a.output,a.initial,a.steps,a.batch,a.replay,a.eval_every,a.stress) diff --git a/bdon_vision/train_scene_encoder.py b/bdon_vision/train_scene_encoder.py deleted file mode 100644 index 081ae2a..0000000 --- a/bdon_vision/train_scene_encoder.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Metric fine-tuning on complete-scene crops; held-out card IDs remain unseen.""" -import argparse -import json -import time -import hashlib -from pathlib import Path - -import numpy as np -import torch -from torch.nn import functional as F - -from .assets import read,write -from .train import CardEncoder,seed_all,tensor_images,export,distort - - -def load_corpus(path,cards): - doc=read(path/'manifest.json') - if not doc['complete']:raise ValueError('Incomplete corpus') - if doc['catalog_ids']!=[[c['kind'],c['id']] for c in cards]:raise ValueError('Catalog order differs; rebuild crop bank') - count=doc['art_count'] - x=torch.from_numpy(np.array(np.load(path/'art.npy',mmap_mode='r')[:count])).permute(0,3,1,2).cuda().float().div_(255) - labels=np.array(np.load(path/'art-labels.npy',mmap_mode='r')[:count]) - return x,torch.from_numpy(labels).long().cuda(),doc - - -def train(data,train_dir,valid_dir,output,initial,steps=2400,batch=128,eval_every=0,stress=None): - if not torch.cuda.is_available():raise RuntimeError('GPU required') - output.mkdir(parents=True,exist_ok=True);seed_all(71933);torch.set_num_threads(8) - cards=read(data/'catalog.json')['cards'];xs,ys,tm=load_corpus(train_dir,cards) - if set(tm['profiles'])!={'train'}:raise ValueError('Encoder training must use only train-profile scenes') - held=np.array([c['id']%7==0 for c in cards]);training=np.flatnonzero(~held) - if held[ys.cpu().numpy()].any():raise ValueError('Held-out card leaked into training crop bank') - vx,vy,vm=load_corpus(valid_dir,cards) - if stress:sx,sy,sm=load_corpus(stress,cards) - canonical=tensor_images([data/c.get('match_file',c['file']) for c in cards],(128,160)).cuda() - train_ids=torch.from_numpy(training).long().cuda() - inverse=torch.full((len(cards),),-1,dtype=torch.long,device='cuda');inverse[train_ids]=torch.arange(len(training),device='cuda') - model=CardEncoder();model.load_state_dict(torch.load(initial,map_location='cpu',weights_only=True));model=model.cuda() - optimizer=torch.optim.AdamW(model.parameters(),lr=.00012,weight_decay=.0001) - started=time.perf_counter() - curve=[] - def metrics(x,y): - was_training=model.training;model.eval();correct=accepted=accepted_correct=0;loss=0. - split={k:{'correct':0,'total':0} for k in ['seen','new_card_holdout']} - with torch.inference_mode(): - gallery=model(canonical) - for start in range(0,len(x),batch): - scores=model(x[start:start+batch])@gallery.T;top=scores.topk(2,dim=1) - truth=y[start:start+batch];ok=top.indices[:,0]==truth - keep=(top.values[:,0]>=.72)&((top.values[:,0]-top.values[:,1])>=.06) - correct+=int(ok.sum());accepted+=int(keep.sum());accepted_correct+=int((ok&keep).sum()) - loss+=float(F.cross_entropy(scores/.08,truth,reduction='sum')) - indices=truth.cpu().numpy();oks=ok.cpu().numpy() - for name,mask in [('seen',~held[indices]),('new_card_holdout',held[indices])]: - split[name]['correct']+=int(oks[mask].sum());split[name]['total']+=int(mask.sum()) - model.train(was_training) - return {'samples':len(x),'raw_correct':correct,'cross_entropy':loss/len(x), - 'accepted_retrieval':accepted,'accepted_correct':accepted_correct,'identity_split':split} - def checkpoint(step,train_loss=None): - row={'step':step,'train_loss':train_loss,'elapsed_s':time.perf_counter()-started,'validation':metrics(vx,vy)} - if stress:row['development_stress']=metrics(sx,sy) - curve.append(row);write(output/'learning-curve.json',{'checkpoints':curve, - 'initial_sha256':hashlib.sha256(initial.read_bytes()).hexdigest(), - 'scope':'Development oracle artwork crops, all-ID gallery; fresh AdamW state for continued fine-tuning'}) - torch.save({k:v.detach().cpu() for k,v in model.state_dict().items()},output/f'encoder-step-{step:06d}.pt') - print(json.dumps({'validation_checkpoint':row}),flush=True) - if eval_every:checkpoint(0) - for step in range(steps): - # The reference branch only sees training IDs. Evaluation galleries may - # add held-out/new cards later without updating any model parameter. - if step%20==0: - model.eval() - with torch.no_grad():references=model(canonical[train_ids]) - model.train() - ix=torch.randint(len(xs),(batch,),device='cuda');targets=inverse[ys[ix]] - query=xs[ix] - if step%3==0:query=distort(query,strong=False) - embeddings=model(query);logits=embeddings@references.T/.08 - loss=F.cross_entropy(logits,targets)+.15*(1-(embeddings*references[targets]).sum(1)).mean() - optimizer.zero_grad(set_to_none=True);loss.backward();optimizer.step() - if step==0 or (step+1)%100==0: - status={'step':step+1,'steps':steps,'loss':float(loss.detach()),'elapsed_s':round(time.perf_counter()-started,1), - 'gpu_allocated_gib':round(torch.cuda.memory_allocated()/2**30,2)} - write(output/'encoder-progress.json',status);print(json.dumps(status),flush=True) - if eval_every and ((step+1)%eval_every==0 or step+1==steps):checkpoint(step+1,float(loss.detach())) - model.eval();predictions=[];similarities=[];margins=[] - with torch.no_grad(): - gallery=model(canonical) - for start in range(0,len(vx),batch): - scores=model(vx[start:start+batch])@gallery.T;top=scores.topk(2,dim=1) - predictions.append(top.indices[:,0].cpu().numpy());similarities.append(top.values[:,0].cpu().numpy()) - margins.append((top.values[:,0]-top.values[:,1]).cpu().numpy()) - pred=np.concatenate(predictions);truth=vy.cpu().numpy();results={} - for name,mask in [('seen',~held[truth]),('new_card_holdout',held[truth])]: - results[name]={'correct':int((pred[mask]==truth[mask]).sum()),'total':int(mask.sum())} - np.savez_compressed(output/'encoder-scene-validation.npz',truth=truth,pred=pred, - similarity=np.concatenate(similarities),margin=np.concatenate(margins)) - torch.save(model.cpu().state_dict(),output/'encoder.pt');export(model,output/'encoder.onnx',(1,3,160,128)) - report={'method':'train-only native scene crops; disjoint acquisition validation; holdout IDs excluded from both training branches', - 'results':results,'training_steps':steps,'batch':batch,'training_scenes':tm['scenes'],'training_crops':len(xs), - 'validation_scenes':vm['scenes'],'validation_crops':len(vx),'train_fingerprint':tm['source_fingerprint'], - 'validation_fingerprint':vm['source_fingerprint'],'elapsed_s':time.perf_counter()-started} - write(output/'encoder-evaluation.json',report);print(json.dumps(report),flush=True) - - -if __name__=='__main__': - p=argparse.ArgumentParser();p.add_argument('--data',type=Path,required=True);p.add_argument('--train',type=Path,required=True) - p.add_argument('--validation',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--initial',type=Path,required=True) - p.add_argument('--steps',type=int,default=2400);p.add_argument('--batch',type=int,default=128) - p.add_argument('--eval-every',type=int,default=0);p.add_argument('--stress',type=Path) - a=p.parse_args();train(a.data,a.train,a.validation,a.output,a.initial,a.steps,a.batch,a.eval_every,a.stress) diff --git a/bdon_vision/web.js b/bdon_vision/web.js index c84398d..6f7a4ac 100644 --- a/bdon_vision/web.js +++ b/bdon_vision/web.js @@ -95,7 +95,9 @@ $('scan').onclick=async()=>{ const body=await response.json();if(!response.ok)throw Error(body.error); for(const url of imageURLs)URL.revokeObjectURL(url);imageURLs=selected.map(f=>URL.createObjectURL(f));result=body;draw(); const other=body.scans.filter(s=>s.cards.length&&s.cards.filter(c=>c.display_mode==='other').length>=Math.ceil(s.cards.length/2)).length; + const unknown=body.scans.reduce((n,s)=>n+s.unidentified.length,0); $('status').textContent='已完成 · '+body.box.unique_count+' 张不同卡片 · '+(body.scans.reduce((n,s)=>n+s.elapsed_ms,0)/1000).toFixed(2)+' 秒'+ + (unknown?'。有 '+unknown+' 张卡片未能确认身份；如果是新卡，请更新 master 数据并重新运行 prepare 后再识别。':'')+ (other?'。部分截图显示其他参数；要补齐等级或特训次数，请上传对应显示模式的截图。':''); }catch(e){$('status').textContent='识别失败：'+e.message} finally{$('scan').disabled=!files.length;$('reset').disabled=false} diff --git a/docs/DECK_PIPELINE.md b/docs/DECK_PIPELINE.md index 64855fb..ad0373b 100644 --- a/docs/DECK_PIPELINE.md +++ b/docs/DECK_PIPELINE.md @@ -18,7 +18,7 @@ CLI 和 HTTP 补齐共用工作区审阅状态。补齐开始即写入 `needs-re ## 数据身份与拥有集合 -`visual-data-manifest.json` 支持 `region`、`masterVersion`、`deckDataSha256`、`resourceVersion`、`resourceHash`、`officialCatalogSha256`、`catalogSha256`、`indexSha256`、`encoderSha256`、`fieldModelSha256`、`gallerySha256` 及 `files` 相对路径哈希字典。出现的声明均与实际加载内容核对；DeckData 从同一字节快照解析和计算 SHA256。篡改 master 行但保留自报版本也会拒绝。每个推荐请求另冻结实际 DeckData／Roster 字节供程序读取，并保存这些真实输入的哈希与命令。 +`visual-data-manifest.json` 支持 `region`、`masterVersion`、`deckDataSha256`、`resourceVersion`、`resourceHash`、`officialCatalogSha256`、`catalogSha256`、`recognitionModelsSha256` 及 `files` 相对路径哈希字典。出现的声明均与实际加载内容核对；DeckData 从同一字节快照解析和计算 SHA256。篡改 master 行但保留自报版本也会拒绝。每个推荐请求另冻结实际 DeckData／Roster 字节供程序读取，并保存这些真实输入的哈希与命令。 无 manifest 的旧目录会明确报告缺失／多余身份，不声称完整覆盖。图库身份覆盖只表示识别能力，不能扩大玩家持有集合。`coverage: observed_only` 在人工明确完成所有持有和培养信息前始终保留。不在绑定 master 的身份（例如旧快照留影 #70 不在当前 JP 数据中）不会被映射成其他卡片。 @@ -88,4 +88,4 @@ python -m pip install -e ".[test]" python -m pytest -q tests ``` -公共回归使用手写 synthetic master 和无游戏图像的 JSON，不分发原始 master、chart、APK、截图、卡面、账号或内部协作文件。原有 scan、serve 与训练路径继续可用。根目录 public-manifest.json 记录原 v0.1.0 发布基线；此 PR 的新输入管线由 git commit 与构建结果记录，未更新或重标旧发布／模型证据。 +公共回归使用手写 synthetic master 和无游戏图像的 JSON，不分发原始 master、chart、APK、截图、卡面、账号或内部协作文件。根目录 public-manifest.json 记录发布源码各文件的字节数和 SHA-256。 diff --git a/schemas/box.schema.json b/schemas/box.schema.json index de6c8b1..1b08640 100644 --- a/schemas/box.schema.json +++ b/schemas/box.schema.json @@ -432,13 +432,13 @@ "required": [ "schema", "files", + "galleries", "inference_core_sha256", - "field_confidence_threshold", "onnxruntime_version" ], "properties": { "schema": { - "const": "ournotes-boxlens.recognition-provenance/1" + "const": "ournotes-boxlens.recognition-provenance/2" }, "files": { "type": "object", @@ -447,15 +447,28 @@ "pattern": "^[0-9a-f]{64}$" } }, + "galleries": { + "type": "object", + "properties": { + "member": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "snap": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "required": [ + "member", + "snap" + ], + "additionalProperties": false + }, "inference_core_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "field_confidence_threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, "onnxruntime_version": { "type": "string" } diff --git a/schemas/examples/scan-response.json b/schemas/examples/scan-response.json index 1903695..e6cda92 100644 --- a/schemas/examples/scan-response.json +++ b/schemas/examples/scan-response.json @@ -1,5 +1,5 @@ { - "schema": "ournotes-boxlens.scan-result/1", + "schema": "ournotes-boxlens.scan-result/2", "box": { "schema": "bdon-box/1", "player": "example", @@ -16,6 +16,7 @@ "height": 64, "source_id": "bd724c65d1ae73db8ab90fce38e0d27028b2ce45cabdcca9e42774871ebce58c", "cards": [], + "unidentified": [], "elapsed_ms": 1.38, "coverage": "observed_only" } diff --git a/schemas/observations.schema.json b/schemas/observations.schema.json index 9f748b8..a0667f7 100644 --- a/schemas/observations.schema.json +++ b/schemas/observations.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:ournotes-boxlens:schema:observations:1", - "title": "Our Notes BoxLens observations v1", + "$id": "urn:ournotes-boxlens:schema:observations:2", + "title": "Our Notes BoxLens observations v2", "type": "array", "items": { "type": "object", @@ -11,6 +11,7 @@ "width", "height", "cards", + "unidentified", "elapsed_ms", "coverage" ], @@ -41,9 +42,10 @@ "rarity", "card_type", "bbox", - "identity_confidence", - "inliers", + "locator_score", "visible_fraction", + "identity_similarity", + "identity_margin", "level", "card_rank", "awake_count", @@ -90,22 +92,29 @@ ], "items": false, "minItems": 4, - "maxItems": 4 + "maxItems": 4, + "description": "Card tile rectangle [x, y, w, h] in image pixels" }, - "identity_confidence": { + "locator_score": { "type": "number", "minimum": 0, "maximum": 1 }, - "inliers": { - "type": "integer", - "minimum": 0 - }, "visible_fraction": { "type": "number", "minimum": 0, "maximum": 1 }, + "identity_similarity": { + "type": "number", + "minimum": -1, + "maximum": 1 + }, + "identity_margin": { + "type": "number", + "minimum": 0, + "maximum": 2 + }, "level": { "type": "object", "required": [ @@ -207,17 +216,206 @@ }, "review": { "type": "boolean" + } + }, + "additionalProperties": false + } + }, + "unidentified": { + "type": "array", + "description": "Located tiles whose artwork matched no gallery card", + "items": { + "type": "object", + "required": [ + "kind", + "bbox", + "locator_score", + "visible_fraction", + "identity_similarity", + "identity_margin", + "candidate", + "level", + "card_rank", + "awake_count", + "display_mode", + "review" + ], + "properties": { + "kind": { + "enum": [ + "member", + "snap" + ] }, - "identity_method": { - "const": "grid_gallery" - }, - "embedding_similarity": { - "type": [ - "number", - "null" + "bbox": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "number", + "exclusiveMinimum": 0 + } ], + "items": false, + "minItems": 4, + "maxItems": 4, + "description": "Card tile rectangle [x, y, w, h] in image pixels" + }, + "locator_score": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "visible_fraction": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "identity_similarity": { + "type": "number", "minimum": -1, "maximum": 1 + }, + "identity_margin": { + "type": "number", + "minimum": 0, + "maximum": 2 + }, + "candidate": { + "description": "Closest gallery card, below the acceptance thresholds", + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "level": { + "type": "object", + "required": [ + "value", + "confidence" + ], + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + { + "type": "null" + } + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reason": { + "type": "string" + }, + "source": { + "const": "manual" + } + }, + "additionalProperties": false + }, + "card_rank": { + "type": "object", + "required": [ + "value", + "confidence" + ], + "properties": { + "value": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 5 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reason": { + "type": "string" + }, + "source": { + "const": "manual" + } + }, + "additionalProperties": false + }, + "awake_count": { + "type": "object", + "required": [ + "value", + "confidence" + ], + "properties": { + "value": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 5 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reason": { + "type": "string" + }, + "source": { + "const": "manual" + } + }, + "additionalProperties": false + }, + "display_mode": { + "enum": [ + "unknown", + "other", + "level", + "training" + ] + }, + "review": { + "const": true } }, "additionalProperties": false @@ -241,13 +439,13 @@ "required": [ "schema", "files", + "galleries", "inference_core_sha256", - "field_confidence_threshold", "onnxruntime_version" ], "properties": { "schema": { - "const": "ournotes-boxlens.recognition-provenance/1" + "const": "ournotes-boxlens.recognition-provenance/2" }, "files": { "type": "object", @@ -256,15 +454,28 @@ "pattern": "^[0-9a-f]{64}$" } }, + "galleries": { + "type": "object", + "properties": { + "member": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "snap": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "required": [ + "member", + "snap" + ], + "additionalProperties": false + }, "inference_core_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "field_confidence_threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, "onnxruntime_version": { "type": "string" } diff --git a/schemas/scan-response.schema.json b/schemas/scan-response.schema.json index b30fbf0..f0a84bc 100644 --- a/schemas/scan-response.schema.json +++ b/schemas/scan-response.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:ournotes-boxlens:schema:scan-response:1", + "$id": "urn:ournotes-boxlens:schema:scan-response:2", "title": "Our Notes BoxLens scan-response v1", "type": "object", "required": [ @@ -10,7 +10,7 @@ ], "properties": { "schema": { - "const": "ournotes-boxlens.scan-result/1" + "const": "ournotes-boxlens.scan-result/2" }, "box": { "type": "object", @@ -443,13 +443,13 @@ "required": [ "schema", "files", + "galleries", "inference_core_sha256", - "field_confidence_threshold", "onnxruntime_version" ], "properties": { "schema": { - "const": "ournotes-boxlens.recognition-provenance/1" + "const": "ournotes-boxlens.recognition-provenance/2" }, "files": { "type": "object", @@ -458,15 +458,28 @@ "pattern": "^[0-9a-f]{64}$" } }, + "galleries": { + "type": "object", + "properties": { + "member": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "snap": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "required": [ + "member", + "snap" + ], + "additionalProperties": false + }, "inference_core_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "field_confidence_threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, "onnxruntime_version": { "type": "string" } @@ -487,6 +500,7 @@ "width", "height", "cards", + "unidentified", "elapsed_ms", "coverage" ], @@ -517,9 +531,10 @@ "rarity", "card_type", "bbox", - "identity_confidence", - "inliers", + "locator_score", "visible_fraction", + "identity_similarity", + "identity_margin", "level", "card_rank", "awake_count", @@ -566,22 +581,29 @@ ], "items": false, "minItems": 4, - "maxItems": 4 + "maxItems": 4, + "description": "Card tile rectangle [x, y, w, h] in image pixels" }, - "identity_confidence": { + "locator_score": { "type": "number", "minimum": 0, "maximum": 1 }, - "inliers": { - "type": "integer", - "minimum": 0 - }, "visible_fraction": { "type": "number", "minimum": 0, "maximum": 1 }, + "identity_similarity": { + "type": "number", + "minimum": -1, + "maximum": 1 + }, + "identity_margin": { + "type": "number", + "minimum": 0, + "maximum": 2 + }, "level": { "type": "object", "required": [ @@ -683,17 +705,206 @@ }, "review": { "type": "boolean" + } + }, + "additionalProperties": false + } + }, + "unidentified": { + "type": "array", + "description": "Located tiles whose artwork matched no gallery card", + "items": { + "type": "object", + "required": [ + "kind", + "bbox", + "locator_score", + "visible_fraction", + "identity_similarity", + "identity_margin", + "candidate", + "level", + "card_rank", + "awake_count", + "display_mode", + "review" + ], + "properties": { + "kind": { + "enum": [ + "member", + "snap" + ] }, - "identity_method": { - "const": "grid_gallery" - }, - "embedding_similarity": { - "type": [ - "number", - "null" + "bbox": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "number", + "exclusiveMinimum": 0 + } ], + "items": false, + "minItems": 4, + "maxItems": 4, + "description": "Card tile rectangle [x, y, w, h] in image pixels" + }, + "locator_score": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "visible_fraction": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "identity_similarity": { + "type": "number", "minimum": -1, "maximum": 1 + }, + "identity_margin": { + "type": "number", + "minimum": 0, + "maximum": 2 + }, + "candidate": { + "description": "Closest gallery card, below the acceptance thresholds", + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "level": { + "type": "object", + "required": [ + "value", + "confidence" + ], + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + { + "type": "null" + } + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reason": { + "type": "string" + }, + "source": { + "const": "manual" + } + }, + "additionalProperties": false + }, + "card_rank": { + "type": "object", + "required": [ + "value", + "confidence" + ], + "properties": { + "value": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 5 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reason": { + "type": "string" + }, + "source": { + "const": "manual" + } + }, + "additionalProperties": false + }, + "awake_count": { + "type": "object", + "required": [ + "value", + "confidence" + ], + "properties": { + "value": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 5 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reason": { + "type": "string" + }, + "source": { + "const": "manual" + } + }, + "additionalProperties": false + }, + "display_mode": { + "enum": [ + "unknown", + "other", + "level", + "training" + ] + }, + "review": { + "const": true } }, "additionalProperties": false @@ -717,13 +928,13 @@ "required": [ "schema", "files", + "galleries", "inference_core_sha256", - "field_confidence_threshold", "onnxruntime_version" ], "properties": { "schema": { - "const": "ournotes-boxlens.recognition-provenance/1" + "const": "ournotes-boxlens.recognition-provenance/2" }, "files": { "type": "object", @@ -732,15 +943,28 @@ "pattern": "^[0-9a-f]{64}$" } }, + "galleries": { + "type": "object", + "properties": { + "member": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "snap": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "required": [ + "member", + "snap" + ], + "additionalProperties": false + }, "inference_core_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "field_confidence_threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, "onnxruntime_version": { "type": "string" } diff --git a/scripts/diagnose_occlusion.py b/scripts/diagnose_occlusion.py deleted file mode 100644 index 6ff716e..0000000 --- a/scripts/diagnose_occlusion.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Evaluate opaque overlays and viewport fades without inventing readability labels. - -Uses saved development scenes, never modifies source images or frozen tests. -Opaque-covered values must remain unknown. Faded values have uncertain human -readability, so their output rate and latent agreement are descriptive only. -""" -import argparse -import hashlib -import json -from collections import Counter, defaultdict -from pathlib import Path - -import cv2 -import numpy as np - -from bdon_vision.assets import read, write -from bdon_vision.engine import Engine -from bdon_vision.synthetic import bbox_iou - -FIELDS = ('level', 'card_rank', 'awake_count') - - -def modify(image, cards, effect): - result = image.copy() - regions = [] - for card in cards: - x, y, w, h = card['bbox'] - # Cover the complete lower UI band, including parameter and rank glyphs. - left, right = max(0, int(x-w*.05)), min(image.shape[1], int(np.ceil(x+w*1.05))) - top, bottom = max(0, int(y+h*.68)), min(image.shape[0], int(np.ceil(y+h*1.03))) - if right <= left or bottom <= top: - continue - if effect == 'opaque': - alpha = np.ones((bottom-top, 1, 1), np.float32) - else: - alpha = np.linspace(0, .98, bottom-top, dtype=np.float32)[:, None, None] - color = np.array([220, 225, 229], np.float32).reshape(1, 1, 3) - region = result[top:bottom, left:right].astype(np.float32) - result[top:bottom, left:right] = np.rint(region*(1-alpha)+color*alpha).astype(np.uint8) - regions.append([left, top, right-left, bottom-top]) - return result, regions - - -def run(data, datasets, output): - output.mkdir(parents=True, exist_ok=False) - engine = Engine(data, threads=2) - groups = defaultdict(Counter) - details, identities, effects = [], set(), ('opaque', 'fade') - catalog = {(c['kind'], c['id']): c for c in engine.cards} - for dataset in datasets: - truth = read(dataset/'truth.json') - for row_index, row in enumerate(truth['screenshots']): - original = cv2.imread(str(dataset/row['file'])) - if original is None: - raise ValueError(f"Unreadable image: {dataset/row['file']}") - for effect in effects: - scene, regions = modify(original, row['cards'], effect) - name = f'{dataset.parent.name}-{dataset.name}-{row_index:06d}-{effect}' - scan = engine.scan(scene, name) - errors = [] - for card in row['cards']: - identity = card['kind'], card['id'] - identities.add(identity) - meta = catalog[identity] - mode = card.get('display_mode', row.get('display_mode', 'unrecorded')) - keys = [effect, f"{effect}/rarity/{card['kind']}/{meta['rarity']}", - f"{effect}/type/{meta['card_type']}", f"{effect}/locale/{row.get('locale','unrecorded')}", - f'{effect}/mode/{mode}', f"{effect}/profile/{truth.get('profile','unrecorded')}"] - found = [c for c in scan['cards'] if (c['kind'], c['id']) == identity - and bbox_iou(c['bbox'], card['bbox']) >= .5] - found = max(found, key=lambda c: bbox_iou(c['bbox'], card['bbox'])) if found else None - counts = Counter(expected_cards=1, identified=int(found is not None)) - for field in FIELDS: - value = found[field]['value'] if found else None - if effect == 'opaque': - counts[field+'_hidden_targets'] += 1 - counts[field+'_hidden_emitted'] += int(value is not None) - if value is not None: - errors.append({'card': list(identity), 'field': field, 'value': value, - 'error': 'value_emitted_under_opaque_overlay'}) - else: - expected = card.get(field) - if expected is not None: - counts[field+'_uncertain_readability_targets'] += 1 - counts[field+'_emitted'] += int(value is not None) - counts[field+'_latent_agreement'] += int(value == expected) - counts[field+'_latent_disagreement'] += int(value is not None and value != expected) - elif value is not None: - counts[field+'_emitted_without_visible_source_truth'] += 1 - for key in keys: - groups[key].update(counts) - # Keep every scan and failure, plus a compact contact-sheet source. - detail = {'source_dataset': str(dataset), 'source_file': row['file'], 'effect': effect, - 'source_sha256': hashlib.sha256((dataset/row['file']).read_bytes()).hexdigest(), - 'modified_pixels_sha256': hashlib.sha256(scene.tobytes()).hexdigest(), - 'overlay_regions': regions, 'errors': errors, 'scan': scan} - details.append(detail) - if row_index < 2 or errors: - cv2.imwrite(str(output/(name+'.jpg')), scene, [cv2.IMWRITE_JPEG_QUALITY, 94]) - if row_index % 10 == 0: - print(json.dumps({'dataset': str(dataset), 'source_processed': row_index+1, - 'source_total': len(truth['screenshots'])}), flush=True) - write(output/'progress.json', {'complete': False, 'processed': len(details), 'groups': dict(groups)}) - report = {'schema': 'ournotes-boxlens.occlusion-diagnostic/1', 'complete': True, - 'source_screenshots': len(details)//2, 'modified_screenshots': len(details), - 'unique_source_identities': len(identities), 'groups': dict(groups), 'details': details, - 'recognition_provenance': getattr(engine, 'provenance', { - 'files': {name: hashlib.sha256((data/name).read_bytes()).hexdigest() - for name in ['catalog.json', 'index.npz', 'models/encoder.onnx', 'models/fields.onnx']}, - 'source': 'frozen evaluation implementation; model and catalog hashes recorded'}), - 'limits': ['Development diagnostic, not a new independent final test.', - 'Opaque values are fully removed; any emitted value is a hidden-field claim.', - 'Fade readability is uncertain: latent agreement is not human-readable accuracy.', - 'Per-card lower-band overlays approximate occlusion; not an exact full-game viewport.', - 'Identity recall is measured; this diagnostic does not score all extra predictions.']} - write(output/'report.json', report) - print(json.dumps({k: report[k] for k in ('modified_screenshots', 'unique_source_identities')} - | {'groups': {e: dict(groups[e]) for e in effects}}), flush=True) - - -if __name__ == '__main__': - p = argparse.ArgumentParser(description=__doc__) - p.add_argument('--data', type=Path, required=True) - p.add_argument('--datasets', type=Path, nargs='+', required=True) - p.add_argument('--output', type=Path, required=True) - a = p.parse_args() - run(a.data, a.datasets, a.output) diff --git a/scripts/publish_metrics.py b/scripts/publish_metrics.py deleted file mode 100644 index 61d09e3..0000000 --- a/scripts/publish_metrics.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Turn raw evaluation evidence into explicit public metrics without hiding abstentions.""" -import argparse -import hashlib -import json -from collections import Counter -from pathlib import Path - -from bdon_vision.assets import read,write - - -def ratio(n,d):return n/d if d else None - - -def summarize_counts(counts): - expected=counts.get('expected_cards',0);identified=counts.get('identified',0);extra=counts.get('extra_predictions',0) - fields={} - for field in ['level','card_rank','awake_count']: - total=counts.get(field+'_total',0);correct=counts.get(field+'_correct',0);wrong=counts.get(field+'_wrong',0);unknown=counts.get(field+'_unknown',0) - fields[field]={'visible_truth':total,'correct':correct,'wrong':wrong,'unknown_after_localization':unknown, - 'not_localized':total-correct-wrong-unknown,'end_to_end_accuracy':ratio(correct,total), - 'accepted_precision':ratio(correct,correct+wrong),'coverage':ratio(correct+wrong,total)} - return {'identity':{'expected':expected,'correct':identified,'extra':extra,'missed':expected-identified, - 'recall':ratio(identified,expected),'precision_on_scored_regions':ratio(identified,identified+extra)}, - 'fields':fields,'hidden_field_predictions':counts.get('hidden_field_predictions',0)} - - -def publish(evaluations,output): - output.mkdir(parents=True,exist_ok=True);runs={};lines=['# End-to-end evaluation metrics','', - 'Counts include abstentions and missed cards. Accepted precision is conditional on producing a value; it is not end-to-end accuracy. Ignored heavily cropped regions follow the raw evaluator policy.','', - '| Run | Screenshots | ID correct / truth | Extras | Hidden field claims |','|---|---:|---:|---:|---:|'] - for name,path in evaluations: - raw=read(path);summary=summarize_counts(raw['strata']['overall']) - hidden=Counter(e['field'] for row in raw['details'] for e in row['errors'] if e.get('error')=='field_not_visible') - boxes=raw.get('merged_boxes',[]);merged={key:Counter() for key in ['level','card_rank','awake_count']} - for box in boxes: - for key in merged:merged[key].update(box.get('fields',{}).get(key,{})) - entry={'screenshots':raw['screenshots'],**summary,'hidden_claims_by_field':dict(hidden), - 'latency_ms':raw['latency_ms'],'cpu_affinity':raw.get('cpu_affinity'),'platform':raw.get('platform'), - 'opencv_threads':raw.get('opencv_threads'),'matching':raw.get('matching'), - 'raw_report_sha256':hashlib.sha256(path.read_bytes()).hexdigest(), - 'strata':{k:summarize_counts(v) for k,v in raw['strata'].items()}, - 'merged_boxes':{'count':len(boxes),'duplicate_invariant':all(b['duplicate_upload_invariant'] for b in boxes), - 'fields':{k:dict(v) for k,v in merged.items()}}} - runs[name]=entry;ident=summary['identity'] - lines.append(f"| {name} | {raw['screenshots']} | {ident['correct']} / {ident['expected']} | {ident['extra']} | {entry['hidden_field_predictions']} |") - write(output/'raw'/f'{name}.json',raw) - for name,entry in runs.items(): - lines+=['',f'## {name}','', '| Field | Correct | Wrong | Unknown after localization | Not localized | Visible truth |', '|---|---:|---:|---:|---:|---:|'] - for key,f in entry['fields'].items():lines.append(f"| {key} | {f['correct']} | {f['wrong']} | {f['unknown_after_localization']} | {f['not_localized']} | {f['visible_truth']} |") - lines+=['',f"Scan median {entry['latency_ms']['median']:.2f} ms; P95 {entry['latency_ms']['p95']:.2f} ms. CPU affinity: {entry['cpu_affinity']}."] - report={'schema':'ournotes-boxlens.metrics/1','runs':runs, - 'limits':['Synthetic distribution metrics do not estimate arbitrary deployment accuracy.','Cards and views within a player box are correlated.','Confidence scores are not calibrated probabilities.','Latency excludes file decoding and HTTP; other host workload may be present.']} - write(output/'metrics.json',report);(output/'METRICS.md').write_text('\n'.join(lines)+'\n',encoding='utf8') - print(json.dumps({'runs':list(runs),'output':str(output)})) - - -if __name__=='__main__': - p=argparse.ArgumentParser();p.add_argument('--evaluation',action='append',required=True,metavar='NAME=PATH') - p.add_argument('--output',type=Path,required=True);a=p.parse_args() - rows=[] - for value in a.evaluation: - name,sep,path=value.partition('=') - if not sep or not name or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-' for c in name):p.error('Use a simple NAME=PATH') - rows.append((name,Path(path))) - if len({n for n,_ in rows})!=len(rows):p.error('Duplicate run names') - publish(rows,a.output) diff --git a/scripts/verify_gallery_update.py b/scripts/verify_gallery_update.py index e1ed5e5..a5d3131 100644 --- a/scripts/verify_gallery_update.py +++ b/scripts/verify_gallery_update.py @@ -1,4 +1,6 @@ -"""Integration check: remove trained-out identities, add their art/catalog, no retraining.""" +"""Integration check: a card missing from the catalog is reported as unidentified, +and becomes identified once its catalog entry and artwork are added. No model +file changes.""" import argparse import hashlib import shutil @@ -7,50 +9,72 @@ import cv2 import numpy as np -from bdon_vision.assets import read,write,build_index +from bdon_vision.assets import read, write from bdon_vision.engine import Engine -from bdon_vision.inference import Gallery +from bdon_vision.kind_encoder import reference_source -def verify(data,output,member_image=None,snap_image=None,member_id=7,snap_id=70): - data=Path(data);output=Path(output);target=output/'gallery-fixture' - paths={'member':Path(member_image) if member_image else data/'examples/member.jpg', - 'snap':Path(snap_image) if snap_image else data/'examples/snap.jpg'} - images={name:cv2.imdecode(np.frombuffer(path.read_bytes(),np.uint8),cv2.IMREAD_COLOR) if path.is_file() else None for name,path in paths.items()} +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def verify(data, output, member_image=None, snap_image=None, member_id=7, snap_id=70): + data = Path(data) + output = Path(output) + target = output / 'gallery-fixture' + paths = {'member': Path(member_image) if member_image else data / 'examples/member.jpg', + 'snap': Path(snap_image) if snap_image else data / 'examples/snap.jpg'} + images = {kind: cv2.imdecode(np.frombuffer(path.read_bytes(), np.uint8), cv2.IMREAD_COLOR) if path.is_file() else None + for kind, path in paths.items()} if any(image is None for image in images.values()): raise ValueError('Provide local --member-image and --snap-image; game screenshots are not distributed.') - cards=read(data/'catalog.json')['cards'];held={('member',member_id),('snap',snap_id)} - if not held.issubset({(c['kind'],c['id']) for c in cards}): + cards = read(data / 'catalog.json')['cards'] + held = {('member', member_id), ('snap', snap_id)} + if not held.issubset({(c['kind'], c['id']) for c in cards}): raise ValueError('Selected test identities must exist in the supplied catalog') - target.mkdir(parents=True,exist_ok=False) - old=[c for c in cards if (c['kind'],c['id']) not in held] - for path in [data/'models/encoder.onnx',data/'models/fields.onnx']: - dest=target/path.relative_to(data);dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(path,dest) + target.mkdir(parents=True, exist_ok=False) + shutil.copytree(data / 'models', target / 'models', ignore=shutil.ignore_patterns('*-gallery.npz')) for card in cards: - path=data/card.get('match_file',card['file']);dest=target/path.relative_to(data) - dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(path,dest) - shutil.copytree(data/'native/game-ui/sprites',target/'native/game-ui/sprites') - model_hash=lambda:hashlib.sha256((target/'models/encoder.onnx').read_bytes()).hexdigest() - before_hash=model_hash() - write(target/'catalog.json',{'cards':old});build_index(target);Gallery(target,read(target/'catalog.json')['cards']) - before=Engine(target);old_counts=len(before.cards) - before_ids={name:sorted((c['kind'],c['id']) for c in before.identify(im)) for name,im in images.items()} - # The updater rebuilds both RootSIFT and neural lookup. It does not train. - write(target/'catalog.json',{'cards':cards});build_index(target);Gallery(target,read(target/'catalog.json')['cards']) - after=Engine(target) - after_ids={name:sorted((c['kind'],c['id']) for c in after.fill_grid(im,after.identify(im))) for name,im in images.items()} - restored=all(identity in after_ids[identity[0]] for identity in held) - report={'removed_ids':sorted(held),'old_catalog_size':old_counts,'new_catalog_size':len(after.cards), - 'encoder_sha256_before':before_hash,'encoder_sha256_after':model_hash(), - 'model_unchanged':before_hash==model_hash(),'new_cards_recognized':restored, - 'before_ids':before_ids,'after_ids':after_ids, - 'scope':'User-provided local test images; incremental update integration, not an independent accuracy estimate'} - write(output/'gallery-update.json',report);print(report) - if not report['model_unchanged'] or not restored:raise RuntimeError('Gallery update regression') - - -if __name__=='__main__': - p=argparse.ArgumentParser();p.add_argument('--data',type=Path,required=True);p.add_argument('--output',type=Path,required=True) - p.add_argument('--member-image',type=Path);p.add_argument('--snap-image',type=Path) - p.add_argument('--member-id',type=int,default=7);p.add_argument('--snap-id',type=int,default=70) - a=p.parse_args();verify(a.data,a.output,a.member_image,a.snap_image,a.member_id,a.snap_id) + path = reference_source(data, card) + dest = target / path.relative_to(data) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, dest) + models = sorted(p for p in (target / 'models').iterdir() if p.suffix in ('.onnx', '.json')) + hashes = lambda: {p.name: digest(p) for p in models} + before_hashes = hashes() + + def observe(engine): + out = {} + for kind, image in images.items(): + scan = engine.scan(image, kind) + out[kind] = {'identified': sorted([c['kind'], c['id']] for c in scan['cards']), + 'unidentified': [[c['kind'], c['bbox']] for c in scan['unidentified']]} + return out + + write(target / 'catalog.json', {'cards': [c for c in cards if (c['kind'], c['id']) not in held]}) + before = observe(Engine(target)) + write(target / 'catalog.json', {'cards': cards}) + after = observe(Engine(target)) + rejected = all([kind, i] not in before[kind]['identified'] and before[kind]['unidentified'] for kind, i in held) + restored = all([kind, i] in after[kind]['identified'] for kind, i in held) + report = {'removed_ids': sorted(held), 'old_catalog_size': len(cards) - len(held), 'new_catalog_size': len(cards), + 'model_sha256_before': before_hashes, 'model_sha256_after': hashes(), + 'model_unchanged': before_hashes == hashes(), 'missing_cards_unidentified': rejected, + 'added_cards_identified': restored, 'before': before, 'after': after, + 'scope': 'User-provided local test images; incremental update integration, not an independent accuracy estimate'} + write(output / 'gallery-update.json', report) + print(report) + if not report['model_unchanged'] or not rejected or not restored: + raise RuntimeError('Gallery update regression') + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--data', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--member-image', type=Path) + p.add_argument('--snap-image', type=Path) + p.add_argument('--member-id', type=int, default=7) + p.add_argument('--snap-id', type=int, default=70) + a = p.parse_args() + verify(a.data, a.output, a.member_image, a.snap_image, a.member_id, a.snap_id) diff --git a/scripts/verify_weights.py b/scripts/verify_weights.py deleted file mode 100644 index e0907ec..0000000 --- a/scripts/verify_weights.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Check that public PyTorch checkpoints and ONNX inference weights agree.""" -import argparse -import hashlib -from pathlib import Path - -import numpy as np -import onnxruntime as ort -import torch -from torch import nn - -from bdon_vision.assets import write -from bdon_vision.train import CardEncoder,LevelNet - - -def verify(encoder_pt,encoder_onnx,fields_pt,fields_onnx,output): - torch.set_num_threads(2);rng=np.random.default_rng(64291) - encoder=CardEncoder();fields=LevelNet();fields.head=nn.Linear(96*3*8,106) - report={'schema':'ournotes-boxlens.weight-parity/1','seed':64291,'cases':{},'passed':True} - for name,model,checkpoint,onnx,shape in [ - ('encoder',encoder,encoder_pt,encoder_onnx,(4,3,160,128)), - ('fields',fields,fields_pt,fields_onnx,(4,3,56,144))]: - model.load_state_dict(torch.load(checkpoint,map_location='cpu',weights_only=True));model.eval() - x=rng.random(shape,dtype=np.float32);x[0]=0.;x[1]=1. - with torch.inference_mode():expected=model(torch.from_numpy(x)).numpy() - opts=ort.SessionOptions();opts.intra_op_num_threads=2;opts.inter_op_num_threads=1 - session=ort.InferenceSession(str(onnx),sess_options=opts,providers=['CPUExecutionProvider']) - actual=session.run(None,{'image':x})[0] - close=np.allclose(expected,actual,atol=2e-4,rtol=2e-4) - report['cases'][name]={'checkpoint_sha256':hashlib.sha256(checkpoint.read_bytes()).hexdigest(), - 'onnx_sha256':hashlib.sha256(onnx.read_bytes()).hexdigest(),'shape':shape, - 'max_absolute_error':float(np.abs(expected-actual).max()),'allclose':bool(close)} - if name=='fields':report['cases'][name]['class_agreement']=bool(np.array_equal(expected.argmax(1),actual.argmax(1))) - report['passed']&=bool(close) - write(output,report);print(report) - if not report['passed']:raise RuntimeError('Checkpoint/ONNX parity failed') - - -if __name__=='__main__': - p=argparse.ArgumentParser() - for name in ['encoder-pt','encoder-onnx','fields-pt','fields-onnx','output']:p.add_argument('--'+name,type=Path,required=True) - a=p.parse_args();verify(a.encoder_pt,a.encoder_onnx,a.fields_pt,a.fields_onnx,a.output) diff --git a/tests/test_deck_inputs.py b/tests/test_deck_inputs.py index 69788b6..22dd7a5 100644 --- a/tests/test_deck_inputs.py +++ b/tests/test_deck_inputs.py @@ -42,10 +42,10 @@ def fixture(tmp_path): inv=inventory(box,path,mock=True,region='test-region') c={'schema':'ournotes.inventory-completion/1','source':'mock-truth','evidence':'Hand-authored synthetic fixture, no actual player','mock':True,'deckDataSha256':digest(path),'ownershipComplete':True,'playerStateComplete':True,'player':{'characterRanks':{str(i):10 for i in range(1,8)},'bandItems':{'100':1},'vipRank':2,'events':[],'memory':{'musicRanks':{},'unlockedMembers':[],'unlockedSupports':[]},'ownedMemberCardIds':list(range(1,6)),'ownedSupportCardIds':[1]},'members':[{'id':i,'level':20,'rank':1,'awake':1,'liveSkillLevel':2,'gekisouSkillLevel':2} for i in range(1,6)],'snaps':[{'id':1,'level':20,'rank':1}],'addMissingIdentities':[],'excludeObservedIdentities':[]} data=tmp_path/'visual' - for name in ['index.npz','models/encoder.onnx','models/fields.onnx']: + for name in ['models/recognition.json']: p=data/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(b'synthetic-not-a-real-model') write(data/'catalog.json',{'schema':'bdon-box-catalog/1','cards':[{'kind':'member','id':i} for i in range(1,8)]+[{'kind':'snap','id':i} for i in range(1,4)]}) - write(data/'visual-data-manifest.json',{'region':'test-region','masterVersion':'synthetic-1','deckDataSha256':digest(path),'files':{'index.npz':digest(data/'index.npz')}}) + write(data/'visual-data-manifest.json',{'region':'test-region','masterVersion':'synthetic-1','deckDataSha256':digest(path),'files':{'models/recognition.json':digest(data/'models/recognition.json')}}) return path,data,inv,c @@ -117,7 +117,7 @@ def test_identity_outside_bound_master_is_not_remapped(fixture): assert any(i['path']=='snap:70' for i in error.value.issues) -@pytest.mark.parametrize('name,value',[('deckDataSha256','0'*64),('region','wrong'),('masterVersion','wrong'),('indexSha256','0'*64)]) +@pytest.mark.parametrize('name,value',[('deckDataSha256','0'*64),('region','wrong'),('masterVersion','wrong'),('recognitionModelsSha256','0'*64)]) def test_visual_manifest_binding(fixture,name,value): path,data,_,_=fixture;m=read(data/'visual-data-manifest.json');m[name]=value;write(data/'visual-data-manifest.json',m) with pytest.raises(ValueError):binding(data,path) @@ -133,7 +133,7 @@ def unexpected(*args,**kwargs):raise AssertionError('OCR worker must not start b def test_changed_visual_file_rejected(fixture): - path,data,_,_=fixture;(data/'index.npz').write_bytes(b'mutated') + path,data,_,_=fixture;(data/'models/recognition.json').write_bytes(b'mutated') with pytest.raises(ValueError,match='hash mismatch'):binding(data,path) diff --git a/tests/test_integrity.py b/tests/test_integrity.py index fe2b689..47f8638 100644 --- a/tests/test_integrity.py +++ b/tests/test_integrity.py @@ -1,4 +1,6 @@ -"""Regressions for cropped icons, hidden fields, and box-level truth auditing.""" +"""Regressions for model pinning, cropped fields and icons, and box-level truth auditing.""" +import hashlib +import json import tempfile import unittest from pathlib import Path @@ -6,25 +8,88 @@ import numpy as np from bdon_vision.audit_merge import audit_box -from bdon_vision.engine import Engine,field -from bdon_vision.ocr import NumberReader +from bdon_vision.engine import MODELS_SCHEMA, Engine, field, load_models from test_merge import observation +class Constant: + """Stand-in session returning fixed logits for every crop.""" + + def __init__(self, logits): + self.logits = np.asarray(logits, np.float32) + + def get_outputs(self): + return [type('Output', (), {'shape': [None, len(self.logits)]})()] + + def run(self, _, feeds): + return [np.repeat(self.logits[None], len(feeds['image']), axis=0)] + + +def certain(classes, index): + logits = np.full(classes, -20.) + logits[index] = 20. + return logits + + +def write_models(folder, mutate=None): + folder.mkdir(parents=True) + entry = lambda name: {'file': name, 'sha256': hashlib.sha256((folder/name).read_bytes()).hexdigest()} + names = ['locator.onnx'] + [f'{k}-{r}.onnx' for r in ('encoder', 'fields', 'rank') for k in ('member', 'snap')] + for name in names: + (folder/name).write_bytes(name.encode()) + manifest = {'schema': MODELS_SCHEMA, 'locator': {**entry('locator.onnx'), 'long_edge': 640, 'threshold': .4}, + 'encoders': {k: {**entry(f'{k}-encoder.onnx'), 'accept': {'similarity': .8, 'margin': .1}} for k in ('member', 'snap')}, + 'fields': {k: {**entry(f'{k}-fields.onnx'), 'accept': {'confidence': .995, 'margin': .5}} for k in ('member', 'snap')}, + 'ranks': {k: {**entry(f'{k}-rank.onnx'), 'accept': {'confidence': .995, 'margin': .5}} for k in ('member', 'snap')}} + if mutate: + mutate(folder) + (folder/'recognition.json').write_text(json.dumps(manifest), encoding='utf-8') + + +def reader(member_field=12, snap_field=40, rank=3): + engine = Engine.__new__(Engine) + accept = {'confidence': .995, 'margin': .5} + engine.models = {'fields': {'member': {'accept': accept}, 'snap': {'accept': accept}}, + 'ranks': {'member': {'accept': accept}, 'snap': {'accept': accept}}} + engine.fields = {'member': Constant(certain(106, member_field)), 'snap': Constant(certain(101, snap_field))} + engine.ranks = {'member': Constant(certain(6, rank)), 'snap': Constant(certain(6, rank))} + return engine + + class IntegrityTests(unittest.TestCase): - def test_icon_outside_right_edge_is_unknown_before_template_matching(self): - engine=Engine.__new__(Engine) - image=np.zeros((405,877,3),np.uint8) - item={'kind':'member','bbox':[763.25,54.33,115.71,152.94]} - result=engine.read_rank(image,item) - self.assertIsNone(result['value']) - self.assertEqual(result['reason'],'cropped_rank_icon') - - def test_level_only_model_cannot_silently_read_parameter_modes(self): + def test_model_manifest_is_required_and_hashes_are_checked(self): with tempfile.TemporaryDirectory() as folder: - path=Path(folder);(path/'models').mkdir();(path/'models/level.onnx').touch() - with self.assertRaisesRegex(FileNotFoundError,'parameter-mode'): - NumberReader(path) + with self.assertRaisesRegex(FileNotFoundError, 'model manifest'): + load_models(folder) + write_models(Path(folder)/'models') + self.assertEqual(load_models(folder)['schema'], MODELS_SCHEMA) + with tempfile.TemporaryDirectory() as folder: + write_models(Path(folder)/'models', lambda models: (models/'snap-rank.onnx').write_bytes(b'changed')) + with self.assertRaisesRegex(ValueError, 'hash mismatch for snap-ranks'): + load_models(folder) + + def test_fields_and_icons_outside_the_image_stay_unknown(self): + engine = reader() + image = np.zeros((405, 877, 3), np.uint8) + inside = {'kind': 'member', 'bbox': [100., 50., 112., 147.]} + right_edge = {'kind': 'member', 'bbox': [763.25, 54.33, 115.71, 152.94]} + bottom_edge = {'kind': 'snap', 'bbox': [300., 340., 163., 92.]} + a, b, c = engine.read_fields(image, [inside, right_edge, bottom_edge]) + self.assertEqual((a['level']['value'], a['card_rank']['value'], a['display_mode']), (12, 3, 'level')) + self.assertEqual(a['awake_count']['reason'], 'not_visible_in_level_view') + self.assertEqual(b['card_rank']['reason'], 'cropped_rank_icon') + self.assertIsNone(c['level']['value']) + self.assertEqual(c['level']['reason'], 'cropped') + + def test_training_count_and_hidden_parameter(self): + image = np.zeros((400, 400, 3), np.uint8) + tile = {'kind': 'member', 'bbox': [100., 50., 112., 147.]} + training = reader(member_field=103).read_fields(image, [tile])[0] + self.assertEqual((training['awake_count']['value'], training['display_mode']), (3, 'training')) + self.assertEqual(training['level']['reason'], 'not_visible_in_training_view') + hidden = reader(member_field=0, rank=0).read_fields(image, [tile])[0] + self.assertEqual((hidden['level']['value'], hidden['display_mode']), (None, 'other')) + self.assertEqual(hidden['card_rank']['reason'], 'no_visible_rank_icon') def test_box_audit_rejects_invented_fields_and_preserves_truth_conflicts(self): rows=[{'cards':[{'kind':'member','id':1,'level':20,'card_rank':None,'awake_count':None}]}, diff --git a/tests/test_ui_geometry.py b/tests/test_ui_geometry.py index 63fd3ba..fa0ef3f 100644 --- a/tests/test_ui_geometry.py +++ b/tests/test_ui_geometry.py @@ -1,24 +1,26 @@ -"""Native Snap aspect-fill placement must preserve full-art reference boxes.""" +"""Tile, artwork-window and reference geometry of the native list layout.""" import unittest -import numpy as np +from PIL import Image -from bdon_vision.engine import Engine +from bdon_vision.kind_geometry import art_window, center_crop_box, reference_image class UIFrameGeometryTests(unittest.TestCase): - def test_snap_full_art_reference_maps_to_native_window(self): - engine = Engine.__new__(Engine) - engine.art = [np.zeros((288, 512, 3), dtype=np.uint8)] - item = {'kind': 'snap', '_index': 0, 'bbox': [10., 20., 314., 176.625]} - self.assertEqual(engine.ui_bbox(item), [10., 22.3125, 314., 172.]) - self.assertEqual(item['bbox'], [10., 20., 314., 176.625]) - - def test_member_canonical_reference_is_not_cropped_again(self): - engine = Engine.__new__(Engine) - engine.art = [np.zeros((282, 212, 3), dtype=np.uint8)] - item = {'kind': 'member', '_index': 0, 'bbox': [10., 20., 212., 282.]} - self.assertEqual(engine.ui_bbox(item), item['bbox']) + def test_artwork_window_is_inset_by_six_logical_units(self): + self.assertEqual(art_window('member', [10., 20., 224., 294.]), (16., 26., 212., 282.)) + self.assertEqual(art_window('snap', [0., 0., 163., 92.]), (3., 3., 157., 86.)) + + def test_snap_reference_takes_the_centre_of_the_full_artwork(self): + left, top, right, bottom = center_crop_box(512, 288, 314/172) + self.assertEqual((left, right), (0., 512.)) + self.assertAlmostEqual(top, (288-512*172/314)/2) + self.assertAlmostEqual(bottom, 288-top) + self.assertEqual(reference_image('snap', Image.new('RGB', (512, 288))).size, (224, 128)) + + def test_member_reference_fills_the_portrait_window(self): + self.assertEqual(reference_image('member', Image.new('RGB', (256, 256))).size, (128, 160)) + self.assertEqual(reference_image('member', Image.new('RGB', (212, 282))).size, (128, 160)) if __name__ == '__main__': From 348da8d6a440c488b876df6704ad556432a2c9d0 Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:45:48 +0800 Subject: [PATCH 6/7] docs: document the 0.2.0 models, training data and evaluation - README, model card and contract describe the locator, encoder, field and rank pipeline, models/recognition.json, the gallery caches and the `unidentified` output - training guide lists the scene, crop-bank, training, export, selection and evaluation commands with the seeds used for the released weights - data card and coverage list every full-screen and crop set, resolutions per profile, identity split and field distributions - evaluation card reports the frozen final-profile results and the real regression set; reports/ holds model selection, scene coverage and the stratified acceptance summaries - reproduction guide covers the training chain of the shared initial encoder and field weights - third-party notices add the MobileNetV3-Large backbone of the locator --- README.md | 35 +- THIRD_PARTY_NOTICES.md | 9 +- docs/ASSET_BOUNDARY.md | 8 +- docs/CONTRACT.md | 25 +- docs/COVERAGE.md | 37 +- docs/DATA_CARD.md | 65 +- docs/EVALUATION.md | 68 +- docs/EXPERIMENTS.md | 25 - docs/MODEL_CARD.md | 75 +- docs/NEW-CARDS.md | 22 +- docs/REPRODUCTION.md | 20 +- docs/RESOURCES.md | 2 +- docs/TRAINING.md | 82 +- reports/README.md | 11 +- reports/acceptance/clear-rank-matrix.json | 421 -- reports/acceptance/holdout-full-screen.json | 1893 ++++++++ .../acceptance/real-sample-regression.json | 426 +- reports/acceptance/synthetic-data-audit.json | 234 - reports/acceptance/synthetic-full-screen.json | 4089 ----------------- reports/acceptance/unknown-card-results.json | 1735 ------- reports/performance/cpu-2-logical-cores.json | 603 --- reports/performance/cpu-4-logical-cores.json | 605 --- reports/training/card-coverage.json | 570 --- reports/training/checkpoint-selection.json | 26 - reports/training/field-learning-curve.json | 499 -- reports/training/hard-field-validation.json | 177 - reports/training/level-distribution.json | 199 - reports/training/model-selection.json | 1035 +++++ reports/training/rank-distribution.json | 26 - reports/training/scene-coverage.json | 604 +++ 30 files changed, 4000 insertions(+), 9626 deletions(-) delete mode 100644 docs/EXPERIMENTS.md delete mode 100644 reports/acceptance/clear-rank-matrix.json create mode 100644 reports/acceptance/holdout-full-screen.json delete mode 100644 reports/acceptance/synthetic-data-audit.json delete mode 100644 reports/acceptance/synthetic-full-screen.json delete mode 100644 reports/acceptance/unknown-card-results.json delete mode 100644 reports/performance/cpu-2-logical-cores.json delete mode 100644 reports/performance/cpu-4-logical-cores.json delete mode 100644 reports/training/card-coverage.json delete mode 100644 reports/training/checkpoint-selection.json delete mode 100644 reports/training/field-learning-curve.json delete mode 100644 reports/training/hard-field-validation.json delete mode 100644 reports/training/level-distribution.json create mode 100644 reports/training/model-selection.json delete mode 100644 reports/training/rank-distribution.json create mode 100644 reports/training/scene-coverage.json diff --git a/README.md b/README.md index 29d186c..b17a4e9 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,26 @@ 仓库名：`ournotes-boxlens`。项目源码及训练权重采用 MIT，保留第三方许可声明；游戏资源不随项目分发，见 [资源边界](docs/ASSET_BOUNDARY.md)。输入输出见 [CONTRACT.md](docs/CONTRACT.md)。 -从成员／留影列表截图识别 master ID，以及截图实际显示的等级、卡阶和成员特训次数。支持多张截图合并、重复上传去重、来源图预览和手工校对。推理使用 CPU；训练才需要 GPU。 +从成员／留影列表截图识别 master ID，以及截图实际显示的等级、卡阶和成员特训次数。支持多张截图合并、重复上传去重、来源图预览和手工校对。推理只用 CPU 和 ONNX Runtime；训练才需要 GPU。 + +## 识别流程 + +1. `locator.onnx` 在整张截图上找出成员和留影卡框，包括被屏幕边缘切掉一部分的卡。 +2. 每类卡各有一个卡面编码器（`member-encoder.onnx`、`snap-encoder.onnx`），把卡框内的卡面窗口编码成 128 维向量，再在本地图库里检索最相近的卡。相似度和与第二名的差距都达到阈值才接受这个 ID；否则该卡框进入 `unidentified`，附最相近的候选供人工确认。 +3. 每类卡各有一个参数读取模型（`member-fields.onnx`、`snap-fields.onnx`）和一个卡阶图标模型（`member-rank.onnx`、`snap-rank.onnx`）。读取区域必须完整位于截图内，置信度达到阈值才输出值；等级还要符合 masterdata 的等级上限。 + +图库由使用者本地的卡面图编码而成，不属于模型权重。新卡只需更新 master 表和卡面，再重建图库，模型不用重训，见 [新卡接入](docs/NEW-CARDS.md)。 + +**当前版本：v0.2.0，面向列表截图。** 无法可靠读取的字段返回 `null`，无法确认身份的卡框列在 `unidentified` 里，二者都需要校对；遮挡、渐隐和重压缩仍可能造成误读，请把卡片滚动到画面中央、重新截取清晰图片后再检查。输出只表示截图观察到的卡片，不能据此认定未出现的卡片不在玩家 box 中。模型置信度和相似度是内部得分，不是经过校准的正确概率。 ## 仓库结构 -- `bdon_vision/`：当前识别、合成数据与本地导入界面。 +- `bdon_vision/`：识别、合成数据、训练与评估模块和本地导入界面。 - `docs/`：使用说明、训练方法、资源边界、模型卡和评估说明。 -- `scripts/`：资源准备、训练复现和评估工具。 +- `scripts/`：资源准备、训练复现和验证工具。 - `tests/`、`schemas/`：数据完整性测试和输入输出契约。 -- `reports/`：只保留训练覆盖、最终验收和性能指标；不含中间 smoke、截图或逐图调试记录，见 [报告索引](reports/README.md)。 -- `reproduction/`：发布权重训练谱系所需的冻结源码和哈希清单；不是运行产物或历史实验堆栈。 - -**当前版本：v0.1.0，面向清晰列表截图。** 无法可靠读取的字段返回 `null`；但遮挡、渐隐和重压缩也可能造成误读，请先将卡片滚动到画面中央并重新截取清晰图片，再检查或手工校对结果。输出只表示截图观察到的卡片，不能据此认定未出现的卡片不在玩家 box 中。模型置信度是内部得分，不是经过校准的正确概率。 +- `reports/`：模型选择、训练数据覆盖和最终验收的精简指标，见 [报告索引](reports/README.md)。 +- `reproduction/`：初始化权重训练链所需的冻结源码和哈希清单。 ## 本地启动 @@ -33,7 +41,7 @@ python -m venv .venv python boxvision.py scan --data DATA --threads 2 --output ./result screenshot-a.jpg screenshot-b.jpg ``` -输出 `observations.json`（每张截图的定位、来源指纹和字段）、`box.json`（合并结果）、带序号及来源指纹的标注图。同名输入文件不会覆盖彼此的标注图。成员与留影使用不同 ID 命名空间。 +输出 `observations.json`（每张截图的卡框、身份、未识别卡框、来源指纹和字段）、`box.json`（合并结果）、带序号及来源指纹的标注图。同名输入文件不会覆盖彼此的标注图。成员与留影使用不同 ID 命名空间。 ## 接入组卡推荐 @@ -47,13 +55,14 @@ python boxvision.py deck-ui --data DATA --deck-data deck-data.json --solver-bin ## 数据和模型 -运行目录需要使用者自行准备 `catalog.json`、`index.npz`、卡面／canonical 图库、卡阶精灵，再将权重包中的 `encoder.onnx` 和 `fields.onnx` 放入该目录的 `models/`。公开附件只包含源码与权重，不包含游戏资源或开箱即用的图库。缺少多模式字段模型时拒绝启动，避免旧的纯等级模型把能力值当等级。`models/gallery.npz` 是可重建的缓存。 +运行目录需要使用者自行准备 `catalog.json` 和卡面图，再把权重包中的 7 个 ONNX 文件和 `recognition.json` 放进该目录的 `models/`。`recognition.json` 记录每个模型的文件名、SHA-256、输入输出和接受阈值；启动时逐个校验哈希，缺文件或哈希不符都拒绝启动。`models/member-gallery.npz`、`models/snap-gallery.npz` 是可重建的图库缓存，指纹包含编码器哈希和每张参考卡面的字节。 -`scripts/prepare.py` 将显式提供的 master、UI、字体与卡面目录组装为数据目录；APK 导出与完整步骤见 [资源准备](docs/RESOURCES.md)。已有完整数据包的使用者无需重新提取。`boxvision.py prepare --data DATA` 从已准备的 master 表和卡面重建索引，`--offline` 可禁止补取素材；见 [新卡更新说明](docs/NEW-CARDS.md)。 +公开附件只包含源码与权重，不包含游戏资源或开箱即用的图库。`scripts/prepare.py` 将显式提供的 master、UI、字体与卡面目录组装为数据目录；APK 导出与完整步骤见 [资源准备](docs/RESOURCES.md)。已有数据目录时，`boxvision.py prepare --data DATA` 从 master 表和卡面重建目录和图库，`--offline` 可禁止补取素材。 ## 校对和冲突语义 - 等级视图只读取等级；特训视图只读取特训次数；能力／百分比视图不转换为等级。 +- `unidentified` 中的卡框不进入合并结果；它们可能是图库外的新卡，也可能是被切掉太多或画面太差的已知卡。 - 同一字段只有一个已观察到的值时合并该值；存在不同值时置为 `null` 并保留 `conflicts` 和全部来源，不取最大值。 - 重复截图不改变最终字段及来源证据；观察次数保留上传事实。 - 手工修正包含之前的值及修正记录。手工校对不能计入模型自动识别准确率。 @@ -61,9 +70,9 @@ python boxvision.py deck-ui --data DATA --deck-data deck-data.json --solver-bin ## 训练和验收 -训练以合成数据为主，使用原始卡面、UI 精灵、prefab 文本位置和客户端字体。简／繁中、日、英、韩分别渲染，并覆盖不同屏幕比例、分辨率、UI 缩放、JPEG/WebP、多次压缩、模糊和滚动裁切。真实截图用于校准和回归，不能充当独立泛化准确率。 +训练以合成数据为主：按列表组件的原生几何把原始卡面、UI 精灵和客户端字体渲染成整屏截图，覆盖简／繁中、日、英、韩五种语言、七种参数视图、不同屏幕比例、分辨率、UI 缩放、JPEG/WebP 多次压缩、模糊、滚动裁切、浮层遮挡和图库外卡面。真实截图只用于回归，不能充当独立泛化准确率。 -见 [训练与验证说明](docs/TRAINING.md)、[字体证据](docs/evidence/language-fonts.json)、[模型卡](docs/MODEL_CARD.md)、[数据卡](docs/DATA_CARD.md)、[覆盖边界](docs/COVERAGE.md) 和 [追加训练对照](docs/EXPERIMENTS.md)。当前版本已完成清晰全卡阶矩阵 2440/2440、修正后的 840 张独立整屏测试、真实回归和 2／4 逻辑 CPU 基准，详见 [评估卡](docs/EVALUATION.md)。压力场景仍会漏卡／拒识，不用接受值精度代替端到端覆盖率。 +见 [训练与验证说明](docs/TRAINING.md)、[字体证据](docs/evidence/language-fonts.json)、[模型卡](docs/MODEL_CARD.md)、[数据卡](docs/DATA_CARD.md) 和 [覆盖边界](docs/COVERAGE.md)。模型和阈值在开发集上冻结后，在两组各 2000 张、分辨率与压缩配置都不同于训练的整屏测试上评估：图库卡身份接受值零错误，召回 96.8%／97.0%；可见等级、特训次数和卡阶在图库卡上零误读，覆盖率分别约 96%、96% 和 99.6%。详见 [评估卡](docs/EVALUATION.md)。 ## 检查 @@ -71,6 +80,6 @@ python boxvision.py deck-ui --data DATA --deck-data deck-data.json --solver-bin python -m unittest discover -s tests -p "test_*.py" ``` -涵盖跨图合并、同名不同来源、冲突保存、隐藏字段及裁切图标等数据完整性回归。 +涵盖模型清单校验、跨图合并、同名不同来源、冲突保存、隐藏字段及裁切字段／图标等数据完整性回归。 可用 `python -m pip install -e ".[test]"` 安装开发入口和 Schema 校验依赖，再执行 `python scripts/validate_contract.py box result/box.json`。安装后提供 `ournotes-boxlens` 命令，参数与 `python boxvision.py` 相同。 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 075d873..151f3b4 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -4,10 +4,11 @@ Our Notes BoxLens's own code, documentation and released trained weights use the MIT license in LICENSE. This does not replace third-party terms or grant rights to game assets, data or trademarks. No game assets are distributed. -The card encoder starts from torchvision's MobileNetV3-Small ImageNet weights -and is fine-tuned as documented in MODEL_CARD.md. torchvision uses BSD-3-Clause; -its notice is retained in licenses/torchvision-BSD-3-Clause.txt. ImageNet is the -pretraining dataset, not a dataset distributed by this project. +The card encoders descend from torchvision's MobileNetV3-Small ImageNet weights +and the card-frame locator starts from torchvision's MobileNetV3-Large ImageNet +backbone; both are fine-tuned as documented in MODEL_CARD.md. torchvision uses +BSD-3-Clause; its notice is retained in licenses/torchvision-BSD-3-Clause.txt. +ImageNet is the pretraining dataset, not a dataset distributed by this project. The optional resource preparation process uses the separately installed nnnotes package at commit bb542af0d7e22a0255a7ee45099a51eb7fbadd12 (MIT), and UnityPy. diff --git a/docs/ASSET_BOUNDARY.md b/docs/ASSET_BOUNDARY.md index 1224958..e1eb696 100644 --- a/docs/ASSET_BOUNDARY.md +++ b/docs/ASSET_BOUNDARY.md @@ -2,7 +2,7 @@ 本项目自身的源码、文档和发布的训练权重采用 MIT；第三方许可和预训练来源见 THIRD_PARTY_NOTICES.md。MIT 不授予任何游戏资源或商标的权利。 -公开仓库及 GitHub Release 附件包括：识别和训练源码、合成／验证工具、JSON Schema、模型 ONNX／PyTorch 权重、训练初始化权重、资源文件哈希、统计报告及复现步骤。 +公开仓库及 GitHub Release 附件包括：识别和训练源码、合成／验证工具、JSON Schema、模型 ONNX／PyTorch 权重、模型清单、资源文件哈希、统计报告及复现步骤。 以下内容不随仓库、wheel、源码包或权重包分发： @@ -10,10 +10,10 @@ - APK、AssetBundle、完整 prefab 导出、masterdata 正文。 - 用户截图，以及含游戏画面的合成训练／验证截图或裁块。 - 原始反编译 C／IL2CPP 导出、私有解密配置、密钥、账号凭据。 -- 含游戏资源的内部一键运行包和图库／特征索引缓存。 +- 含游戏资源的一键运行包和图库向量缓存。 -使用者在本地自行准备相应资源，并按 RESOURCES.md 组装数据目录。发布权重不替代图库和原生 UI 资源；新卡接入仍需使用者更新自己的 masterdata／素材并重建索引。 +使用者在本地自行准备相应资源，并按 RESOURCES.md 组装数据目录。发布权重不替代图库和原生 UI 资源；新卡接入仍需使用者更新自己的 masterdata／素材并重建图库。 公开资源哈希用于确认实验输入，语言字体名称和接口事实用于说明兼容范围；它们不包含字体文件、资源正文或可重建游戏图像的编码。模型权重与直接分发训练图像是不同交付项；本项目不声称对训练使用的游戏美术或文字拥有版权。 -公开构建使用明确的文件清单，拒绝图片、字体、APK、资源目录、反编译代码和私有配置。构建后的 source／wheel／weights 附件均需再次检查文件清单，不复用研究阶段含素材的旧运行包。 +公开构建使用明确的文件清单，拒绝图片、字体、APK、资源目录、反编译代码和私有配置。构建后的 source／wheel／weights 附件均需再次检查文件清单。 diff --git a/docs/CONTRACT.md b/docs/CONTRACT.md index 440cf5c..0e67ca2 100644 --- a/docs/CONTRACT.md +++ b/docs/CONTRACT.md @@ -1,4 +1,4 @@ -# 输入输出契约 v1 +# 输入输出契约 Schema 位于 `schemas/`，使用 JSON Schema 2020-12。发布的顶层字段和含义发生不兼容变化时必须升级版本；不静默改变 `null`、卡片 ID 命名空间或冲突语义。 @@ -16,9 +16,9 @@ Schema 位于 `schemas/`，使用 JSON Schema 2020-12。发布的顶层字段和 } ``` -`data` 为 base64 编码图片，允许带 `data:image/...;base64,` 前缀，不接受下载 URL。`images` 为 1–30 个有序元素；`name` 可省略，提供时为 1–150 字符；`player` 可省略，最多 100 字符。兼容旧界面时允许省略 `schema`，其含义仍固定为 v1。未知字段、版本和类型返回 HTTP 400，不再静默转字符串或截断。 +`data` 为 base64 编码图片，允许带 `data:image/...;base64,` 前缀，不接受下载 URL。`images` 为 1–30 个有序元素；`name` 可省略，提供时为 1–150 字符；`player` 可省略，最多 100 字符。允许省略 `schema`，其含义仍固定为请求 v1。未知字段、版本和类型返回 HTTP 400，不静默转字符串或截断。 -请求体必须小于 48 MiB；单图解码后最多 2000 万像素。失败响应符合 `error.schema.json`：`{"error":"说明"}`。成功响应包含 `schema: ournotes-boxlens.scan-result/1`、合并 `box` 和与输入顺序一致的 `scans`，符合 `scan-response.schema.json`。 +请求体必须小于 48 MiB；单图解码后最多 2000 万像素。失败响应符合 `error.schema.json`：`{"error":"说明"}`。成功响应包含 `schema: ournotes-boxlens.scan-result/2`、合并 `box` 和与输入顺序一致的 `scans`，符合 `scan-response.schema.json`；`scans` 的每个元素与 `observations.json` 中的单张截图结构相同。 ## CLI 和 GUI 导出 @@ -27,20 +27,27 @@ Schema 位于 `schemas/`，使用 JSON Schema 2020-12。发布的顶层字段和 | 文件 | 契约 | |---|---| | `box.json` | `box.schema.json`；GUI 导出的 JSON 也使用同一契约 | -| `observations.json` | `observations.schema.json`，按输入顺序排列 | +| `observations.json` | `observations.schema.json`（`urn:ournotes-boxlens:schema:observations:2`），按输入顺序排列 | | `NNN-HASH-annotated.jpg` | 标注图；相对文件名记录在对应 scan 的 `annotated_file` | -已存在的合并数据版本标识 `bdon-box/1` 继续保留，仓库更名不改变其语义。`observations` 计上传观察次数，`unique_count` 为唯一卡数，`duplicate_observations` 为两者之差。重复上传不改变合并字段、冲突和来源证据，但观察计数反映上传事实。 +合并数据版本标识为 `bdon-box/1`。`observations` 计上传观察次数，`unique_count` 为唯一卡数，`duplicate_observations` 为两者之差。重复上传不改变合并字段、冲突和来源证据，但观察计数反映上传事实。 + +## 单张截图 + +- `cards`：已确认身份的卡。每张卡以 `(kind,id)` 标识，`kind` 为 `member` 或 `snap`。`id` 是 master ID；`name`、`rarity`、`card_type` 取自当前图库，不用于身份匹配。 +- `unidentified`：定位到但身份未被接受的卡框，包括图库外的新卡、被切掉或遮挡过多的卡和画面太差的卡。`candidate` 为最相近的图库卡 `{id,name}` 或 `null`，只供人工确认，不是识别结果；`review` 恒为 `true`。这些卡框不进入合并 `box`。 +- 两个列表的元素都含 `bbox`、`locator_score`、`visible_fraction`、`identity_similarity`、`identity_margin`、`level`、`card_rank`、`awake_count`、`display_mode` 和 `review`。`identity_similarity` 是与最相近图库卡的余弦相似度，`identity_margin` 是它与第二名的差值；二者和 `locator_score` 都是内部得分，不是校准概率。 +- `cards` 中的 `review` 在等级或卡阶未知时为 `true`。 +- 列表按卡框所在行、再按横坐标排序。 ## 字段和证据 -- 每张卡以 `(kind,id)` 标识，`kind` 为 `member` 或 `snap`。`id` 是 master ID；`name` 是当前图库展示名，不用于身份匹配。 -- `level`、`card_rank`、`awake_count` 均为 `{value,confidence,reason?}`。`value: null` 表示未知；卡阶／特训范围为 1–5，等级还要遵守目录中的培养上限。Snap 的 `awake_count` 保持未知。 +- `level`、`card_rank`、`awake_count` 均为 `{value,confidence,reason?}`。`value: null` 表示未知，`reason` 说明原因（如 `cropped`、`cropped_rank_icon`、`not_visible_in_level_view`、`outside_masterdata_level_limit`）。卡阶／特训范围为 1–5，等级还要遵守目录中的培养上限。Snap 的 `awake_count` 保持未知。 - `display_mode` 是当前参数裁块的识别类别：`level`、`training`、`other` 或 `unknown`。`other` 包含能力值、百分比或隐藏参数，不能据此补全培养值。 -- bbox 是解码图上的浮点像素 `[x,y,width,height]`；原点左上，x 向右、y 向下。部分可见卡的 bbox 可超出图像。 +- bbox 是解码图上的浮点像素 `[x,y,width,height]`，表示整张卡框；原点左上，x 向右、y 向下。部分可见卡的 bbox 可超出图像，`visible_fraction` 为图像内面积占比。 - `source_id` 对解码后的 BGR 像素和形状计算 SHA-256；压缩文件哈希和像素哈希不可混称。`source`／`sources` 是展示名称，不能单独作为唯一标识。 - 冲突对象的键是十进制字符串值，对应全部来源证据。冲突字段最终值为 `null`，不取最大值。 - 手工修正的字段增加 `source: manual`，`corrections` 记录原值、原冲突和新值；原始 scan 观察不被修改。 -- 可选的 `recognition_provenance` 记录识别时的模型、图库／索引及识别代码哈希、阈值和 ONNX Runtime 版本；合并 box 保留所有不同来源版本，便于追溯。旧导出没有此字段时仍符合契约，不能为旧数据补造来源记录。 +- `recognition_provenance`（`ournotes-boxlens.recognition-provenance/2`）记录识别时 `catalog.json` 与 `models/recognition.json` 的哈希、两个图库缓存的指纹、识别代码哈希和 ONNX Runtime 版本；合并 box 保留所有不同来源版本，便于追溯。没有此字段的数据不能补造来源记录。 Schema 检查类型与结构；跨字段约束还需数据审计，包括唯一键、计数关系、合法培养范围及冲突值与最终值一致性。模型指标不能包含手工修正结果。 diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md index 3fd02ad..160a41b 100644 --- a/docs/COVERAGE.md +++ b/docs/COVERAGE.md @@ -1,37 +1,38 @@ # 卡片与截图覆盖边界 -核对对象是本项目保存的 1.0.1-25 客户端资源／masterdata 快照，不代表所有区服今后新增卡片。统计来自实际整屏 truth 文件，工具为 `scripts/audit_coverage.py`。出现过某类样本，不等于该类识别准确率已达到要求。 +核对对象是本项目使用的 129 张卡的 master 快照与 1.0.1-25 客户端 UI 资源，不代表所有区服今后新增卡片。统计来自实际整屏 truth 文件，见 `reports/training/scene-coverage.json`。出现过某类样本，不等于该类识别准确率已达到要求。 -## 稀有度与属性 +## 稀有度 -经哈希校验的 `App.Master.CardRarity` 枚举：R=2、SR=3、SSR=4、EX=10、BD=20；`CardType` 为 Red/Blue/Green/Yellow/Purple=1/2/3/4/5。 +`App.Master.CardRarity` 枚举：R=2、SR=3、SSR=4、EX=10、BD=20。 -| 类型 | 稀有度 | 图库身份 | 每组整屏训练出现的身份 | 每组开发验证出现的身份 | +| 类型 | 稀有度 | 图库身份 | 每组整屏训练出现的身份 | 每组开发／最终测试出现的身份 | |---|---|---:|---:|---:| | 成员 | R | 25 | 22 | 25 | -| 成员 | SR | 25 | 21 | 25 | -| 成员 | SSR | 10 | 9 | 10 | +| 成员 | SR | 26 | 21 | 26 | +| 成员 | SSR | 12 | 11 | 12 | +| 成员 | BD | 1 | 1 | 1 | | 留影 | R | 25 | 22 | 25 | -| 留影 | SR | 25 | 21 | 25 | -| 留影 | SSR | 10 | 9 | 10 | +| 留影 | SR | 26 | 22 | 26 | +| 留影 | SSR | 12 | 10 | 12 | | 留影 | EX | 2 | 1 | 2 | -两组训练为 scale-v6/train（2048 屏）及 modes-v7/train（350 屏）；四组开发验证为这两个系列的 validation、stress。每组训练都实际出现 105 个身份；每组开发验证都实际出现全部 122 个身份。17 个身份按 `id % 7 == 0` 留给身份编码器训练外验证。字段预训练使用过全图库背景，因此不能称为所有网络都未见过这些像素。 - -成员五种属性各有 12 张；留影 Red/Blue/Green/Yellow/Purple 分别为 14/12/12/11/13 张。两组训练及四组开发验证均覆盖五种属性。 +两组整屏训练各出现 110 个身份；所有开发与最终测试组都出现全部 129 个身份。19 个身份按 `id % 7 == 0` 留给身份编码器训练外验证。 ## “风格全覆盖”的边界 -- 现有整屏训练覆盖成员竖向列表、留影横向列表及其原生卡框／属性标志；成员七种参数显示模式，留影使用适用的五种模式；含选中状态。 -- 当前快照没有 BD 身份，不能声称 BD 已训练或验证。枚举支持某个值，不等于拥有对应素材与样本。 -- 编队画面不是当前整屏识别验收范围。渲染器的成员编队分支明确只备有 SSR 资源；不能把列表中的 R/SR 覆盖延伸成编队全覆盖。 +- 整屏数据覆盖成员竖向列表、留影横向列表及其原生卡框／属性标志；成员七种参数显示模式，留影使用适用的五种模式；含选中状态、滚动裁切、界面浮层和图库外卡面。 +- BD 只有 1 个成员身份，EX 只有 2 个留影身份，样本量很小，不能据此宣称这两种稀有度的准确率。 +- 编队画面不是整屏识别验收范围。 - 事件角标、所有动画／皮肤／特殊卡面变体没有完整的系统覆盖证明。未建立独立“特训前后所有替换卡面”的覆盖表，不作此承诺。 - 五语言字体规则来自已提取客户端；不是任意未来区服版本的字体认证。 -## 真实图遮挡与指标口径 +## 遮挡与部分出屏 + +整屏训练包含部分出屏（约 14%）和界面浮层遮挡（约 5%）的卡，采集降质裁块中 8% 带浮层。识别策略是保守的：卡面窗口不完整或被遮挡时身份多半进入 `unidentified`，参数区或卡阶图标不完整位于截图内时字段输出未知。最终测试中这两类卡没有身份错误，但身份召回只有约 75%–77%，见 [EVALUATION.md](EVALUATION.md)。 -原始成员等级截图底部一排（成员 19–24）存在视口渐隐及边缘遮挡。卡面身份仍可见，等级／卡阶值保留未知；不通过其他截图或已知培养状态替代当前图的可见真值。 +半透明渐隐、浮层遮挡仍可能造成字段误读，不可据此宣称任意遮挡都能安全拒识。请将卡片滚动到可见区域后重截，并检查或手工校对结果。 -前三排的等级仍保留人工可读标签，包括成员 15。v13 对该格拒识，不能因整张图存在局部遮挡而删除这一可读样本。十张真实回归图共 156 次卡片观察：身份 156/156，等级 20/21（1 次拒识），可见卡阶 120/120，特训 18/18，隐藏字段输出 0。它们来自同一批卡片和设备，是校准回归，不是独立真实泛化准确率。 +## 真实图口径 -现有整屏合成覆盖滚动裁切、尺寸／重采样、多轮压缩与模糊；半透明渐隐和浮层遮挡没有纳入发布训练保障。420 张诊断图暴露渐隐情况下的卡阶误读；不可据此宣称任意遮挡都能安全拒识。发布范围以清晰列表截图为准，不追加遮挡训练或遮挡专项推理规则。请将卡片滚动到可见区域后重截，并检查或手工校对结果。 +8 张真实回归图来自同一批卡片和设备，覆盖成员与留影列表、等级与特训视图，共 129 次卡片观察。局部被遮挡的字段保留未知，不通过其他截图或已知培养状态替代当前图的可见真值。它们是回归样本，不是独立真实泛化准确率。 diff --git a/docs/DATA_CARD.md b/docs/DATA_CARD.md index 17d4cf5..2e2ba89 100644 --- a/docs/DATA_CARD.md +++ b/docs/DATA_CARD.md @@ -1,49 +1,60 @@ # Our Notes BoxLens 数据卡 -训练与主要评估使用合成数据。真实图只用于校准和回归，未作为可泛化准确率的独立测试集。用户截图不自动加入公开仓库或训练数据发布包。 +训练与主要评估使用合成数据。真实图只用于回归，未作为可泛化准确率的独立测试集。用户截图不自动加入公开仓库或训练数据发布包。 -稀有度／属性的实际训练与验证数量、未覆盖 BD／编队／卡面变体，以及真实图局部遮挡口径，见 [COVERAGE.md](COVERAGE.md)。不能将当前资源快照的身份覆盖表述为所有游戏 UI 风格全覆盖。 +稀有度覆盖、未覆盖的编队画面与卡面变体，以及遮挡口径见 [COVERAGE.md](COVERAGE.md)。不能将当前资源快照的身份覆盖表述为所有游戏 UI 风格全覆盖。 ## 资源依据 -游戏 APK 1.0.1-25 的 prefab、UI 精灵、语言字体规则及文本材质；122 张成员／留影图库及对应 masterdata。使用成员 square 图按列表的竖向窗口等比填充裁切。五语言字体分别来自客户端规则，细节见 RESOURCES.md 和证据文件。 +游戏 APK 1.0.1-25 的 prefab、UI 精灵、语言字体规则及文本材质；129 张成员／留影卡的 master 快照与对应卡面（目录 SHA-256 `c6995ccac52fa98438eeca3cd82e375e6e417e64b5b0f3e91ecacfb5fee8647e`）。成员卡面使用 square 图按列表的竖向窗口等比填充裁切。五语言字体分别来自客户端规则，细节见 RESOURCES.md 和证据文件。 合成培养状态遵守 masterdata 的等级、觉醒、卡阶范围。显示模式区分 level、training、total、performance、technic、visual、hide；不可见字段真值为未知，潜在培养状态另存用于生成审计。 -## 数据规模和分割 +## 整屏数据 -| 数据组 | 用途 | 整屏数 | seed | -|---|---|---:|---:| -| scale-v6/train | 训练，等级视图，简中默认 | 2048 | 50021 | -| modes-v7/train | 训练，五语言、多模式 | 350 | 91703 | -| scale-v6/validation | 开发验证 | 512 | 60017 | -| modes-v7/validation | 开发验证，多语言多模式 | 140 | 92711 | -| scale-v6/stress | 开发压力测试 | 256 | 70019 | -| modes-v7/stress | 开发压力测试，多语言多模式 | 70 | 93713 | -| final-v9/validation | 阈值冻结后的新整屏测试 | 560 | 908271 | -| final-v9/stress | 阈值冻结后的新压力测试 | 280 | 708289 | +| 数据组 | 用途 | 整屏数 | 图库卡 | 图库外卡面 | 无卡负样本 | seed | +|---|---|---:|---:|---:|---:|---:| +| scenes-v2 train | 训练 | 16000 | 187844 | 26722 | 802 | 20261101 | +| scenes-v2.1 train | 训练 | 16000 | 188200 | 27306 | 808 | 20261111 | +| scenes-v2 validation | 开发验证 | 2000 | 24326 | 3419 | 98 | 20261102 | +| scenes-v2.1 validation | 开发验证 | 2000 | 24031 | 3642 | 106 | 20261112 | +| scenes-v2 stress | 开发压力集 | 1000 | 11239 | 1663 | 58 | 20261103 | +| scenes-v2.1 stress | 开发压力集 | 1000 | 11831 | 1656 | 49 | 20261113 | +| scenes-v2 final | 冻结后最终测试 | 2000 | 24405 | 3493 | 106 | 20261104 | +| scenes-v2.1 final | 冻结后最终测试 | 2000 | 24604 | 3725 | 99 | 20261114 | -整屏合计 4216 张。训练 2398 张产生 37310 个完整裁块；开发普通验证 652 张产生 10278 个完整裁块。表中开发集曾用于诊断和阈值选择，不再称为最终未见测试。最后两组虽沿用数据目录名 final-v9，正式验收使用 v10 阈值。 +四档的屏幕原始分辨率互不相同：train 为 1280×720、1440×1080、1600×900、1920×1080、2160×1080；validation 为 1366×768、1920×1200、2048×1536、2340×1080、2560×1440；stress 为 960×540、2280×1080、2532×1170、2800×1752、3840×2160；final 为 1334×750、1792×828、2224×1668、2388×1668、2400×1080。压缩质量离散值和其他采集参数也按档区分。五种语言和七种显示模式在每组中大致均匀。 -此前还有 202000／30300 个等级预训练／验证裁块，以及 106000／21200 个多语言原生参数预训练／验证裁块。它们是不同阶段的字段数据，不能把数量简单累加后称为独立整屏截图数量。字段识别预训练覆盖 1–100 的数字字形；完整截图才检查每张卡的合法培养状态。 +训练集中约 14% 的卡部分出屏、约 5% 被界面浮层遮挡。图库外卡面使用不属于目录的美术图，只出现在 `foreign_cards` 中，用于训练和检验未知卡拒识。 -实际保存的整屏训练真值含 32938 个可见等级：Lv.1 为 643 个，Lv.2–20 为 11833 个，Lv.21–40 为 11416 个，Lv.41–60 为 6659 个，Lv.61–90 为 2387 个，合计覆盖 1–90 每一级。困难字段训练的 63600 个等级裁块覆盖 1–100，每级 576–709 个；其余字段裁块为其他／隐藏参数或特训类别。逐级计数见 `reports/training/level-distribution.json`。 +此外，定位器训练混入 640 张 `bdon_vision.synthetic` 生成的单类卡片截图（16 组，种子 300000–315000）。 -清晰卡阶专项矩阵把等级固定为 Lv.1，是为独立检查 1–5 阶图标；该矩阵不是训练集，其等级成绩不能外推为其他等级的准确率。 +## 裁块数据 -整屏训练与身份编码器的 ID 排除规则为 `id % 7 == 0`，成员／留影分别处理；122 个身份中保留 17 个编码器训练外身份。字段预训练曾使用全图库底部图像作为随机文字背景，因此这里的“未见身份”严格指身份编码器，不能扩展成所有网络都未见过任何相关像素。普通／压力验证的屏幕分辨率列表和离散压缩质量值与训练不同，同时使用独立 seed。图片有多次 JPEG/WebP 压缩、缩放、模糊、UI 尺寸和滚动裁切。 +| 裁块库 | 用途 | 数量（每类卡） | seed（成员／留影） | +|---|---|---:|---| +| 场景参数／卡阶裁块 | 参数与卡阶训练、开发评估 | 取自 scenes-v2.1 各档 | — | +| hard-g | 参数训练；留影卡阶训练；8% 带浮层遮挡 | 60000 | 190001／190003 | +| hard-n | 成员卡阶训练；无遮挡 | 60000 | 180001／180003 | +| hard-l | 开发验证 | 16000 | 170001／170003 | +| 场景卡框裁块 | 编码器训练与开发评估 | 取自 scenes-v2.1 各档 | — | -## 关联性和泄漏审计 +采集降质裁块的生成方法见 [TRAINING.md](TRAINING.md)。它们与整屏数据是不同粒度的样本，不能把数量简单累加后称为独立截图数量。 -同一玩家状态可出现在相邻截图，用于多图合并测试，因此卡片观察不是相互独立的真实用户样本。数据切分以生成配置和数据组为边界；真实统计外推不能直接使用合成样本数。 +## 字段分布 -8 组数据已完成文件哈希、重复图、训练 ID 排除、培养合法性和可见标签一致性审计，未发现错误。旧 v6/v7 真值文件没有逐图 SHA-256，审计读取实际图片重新计算，并明确记录此情况；v8 裁块构建也将全部实际图片字节纳入源数据指纹。新数据的真值直接包含逐图哈希。 +两组整屏训练真值中可见等级共 59040 个：Lv.1 为 1061 个，Lv.2–20 为 21111 个，Lv.21–40 为 20593 个，Lv.41–60 为 12023 个，Lv.61–80 为 3963 个，Lv.81–90 为 289 个；1–90 每一级都出现。卡阶 1–5 在真值中各约 56600–57300 次，特训次数 1–5 各约 4600–4800 次。逐组计数见 `reports/training/scene-coverage.json`。 + +## 身份分割与泄漏审计 + +整屏训练与身份编码器的 ID 排除规则为 `id % 7 == 0`，成员／留影分别处理；129 个身份中保留 19 个编码器训练外身份，它们出现在所有开发和最终测试组中。参数与卡阶模型的初始化权重曾在全图库卡面上训练，采集降质裁块也使用全图库卡块，因此这里的“训练外身份”严格指身份编码器，不能扩展成所有网络都未见过相关像素。 + +同一玩家状态可出现在相邻截图，用于多图合并测试，因此卡片观察不是相互独立的真实用户样本。数据切分以生成配置和数据组为边界；真实统计外推不能直接使用合成样本数。每张合成图的真值都包含逐图 SHA-256，`bdon_vision.scenes verify` 检查哈希、完整性和真值格式。 ## 真实性限制 -- 背景为生成的模糊色块，不是完整游戏场景模拟；卡片 UI 使用原生资源。 +- 背景为卡面模糊、噪声和渐变，或使用者提供的场景图；卡片 UI 使用原生资源。 - SDF 栅格化和抗锯齿为近似，未宣称所有设备像素一致。 -- 能力／百分比视图中的 power 值用于干扰项，目前没有完整复现 native 战力计算；不将这些值列为目标输出。 -- 原训练与现行多语言渲染器有版本演进；复现具体 checkpoint 要使用保存的阶段源码与资源清单，不应用最新生成器冒充原始逐字节数据。 -- 历史整屏合成在拼接时按卡框裁切，截掉了部分外伸属性／卡阶图标。现行生成器改为保留原生透明边距，完整卡阶矩阵使用修正后的拼接。历史训练和测试报告保留其原始输入条件，不改写成完整图标测试；本次未因此重训模型。 -- 真实回归图共 10 张，覆盖同一批 24 成员／3 留影及不同参数视图，来源、玩家、设备多样性有限。 +- 能力／百分比视图中的 power 值用于干扰项，没有完整复现 native 战力计算；不将这些值列为目标输出。 +- scenes-v2 数据由上一生成器版本产生，只能以真值哈希核对。 +- 真实回归图共 8 张，来自同一玩家和设备，来源、玩家、设备多样性有限。 diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 29ec16a..f10d58f 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -1,48 +1,58 @@ # 验收与指标 -公开版本使用已选定的训练权重，修正留影卡面的显示窗口坐标，并修正合成拼接以保留透明外边距；没有为此重训，也没有把遮挡实验加入发布推理策略。模型、数据和许可边界见 [模型卡](MODEL_CARD.md)、[数据卡](DATA_CARD.md) 与 [资源边界](ASSET_BOUNDARY.md)。可复核的精简指标在 `reports/`，不含截图或逐图调试产物。 +模型、数据和许可边界见 [模型卡](MODEL_CARD.md)、[数据卡](DATA_CARD.md) 与 [资源边界](ASSET_BOUNDARY.md)。可复核的精简指标在 `reports/`，不含截图或逐图调试产物。 -## 清晰卡阶矩阵 +## 评估方法 -使用 122 个身份，覆盖 1–5 阶和四种屏幕分辨率（1280×720、1440×1080、1920×1080、2560×1440）；共 220 张清晰完整截图、2440 次卡片观察。身份与卡阶均为 2440/2440，零错读、零拒识和零多报；每个卡阶均为 488/488。分语言、稀有度和分辨率计数见 `reports/acceptance/clear-rank-matrix.json`。这项专项测试把等级固定为 Lv.1，因此不代表不同等级的准确率。 +`python -m bdon_vision.evaluate_engine --data DATA --set NAME=DIR --output OUT.json` 对 `Engine.scan` 做端到端评估，只依赖扫描输出契约： -卡阶识别采用精灵模板匹配，不是独立的神经网络卡阶分类器。场景训练样本包含 1–5 阶；逐阶训练数量和本矩阵的测试结果分别见 `reports/training/rank-distribution.json` 与上面的专项报告。 +- 每张截图的真值卡（图库卡 `cards` 和图库外卡面 `foreign_cards`）与输出卡框（`cards` 和 `unidentified` 都参与）按 IoU≥0.5 一对一配对；未配对、也不与忽略区域重叠的输出计为多余检测。 +- 图库卡的身份分为正确、错误和未识别（进入 `unidentified`）；图库外卡面必须进入 `unidentified`，被赋予任何 ID 都算误接受。 +- 等级、特训次数和卡阶：列表显示该字段、且读取区域（参数框或整个卡阶图标）完整位于截图内时为可见目标，分为正确、错误和未知；对不可见目标输出值时，与卡片真实状态不同即计为误读。 +- 结果按卡类、语言、参数视图、屏幕原始分辨率、卡片宽度、图库外卡面、训练外身份、遮挡和部分出屏分层。 -## 独立整屏合成测试 +## 最终整屏测试 -权重和阈值冻结后评估 560 张普通截图、280 张压力截图。测试包含五种语言、多个列表参数视图、合法随机等级、独立分辨率与压缩配置。卡片身份和定位框 IoU≥0.5 才算识别正确；压力裁切可能导致卡片或字段拒识／未定位。 +模型和阈值在开发集冻结后，评估两组未参与任何选择的整屏测试：`v2.1-final` 与 `v2-final`，各 2000 张，五种语言、七种参数视图，屏幕原始分辨率和压缩配置都与训练、开发集不同。两组分别由两个生成器版本产生，真值哈希见报告。 -| 项目 | 普通：560 屏 | 压力：280 屏 | +| 项目 | v2.1-final | v2-final | |---|---:|---:| -| 身份正确／目标卡片 | 9673/9779（98.92%） | 3956/4531（87.31%） | -| 等级正确／可见目标 | 1518/1522（99.74%） | 597/688（86.77%） | -| 卡阶正确／可见目标 | 7249/7407（97.87%） | 2391/3281（72.87%） | -| 特训次数正确／可见目标 | 637/637（100%） | 238/279（85.30%） | -| 错误字段输出 | 0 | 0 | -| 隐藏字段输出 | 0 | 0 | -| 计分区域内多报 | 0 | 0 | +| 计分卡片（图库卡＋图库外卡面） | 28329 | 27898 | +| 定位到的卡片 | 28308（99.93%） | 27875（99.92%） | +| 图库卡身份正确 | 23817/24604（96.80%） | 23677/24405（97.02%） | +| 图库卡身份错误 | 0 | 0 | +| 图库卡进入 `unidentified` | 774 | 708 | +| 训练外身份正确 | 3353/3649（91.89%），错误 0 | 3321/3573（92.95%），错误 0 | +| 图库外卡面误接受 | 49/3725（1.32%） | 32/3493（0.92%） | +| 多余检测（带 ID／未识别） | 0／6 | 0／2 | -其余字段目标因拒识或未定位而未读出：普通截图中等级 3 次拒识、1 次未定位，卡阶 124 次拒识、34 次未定位；压力截图中等级 50 次拒识、41 次未定位，卡阶 481 次拒识、409 次未定位，特训 1 次拒识、40 次未定位。零误读只描述这些合成测试中已输出的值，不等于零漏读或对任意输入都正确。逐语言、分辨率、压缩和参数视图统计见 `reports/acceptance/synthetic-full-screen.json`。 +图库卡上的可见字段： -测试数据清单覆盖十组、共 5056 张生成截图。文件哈希、完全重复、身份隔离、培养合法性和可见标签语义审计均通过；审计不证明合成图与真实客户端逐像素一致。摘要见 `reports/acceptance/synthetic-data-audit.json`。 +| 字段 | v2.1-final 正确／可见 | 误读 | v2-final 正确／可见 | 误读 | +|---|---:|---:|---:|---:| +| 等级 | 4008/4181（95.86%） | 0 | 3946/4100（96.24%） | 0 | +| 特训次数 | 1497/1560（95.96%） | 0 | 1718/1789（96.03%） | 0 | +| 卡阶 | 18736/18817（99.57%） | 0 | 18817/18895（99.59%） | 0 | -## 未知身份与真实截图回归 +图库外卡面也会读取字段：两组合计等级正确 1158/1288、卡阶 5500/5555、特训次数 444/489，唯一一次误读是 `v2.1-final` 中一张图库外卡面的等级 6 被读成 61。对不可见目标输出的值（等级 14 次、卡阶 4 次、特训次数 2 次）全部与卡片真实状态一致。 -未知身份开发检查从图库移除 17 个身份后，在 1258 次未知卡片观察中未误报为已知卡；7629 次已知卡观察中正确识别 7579 次。另有 8 张纯色／模糊背景没有卡片输出。原检查还记录 235 个未匹配到计分真值的预测（含需要另行分类的裁切区域），因此不声称对任意未知图像或背景都零误报。计数见 `reports/acceptance/unknown-card-results.json`。 +### 分层结果 -十张真实截图只用于校准和回归，来自同一用户的一批 24 张成员卡和 3 张留影卡，不是独立真实用户测试：身份 156/156、可读等级 20/21、卡阶 120/120、特训 18/18；一个等级被拒识，没有错误字段或隐藏字段输出。局部被遮挡的字段保留未知。汇总见 `reports/acceptance/real-sample-regression.json`；原始图片及含本机路径的报告不随公开包发布。 +- 部分出屏的卡（约占 12%–13%）身份召回约 77%，其余多数进入 `unidentified`；被浮层遮挡的卡身份召回约 75%。这两类都没有身份错误。 +- 卡片宽度 60–119 px 的小卡身份召回约 96%，等级覆盖率 89%–93%，低于大卡。 +- 英文界面的特训次数覆盖率较低（81%–88%），其他语言为 96%–99%。 +- 逐语言、分辨率、参数视图和卡片宽度统计见 `reports/acceptance/holdout-full-screen.json`。 -渐隐、浮层或大面积遮挡可能让字段变成未知，也可能造成误读。此次只发布清晰完整画面上验证的策略；遇到遮挡请调整位置后重截，并检查或手工校对，不依赖模型还原被遮住的值。 +零误读只描述这些测试中已输出的值，不等于对任意输入都正确。 -## CPU 性能 +## 真实截图回归 -Windows 10、Intel Core i5-12400F（主机 12 逻辑 CPU），扫描 10 张真实截图；每张热身一次，再分别测 5 次，共 50 次／配置。计时含定位和字段读取，不含图片解码及 HTTP。 +8 张真实截图来自同一玩家、同一设备的一批成员与留影列表：身份 129/129（成员 120、留影 9），可见特训次数 18/18，可见卡阶 99/99，没有错误字段、多余检测或未识别卡框。这批截图中没有完整位于画面内的可见等级。它们只用于回归，不是独立真实用户测试；汇总见 `reports/acceptance/real-sample-regression.json`，原始图片不随公开包发布。 -| 指标 | 2 逻辑 CPU | 4 逻辑 CPU | -|---|---:|---:| -| 扫描中位数 | 1821.57 ms | 1661.27 ms | -| P95 | 2427.30 ms | 2900.11 ms | -| 最大值 | 2528.20 ms | 6893.82 ms | -| 进程内存 | 约 628 MB | 约 632 MB | +## 新卡接入 + +`scripts/verify_gallery_update.py` 在独立目录里从图库移除指定身份，确认这些卡进入 `unidentified`；再加回其目录条目和卡面，确认重新识别，并检查模型文件哈希不变。它是集成检查，不是独立准确率；使用方法见 [NEW-CARDS.md](NEW-CARDS.md)。 + +## 使用边界 -保留离群值，测试没有隔离主机其他负载，不据此推断 4 核一定更快。逐次结果、CPU 限制和代码指纹见 `reports/performance/`。 +渐隐、浮层或大面积遮挡可能让身份或字段变成未知，也可能造成误读。遇到遮挡请调整位置后重截，并检查或手工校对，不依赖模型还原被遮住的值。 diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md deleted file mode 100644 index 6b7d2ac..0000000 --- a/docs/EXPERIMENTS.md +++ /dev/null @@ -1,25 +0,0 @@ -# 训练收敛与数据改进 - -## 同一批数据上的追加训练 - -编码器和字段模型分别从原 checkpoint 用相同训练数据追加 2400 steps；优化器状态重新初始化为 AdamW，因此不把前后 loss 当作同一条连续曲线。 - -| 指标 | 初始模型 | 追加训练后 | -|---|---:|---:| -| 字段模型：普通裁块 top-1 | 10278/10278 | 10278/10278 | -| 字段模型：困难裁块高置信且正确 | 2836/4058 | 2873/4058 | -| 字段模型：困难裁块错误 | 1 | 1 | -| 字段模型：困难裁块交叉熵 | 2.53904 | 2.68795 | -| 编码器：困难裁块 top-1 | 4651/4766 | 4706/4766 | -| 编码器：高置信检索错误 | 10 | 1 | -| 编码器：训练外身份困难裁块 top-1 | 642/655 | 642/655 | - -字段困难集的 loss 在追加 600 steps 时最低，之后上升；正确高置信覆盖率仅从 69.89% 升至 70.80%。这表明只对同一批数据继续拟合，字段模型收益有限。 - -## 更贴近屏幕采集的合成数据 - -旧的困难裁块路径在放大后才压缩，与真实流程中截图先缩小／压缩、再被识别器放大的顺序不同。修正数据采集顺序后，生成 106000 个训练裁块和 21200 个独立验证裁块：先在低分辨率施加模糊及多轮 JPEG/WebP 压缩、改变编码块相位，再放大到模型输入尺寸。训练集和验证集使用不同采集配置；训练混入整屏裁块回放。 - -从同一初始字段权重训练 8000 steps，每 500 steps 用普通与压力开发集选模。第 5500 步普通可见字段正确 8671 个、错误 0 个；压力字段正确 3245 个、错误 1 个。第 8000 步压力正确数升至 3281，但误读增至 4 次、普通集正确数降至 8659，因此没有采用最后一步。按事先确定的选择规则，发布第 5500 步权重。 - -冻结候选后，在独立困难字段验证集的 16960 个数值目标中，初始模型正确 12036 个、误读 3 个；候选正确 15148 个、误读 4 个，覆盖率从 70.97% 升到 89.32%。两个模型在其他／隐藏参数负例上都没有输出数值。这是字段裁块结果，不代表整屏准确率；整屏独立测试见 [EVALUATION.md](EVALUATION.md)。开发曲线和裁块验证摘要见 `reports/training/`。 diff --git a/docs/MODEL_CARD.md b/docs/MODEL_CARD.md index e963a08..e3c9d34 100644 --- a/docs/MODEL_CARD.md +++ b/docs/MODEL_CARD.md @@ -1,57 +1,68 @@ # Our Notes BoxLens 模型卡 -v0.1.0 使用冻结的编码器和字段模型权重，并包含留影显示窗口坐标修正；权重选择分别为编码器追加微调第 2400 步、字段困难采集训练第 5500 步，字段阈值保持 0.995。修正后另行完成 840 张独立整屏最终测试。目标输入是清晰列表截图；遮挡可能误读或拒识，不追加遮挡训练或遮挡专项推理规则。完整结果见 EVALUATION.md，公开内容边界见 ASSET_BOUNDARY.md。 +v0.2.0 由 7 个 ONNX 模型组成：一个卡框定位器，以及成员、留影各一套卡面编码器、参数读取模型和卡阶图标模型。所有检查点和阈值只依据开发集选择，冻结后才在最终测试集上评估。目标输入是成员／留影列表截图；完整结果见 [EVALUATION.md](EVALUATION.md)，公开内容边界见 [ASSET_BOUNDARY.md](ASSET_BOUNDARY.md)。 ## 用途和输出 -成员／留影列表截图导入：定位卡片、检索 master ID，读取可见等级、卡阶与成员特训次数。不可见或不确定字段输出未知，跨图分歧保留冲突。模型不判断玩家未展示的收藏，不从卡面推测培养值。 +成员／留影列表截图导入：定位卡框、在本地图库中检索 master ID，读取可见等级、卡阶与成员特训次数。不可见或不确定的字段输出未知；无法确认身份的卡框单独列出，不进入合并结果；跨图分歧保留冲突。模型不判断玩家未展示的收藏，不从卡面推测培养值。 -## 两个神经网络 +## 模型 + +所有输入均为 float32 RGB、NCHW、取值 0–1，ONNX opset 17，batch 维可变（定位器 batch 为 1）。模型内部执行所需的均值／方差归一化。 | 文件 | 输入 | 输出 | 架构 | |---|---|---|---| -| `encoder.onnx`／`encoder.pt` | float32 RGB、NCHW、`N×3×160×128`、0–1 | L2 归一化的 128 维向量 | MobileNetV3-Small features、固定池化和线性投影；内部执行 ImageNet mean/std 归一化 | -| `fields.onnx`／`fields.pt` | float32 RGB、NCHW、`N×3×56×144`、0–1 | 106 个未归一化 logits | 24/48/64/96 通道卷积、池化、线性分类头 | - -字段类别：0 为其他／隐藏参数；1–100 为等级；101–105 为特训次数 1–5。ONNX opset 17，batch 维可变。字段 softmax 接收阈值为 0.995、第一二候选差值至少 0.5。得分不是经过校准的概率。 +| `locator.onnx` | `1×3×H×W`；长边缩放到 640，左上对齐，右侧和下侧用 128/255 填充到 32 的倍数 | `heatmap`、`size`、`offset`，均为 `1×2×H/8×W/8`，两个通道分别对应成员和留影 | MobileNetV3-Large 特征、步长 8 的特征金字塔（宽 96）、中心热图与卡框尺寸／中心偏移回归头 | +| `member-encoder.onnx` | `N×3×160×128` 卡面窗口 | L2 归一化的 128 维向量 | MobileNetV3-Small 特征、固定 2×2 平均池化、线性投影 | +| `snap-encoder.onnx` | `N×3×128×224` 卡面窗口 | 同上 | 同上 | +| `member-fields.onnx` | `N×3×56×144` 参数区裁块 | 106 个 logits：0 为其他／不可读，1–100 为等级，101–105 为特训次数 1–5 | 24/48/64/96 通道卷积、池化、线性分类头 | +| `snap-fields.onnx` | `N×3×56×144` 参数区裁块 | 101 个 logits：0 为其他／不可读，1–100 为等级 | 同上 | +| `member-rank.onnx`、`snap-rank.onnx` | `N×3×48×48` 卡阶图标裁块 | 6 个 logits：0 为无可读图标，1–5 为卡阶 | 32/32/64/96/128 通道卷积、池化、线性分类头 | -权重不独立承担全部识别。RootSIFT/RANSAC 定位后通过卡面像素核验；弱纹理网格位置由神经图库和像素匹配共同确认。卡阶用原始精灵与卡面背景合成匹配，检查图标是否完整可见。等级进一步检查 masterdata 上限。 +`recognition.json` 是模型清单，记录每个文件的 SHA-256、输入输出说明和接受阈值；推理启动时逐个校验。 -卡阶模板接受条件为匹配分数至少 0.78、前两名分差至少 0.007；0 阶模板用于拒绝非已拥有状态，不输出合法卡阶 0。图标越界或分数不足时返回未知。模板分数也不是校准概率。 +## 判定规则 -读取位置修正保持选定权重和阈值不变：成员参考图已按 UI 比例规范化，留影参考图仍是完整原图，因此先将留影匹配框转换成 314×172 的等比填充显示窗口，再读取参数／卡阶；背景匹配仍按完整原图坐标采样。否则 512×288 留影图的上下裁切量会变成随分辨率放大的读取偏差。本修正不引入新的训练，也不降低接受阈值。 +- 定位：热图峰值不低于 0.4 时输出卡框。卡框可超出图像边界，`visible_fraction` 记录图像内面积占比。 +- 身份：卡面窗口为卡框向内缩进 6 个逻辑单位的区域。编码后与图库所有同类卡比较余弦相似度，最高相似度不低于 `similarity` 且与第二名差值不低于 `margin` 时接受。成员阈值 0.88／0.15，留影 0.81／0.16。未接受的卡框输出到 `unidentified`，附最相近的候选。 +- 参数：参数区裁块须完整位于图像内；softmax 最高概率不低于 `confidence`、与第二名的差值不低于 0.5 时输出。成员阈值 0.9994，留影 0.995。等级视图只输出等级，特训视图只输出特训次数，其他视图输出未知；已识别卡的等级超出 masterdata 上限时改为未知。 +- 卡阶：整个图标须位于图像内；成员阈值 0.9998，留影 0.995，第二名差值至少 0.5。类别 0 不输出合法卡阶。 -## 实际训练链 +阈值的选择规则：编码器在开发验证集上，在训练外身份误接受和图库外卡面误接受都不超过 1%、接受精度不低于 99.5% 的条件下，取已知与训练外身份平均接受正确率最高的阈值；参数和卡阶模型取开发集所有裁块零误读的最低置信度阈值（不低于 0.995）。得分和相似度都不是经过校准的概率。 -| 阶段 | 初始化 | 数据／步数 | 优化器 | -|---|---|---|---| -| encoder v4 | torchvision MobileNetV3-Small ImageNet 权重 | 训练身份卡面增强，1200 steps，batch 128，seed 7319 | AdamW，lr 0.0004、weight_decay 0.0001 | -| level v6 | 随机初始化，seed 8317 | 202000 合成等级裁块；12000 steps，batch 512 | AdamW，lr 0.001 | -| fields v7 | level v6，扩展至 106 类 | 106000 多语言原生字段裁块；8000 steps，batch 512，seed 90011 | AdamW，lr 0.0003、weight_decay 0.0001 | -| encoder v8 | encoder v4 | 2398 张训练整屏中的 37310 个完整卡片裁块；2400 steps，batch 128，seed 71933 | AdamW，lr 0.00012、weight_decay 0.0001 | -| fields v8 | fields v7 | 同批场景字段裁块，约三分之一 batch 回放原生字段预训练集；2400 steps，batch 512，seed 90011 | AdamW，lr 0.0003、weight_decay 0.0001 | -| encoder v11 | encoder v8 | 相同场景数据追加 2400 steps，batch 128，每 300 步评估 | 新建 AdamW，lr 0.00012、weight_decay 0.0001 | -| fields v12 | fields v8 | 106000 个按截图采集顺序生成的困难字段；回放场景字段，训练 8000 steps，batch 512，每 500 步评估；选用第 5500 步 | 新建 AdamW，lr 0.0003、weight_decay 0.0001 | +## 训练 -候选仅依据开发集选择：普通可见字段正确数不得低于原模型、普通误读为零、压力集错误和隐藏字段输出不高于原模型，再在符合条件的 checkpoint 中提高压力集正确覆盖数。第 8000 步虽接受更多困难字段，错误也更多，因此未采用。选择规则及权重哈希保存在 `selection.json`；最终测试指标未用于选择。 +| 模型 | 初始化 | 训练数据 | 主要设置 | 选用 | +|---|---|---|---|---| +| 定位器第 1 阶段 | torchvision MobileNetV3-Large ImageNet 权重（仅主干） | scenes-v2 训练集 16000 屏，加 640 屏单类卡面合成图 | 15000 steps，batch 32，512 随机裁块，lr 0.002（主干 ×0.5），AdamW，weight_decay 0.0001，seed 1 | 开发验证 AP50 最高的第 3000 步 | +| 定位器第 2 阶段 | 第 1 阶段选用检查点 | scenes-v2.1 与 scenes-v2 训练集共 32000 屏，加同样 640 屏 | 4000 steps，lr 0.0006，warmup 100，seed 3 | 第 500 步 | +| 成员／留影编码器 | 上一版本共享卡面编码器 `encoder.pt` | scenes-v2.1 训练集卡框裁块，`id % 7 == 0` 的身份不进入查询和参考 | 3000 steps，batch 256，lr 0.00012，对比学习温度 0.08；参考向量每 20 步无梯度重算；另一类卡的训练身份作为额外身份；对初始模型做余弦蒸馏；seed 52817 | 成员第 1750 步，留影第 1000 步 | +| 成员参数第 1 阶段 | 上一版本共享参数模型 `fields.pt` | scenes-v2.1 训练集参数裁块，加 60000 个采集降质裁块（8% 带遮挡） | 8000 steps，batch 512，lr 0.0003，困难裁块占 40%，5% 线性 warmup 后余弦衰减 | 第 6500 步 | +| 成员参数第 2 阶段 | 第 1 阶段选用检查点 | 同上 | 3000 steps，lr 0.0001，裁切位置抖动 4／7 px、尺度 3.5% | 第 2500 步 | +| 留影参数 | 上一版本共享参数模型 `fields.pt`，分类头保留 0–100 行 | 留影参数裁块，加 60000 个采集降质裁块（8% 带遮挡） | 8000 steps，batch 512，lr 0.0003 | 第 4000 步 | +| 成员卡阶 | 随机初始化 | 成员卡阶图标裁块，加 60000 个无遮挡采集降质裁块 | 6000 steps，batch 256，lr 0.002，困难裁块占 40% | 第 3500 步 | +| 留影卡阶 | 随机初始化 | 留影卡阶图标裁块，加 60000 个采集降质裁块（8% 带遮挡） | 6000 steps，batch 256，lr 0.002 | 第 6000 步 | -编码器训练排除 `id % 7 == 0` 的 17 个身份，训练查询及参考分支不使用其卡面。验证时才将新卡向量加入图库。固定模型权重可通过 catalog／素材／索引更新接入新卡。 +每个模型的完整参数、开发集曲线上的选用点和选择规则见 `reports/training/model-selection.json`。上一版本共享编码器和参数模型的训练链见 [REPRODUCTION.md](REPRODUCTION.md)。 -实际训练机器为用户提供的 ModelScope DSW-AMD，框架报告显存 191.69 GiB，设备名称字符串为空，因此不推断具体 GPU 型号。实际环境：Python 3.12.13、torch 2.12.0+git6bbd260、torchvision 0.27.1+df56172、HIP 7.2.53211、Pillow 12.3.0、numpy 2.3.5、ONNX 1.22.0、ONNX Runtime 1.30.0。公开这些实际版本，不将自定义镜像版本写成普通 PyPI 通用安装组合。 +编码器训练排除 `id % 7 == 0` 的身份（当前图库共 19 个），训练查询及参考分支都不使用其卡面；它们只在评估时作为图库成员出现，用于检验未训练身份的检索能力。参数与卡阶模型训练时使用过这些卡的整屏画面，因此“训练外身份”只针对编码器。 -设置随机种子用于数据和训练复现；未宣称跨 GPU、驱动及框架实现的训练权重逐字节确定。发布 checkpoint 哈希用于精确确认使用的权重。 +设置随机种子用于数据和训练复现；未宣称跨 GPU、驱动及框架实现的训练权重逐字节确定。发布检查点哈希用于精确确认使用的权重。 ## 权重指纹 -| 权重 | SHA-256 | -|---|---| -| 发布 `encoder.onnx` | `a0a64115241a0eb22e2aab3b151ee3a3f57dc317d88524c027aaa443d051efa1` | -| 发布 `fields.onnx` | `768889db7b475dce28ed949299d2e7a624faa3ee18f3c28bb22deac2bcdd1a8d` | -| 发布 `encoder.pt` | `8d67b87580188473d3408c4c1995f0e20b943d40ddcd2f8fc6a0f7a7fd26abda` | -| 发布 `fields.pt` | `15351548a593c67cd939b72c3b03908e2665be5c39a0ca8148bd0cfc33feb0ce` | +| 权重 | ONNX SHA-256 | PyTorch state_dict SHA-256 | +|---|---|---| +| locator | `f160f2a5022a9ff1295426014bd03a76815d42a0bba8826dda93d8dc54120ec6` | `67b2be9f43f301790381379358198b9384afb06a6670ea3c3090c22cb28f36ca` | +| member-encoder | `e2110efcbef161a7f0783841d07be6f7da742ccd6a5406361e59ae728a837b9a` | `1cca598adf035eedccf6c15921fb99c2d8aaec84cf5a038e90002d2acf6bd0cb` | +| snap-encoder | `ef3db30237d3d840727c298c7100b86c56dfec77a83aff6faa7712bfcee33b93` | `75b41723536e86a0044be429a40de83f7529bd4ce291aab6936b9367ee46d182` | +| member-fields | `bc6ff2c7221d0ea197661dd6670fb12d3bd8d0e6811c00b8caa904a793b59264` | `d86eb13d221864f87de0d93c4d7898c5761d3968c2f2c040f2203388fa7da603` | +| snap-fields | `fa35b8a80688e3c1f92b3dbd6c567364b842a3a8f25387fac0879b7f740591b4` | `3917584b999cfe3ea9d359c6da3af38ee06c3631fc68b3a56b6d708ec3f6891b` | +| member-rank | `e8e8daa2a28def53b7d1baca281980db118fc9a3364cfde02f4bd8b3f8af68fc` | `d918e88fac08ced7c2687b2cae654261e96ec1e51ea7e5b5a74d76b636044e01` | +| snap-rank | `38c5eb44cecc3380bcc2adf3c0f59049e904b5618593950b27c2d552412883f1` | `6fc054aae5f03f43f51ee4b64692f9a13a324a7f60f536c2bbcdb4c43245a363` | ## 验证边界 -完整指标见最终评估卡及 JSON 报告。场景裁块验证上的 100% 分类结果不能代替完整截图定位／裁切后的准确率。真实 10 张图属于校准和回归，不能据此宣称跨玩家真实分布准确率。 +完整指标见评估卡及 JSON 报告。最终测试是合成整屏，覆盖训练没有用过的屏幕分辨率与压缩配置，但仍出自同一渲染器；8 张真实截图只用于回归，不能据此宣称跨玩家真实分布准确率。 -低分辨率、重压缩、模糊和裁切会降低覆盖率；未知字段需要校对。未知卡拒识测试只覆盖测试集中移除图库的身份及指定负样本，不保证所有未知图像无误报。布局或客户端字体变化需要重新验证。 +被屏幕边缘切掉较多或被浮层遮挡的卡，身份大多进入 `unidentified`；低分辨率、重压缩和模糊会降低字段覆盖率。图库外卡面的误接受率在测试中约为 1%，新卡上线后应先更新图库。布局或客户端字体变化需要重新验证。 diff --git a/docs/NEW-CARDS.md b/docs/NEW-CARDS.md index e44bdb6..5b2416b 100644 --- a/docs/NEW-CARDS.md +++ b/docs/NEW-CARDS.md @@ -1,21 +1,23 @@ -# 新卡接入，无需重新训练 ID 分类器 +# 新卡接入，无需重新训练 -系统把截图卡面与可更新的图库匹配。卡面编码模型输出通用向量，不输出固定 master ID 分类；RootSIFT 定位索引和神经检索图库共同更新。 +卡面编码器输出通用向量，不输出固定 master ID 分类；身份由截图卡面与本地图库的向量检索决定。新卡只需加入图库，模型权重不变。 1. 备份当前完整数据目录，更新 `master/MasterMemberCard.json`／`MasterSupportCard.json` 及对应的 `MasterText.json`。培养规则变化时同步等级、觉醒和卡阶表；不要混用不兼容版本。 -2. 放入对应卡面文件。成员需要 `native/assets/member-.webp` 和 `member--square.webp`；留影需要 `snap-.webp`。当前准备程序对缺失文件使用已有 bdon.moe 资源地址补取，并验证图像可解码。 -3. 执行 `python boxvision.py prepare --data DATA`。检查输出 `unavailable` 列表，缺失素材的卡片不会进入可识别目录。成员 square 图按当前列表裁切规则构建 canonical 图。 -4. 重启识别服务。新目录的顺序、RootSIFT 所有者索引和图库必须是一套，不能只手改 `catalog.json` 后继续使用旧索引。 -5. 用新卡截图确认 ID、定位、字段和旧卡回归。图库缓存指纹包含编码器字节、卡片 ID 顺序、canonical 图字节；变化时重建向量。保存 master 表、素材和模型哈希及验证报告。 +2. 放入对应卡面文件。成员需要 `native/assets/member-.webp` 和 `member--square.webp`；留影需要 `snap-.webp`。准备程序对缺失文件使用已有 bdon.moe 资源地址补取，并验证图像可解码；`--offline` 禁止补取。 +3. 执行 `python boxvision.py prepare --data DATA`。检查输出的 `unavailable` 列表，缺失素材的卡片不会进入可识别目录。准备结束时会用当前编码器重建 `models/member-gallery.npz` 和 `models/snap-gallery.npz`。 +4. 重启识别服务。图库缓存指纹包含编码器哈希、卡片顺序和每张参考卡面的字节，目录或卡面变化后自动重建，不能只手改 `catalog.json` 而沿用旧缓存。 +5. 用新卡截图确认 ID、定位、字段和旧卡回归。保存 master 表、素材和模型哈希及验证报告。 -更新流程不修改模型权重。新卡使用旧的 UI／字体时通常不需模型训练；如果布局、数字字体、卡阶图标或渲染发生变化，需要额外验证，必要时更新渲染器与字段模型。 +新卡加入图库之前，截图中的新卡通常列在 `unidentified` 中，也可能被误认成外观相近的已知卡（最终测试中图库外卡面的误接受率约 1%）。新卡上线后应尽快更新图库。 -`scripts/verify_gallery_update.py` 是集成验证：在独立目录暂时移除指定身份，重建索引，再加入其目录及素材；检查卡片重新可识别且编码器哈希未变。请提供自己的本地图片，包含待验证的身份；仓库不分发游戏截图。示例： +新卡使用旧的 UI／字体时通常不需模型训练；如果布局、数字字体、卡阶图标或渲染发生变化，需要额外验证，必要时更新渲染器并重训参数或卡阶模型。 + +`scripts/verify_gallery_update.py` 是集成验证：在独立目录里从目录中暂时移除指定身份，确认它们进入 `unidentified`；再加回目录条目，确认重新识别，并检查模型文件哈希未变。请提供自己的本地图片，包含待验证的身份；仓库不分发游戏截图。示例： ```bash python scripts/verify_gallery_update.py --data DATA --output EMPTY_OUTPUT --member-image member-test.jpg --snap-image snap-test.jpg --member-id 7 --snap-id 70 ``` -它不改生产数据，不把集成检查称为独立准确率。已完成的公开统计使用 member 7／snap 70 的校准回归；其他使用者可选择自己截图中存在的 ID。 +它不改生产数据，不把集成检查称为独立准确率。 -训练外身份测试采用 `id % 7 == 0` 的 ID，禁止进入编码器训练的查询和参考分支。验证阶段才加入图库。另将这些 ID 从图库彻底移除，测量未知卡是否被误认成已知卡；这两项测试分别验证“可添加”和“未添加时拒识”。 +训练外身份测试采用 `id % 7 == 0` 的 ID，它们不进入编码器训练的查询和参考分支，只在评估时作为图库成员出现。图库外卡面测试使用不属于目录的美术图，测量未知卡是否被误认成已知卡；这两项测试分别验证“加入图库即可识别”和“未加入时拒识”。 diff --git a/docs/REPRODUCTION.md b/docs/REPRODUCTION.md index 71cc702..1f9e924 100644 --- a/docs/REPRODUCTION.md +++ b/docs/REPRODUCTION.md @@ -1,8 +1,10 @@ -# 可执行训练复现入口 +# 初始化权重的训练复现 -`scripts/reproduce_training.py` 将实际训练链整理为顺序执行的流程。它使用 `reproduction/stages/` 保存的阶段源码，而不是把后来修改过的生成器冒充原始训练版本。这里仅保留重建已发布 checkpoint 所需的训练源码、渲染器和哈希清单；训练日志、smoke 产物、旧报告和截图不放在发布仓库。 +v0.2.0 的成员／留影编码器从共享卡面编码器 `encoder.pt`（SHA-256 `8d67b87580188473d3408c4c1995f0e20b943d40ddcd2f8fc6a0f7a7fd26abda`）初始化，成员／留影参数模型从共享参数模型 `fields.pt`（SHA-256 `15351548a593c67cd939b72c3b03908e2665be5c39a0ca8148bd0cfc33feb0ce`）初始化。这两个权重随 v0.1.0 发布附件分发。本文件说明如何重建它们；v0.2.0 各模型自身的训练步骤见 [TRAINING.md](TRAINING.md)。 -先按 RESOURCES.md 准备 `data`。训练环境需要可用的 GPU PyTorch、对应的 torchvision，以及 numpy、Pillow、opencv、onnx；本次实际环境是 ModelScope 的定制 HIP 镜像，版本见 `reproduction/metadata/environment.json`。不要把其中的 git 构建版本误当作普通 PyPI 安装命令。当前训练器把裁块整体放入显存，预训练数据规模较大，不能根据推理模型体积估计训练显存需求。 +`scripts/reproduce_training.py` 将这两个权重的训练链整理为顺序执行的流程。它使用 `reproduction/stages/` 保存的阶段源码和 `reproduction/renderers/` 中的渲染器，而不是当前源码；`reproduction/metadata/` 记录检查点、资源和环境的哈希与版本。 + +该训练链所用的数据目录需按 v0.1.0 标签的源码和 RESOURCES.md 组装（`git checkout v0.1.0` 后运行 `boxvision.py prepare`），其中包含阶段源码需要的 `index.npz`。训练环境需要可用的 GPU PyTorch、对应的 torchvision，以及 numpy、Pillow、opencv、onnx；原训练的库版本见 `reproduction/metadata/environment.json`，其中的 git 构建版本不是普通 PyPI 安装命令。训练器把裁块整体放入显存，不能根据推理模型体积估计训练显存需求。 先只检查源码、master 快照和命令计划： @@ -16,7 +18,7 @@ python scripts/reproduce_training.py --data data --output training-run --dry-run python scripts/reproduce_training.py --data data --output training-run ``` -输出目录必须为空，避免覆盖旧实验。流程包含基础卡面／等级模型、多语言原生字段训练、两类整屏生成、整屏微调、相同数据追加训练对照，以及修正截图压缩顺序后的困难字段训练。每个子阶段日志写入 `logs/`，`run.json` 记录退出码与耗时；失败会停止并保留已有产物。 +输出目录必须为空，避免覆盖旧实验。流程包含基础卡面／等级模型、多语言原生字段训练、两类整屏生成、整屏微调和困难字段训练。每个子阶段日志写入 `logs/`，`run.json` 记录退出码；失败会停止并保留已有产物。 已有公开初始化权重时，可跳过三个基础模型训练阶段，继续重建对应数据及后续训练： @@ -24,12 +26,8 @@ python scripts/reproduce_training.py --data data --output training-run python scripts/reproduce_training.py --data data --output training-run --initializers weights/initializers ``` -初始化文件必须是 `encoder-v4.pt`、`level-v6.pt`、`fields-v7.pt`，并通过已记录 SHA-256 校验。`--baseline-only` 只运行原 v8 模型训练链，不执行后续对照和困难数据实验。 - -master 字节与历史快照不一致时默认停止。若有意用新卡或更新数据做新实验，可显式使用 `--allow-resource-change`；这种结果必须标记为新资源实验，不声称复现历史输入。该检查不能替代完整的数据集和渲染验证，运行中还会记录输入 provenance 与源码清单哈希。 - -## checkpoint 选择和最终测试 +初始化文件必须是 `encoder-v4.pt`、`level-v6.pt`、`fields-v7.pt`，并通过已记录 SHA-256 校验。`--baseline-only` 只运行到整屏微调。 -训练完成不自动将最后一个 checkpoint 作为发布模型。普通开发集与开发压力集用于模型选择，最终测试单独生成和执行。当前候选选择是追加训练编码器第 2400 步，以及困难字段模型第 5500 步，选择依据单独保存。 +master 字节与记录的快照不一致时默认停止。若有意用新卡或更新数据做新实验，可显式使用 `--allow-resource-change`；这种结果必须标记为新资源实验，不声称复现历史输入。运行中还会记录输入 provenance 与源码清单哈希。 -各阶段已分别实际执行完整训练；新包装入口完成了计划检查和小规模集成验证，但尚未通过此入口从零再次执行完整规模的训练。集成验证不产出可部署权重。GPU／驱动／算子版本变化也可能导致权重字节及指标变化，随机种子不构成跨环境逐字节确定性保证。 +训练完成不自动将最后一个检查点作为结果；开发集用于选择，最终测试单独执行。GPU／驱动／算子版本变化可能导致权重字节及指标变化，随机种子不构成跨环境逐字节确定性保证。 diff --git a/docs/RESOURCES.md b/docs/RESOURCES.md index 8b0d119..701c7a1 100644 --- a/docs/RESOURCES.md +++ b/docs/RESOURCES.md @@ -21,7 +21,7 @@ python scripts/prepare.py --master-dir MASTER --ui-dir exports/game-ui --font-di `--offline` 禁止联网。省略该选项时，缺少的卡图按准备器中的现有 bdon.moe 路由下载；实际目录记录来源 URL 和哈希。解密后的 UI/font 数据和用户截图不因运行本工具而上传到任何地方。 -组装过程复制本项目内的静态渲染器，解码卡阶等精灵，生成 canonical 图、catalog 和 RootSIFT 索引，并为导出资源记录哈希。模型权重单独放入 `data/models/encoder.onnx` 和 `data/models/fields.onnx`；第一次推理会生成可重建图库缓存。没有权重也可以先运行合成数据生成。以上提取结果仅保留在使用者本地，不属于公开源码／权重附件；禁止将其加入本项目公开发布包。 +组装过程复制本项目内的静态渲染器，解码卡阶等精灵，生成 canonical 图和 catalog，并为导出资源记录哈希。权重包 `models/` 中的 7 个 ONNX 文件和 `recognition.json` 单独放入 `data/models/`；放入后运行 `boxvision.py prepare --data data` 或第一次推理时会生成可重建的图库缓存。没有权重也可以先运行合成数据生成。以上提取结果仅保留在使用者本地，不属于公开源码／权重附件；禁止将其加入本项目公开发布包。 ## 已验证的复现范围 diff --git a/docs/TRAINING.md b/docs/TRAINING.md index 04ceee1..e78d364 100644 --- a/docs/TRAINING.md +++ b/docs/TRAINING.md @@ -1,58 +1,84 @@ # 合成训练与验收复现 -完整记录的可执行入口和历史源码使用方式见 [REPRODUCTION.md](REPRODUCTION.md)。当前文件介绍各模块；复现具体历史权重时应使用固定阶段源码。 +本文件介绍 v0.2.0 模型的数据生成、训练、选择和评估入口。各模型的实际参数与选用检查点见 `reports/training/model-selection.json`，初始化权重的训练链见 [REPRODUCTION.md](REPRODUCTION.md)。 ## 数据边界 -训练主力为客户端资源合成；真实截图只是校准和回归。每张合成图记录哈希、屏幕原始分辨率、上传分辨率、缩放、模糊、压缩顺序、locale、显示模式和可见字段真值。潜在培养状态另存 `latent_state`，禁止用它当不可见字段的识别目标。 +训练主力为客户端资源合成；真实截图只是回归。每张合成图记录哈希、屏幕原始分辨率、上传分辨率、缩放、模糊、压缩顺序、locale、显示模式和可见字段真值。潜在培养状态另存 `latent_state`，禁止用它当不可见字段的识别目标。 -`scenarios.py` 的 train／validation／stress 使用不同的原始分辨率列表和压缩质量离散值，并有不同 UI 尺度、模糊和重复压缩范围。不是只换随机种子。四个相邻场景共享玩家状态，以测试重排、重叠和去重。成员和留影 ID 分命名空间。 +`scenarios.py` 的 train／validation／stress／final 四档使用不同的原始分辨率列表和压缩质量离散值，并有不同 UI 尺度、模糊和重复压缩范围，不是只换随机种子。成员和留影 ID 分命名空间。 APK 1.0.1-25 的语言选择证据：日／英主字体为 A-OTF-ShinGoPr6N-Regular，简／繁中为 FZLTH_GB18030L2_R，韩语为 Pretendard-SemiBold；数字角色均为 VibeMOPro-Medium。字体由语言配置驱动，不能仅由“区服”推断。使用真实 glyph、材质配置、字距和回退信息；SDF 栅格化仍是近似，不能声称与所有设备逐像素一致。 -等级上限与卡阶／觉醒范围使用 masterdata。能力视图中的 power 数字目前是随机干扰值，**不是经过 native 算法校验的完整战力模拟**。真实力量值相关逻辑不参与当前导出目标，仍应保留该合成真实性限制。 +等级上限与卡阶／觉醒范围使用 masterdata。能力视图中的 power 数字是随机干扰值，**不是经过 native 算法校验的完整战力模拟**，不参与导出目标。 -## 生成与验证 +## 列表整屏场景 -在本目录作为 Python 模块入口运行： +`bdon_vision.scenes` 按列表组件的原生几何放置原生卡块：1920×1080 参考画布按 `min(W/1920, H/1080)` 缩放，成员每行 6 张、留影每行 4 张并水平居中，顶部留白 180 个参考单位，再加滚动偏移。画面另含带雾感的模糊背景、顶栏标题与排序／视图按钮、少量界面浮层、图库外卡面和无卡负样本。可选的场景背景放在 `DATA/backgrounds/adv-stage/*.webp`，由使用者自行准备；缺失时只用卡面模糊、噪声和渐变背景。 ```bash -python -m bdon_vision.synthetic generate --data DATA --output synthetic/train --count 2048 --profile train --seed 5201 --modes level,training,total,performance,technic,visual,hide --locales ja,en,zh-Hant,zh-Hans,ko -python -m bdon_vision.synthetic generate --data DATA --output synthetic/validation --count 560 --profile validation --seed 6203 --modes level,training,total,performance,technic,visual,hide --locales ja,en,zh-Hant,zh-Hans,ko -python -m bdon_vision.synthetic evaluate --data DATA --dataset synthetic/validation --output results/validation -python -m bdon_vision.evaluate_open_set --data DATA --dataset synthetic/validation --output results/open-set -python -m bdon_vision.audit_merge --dataset synthetic/validation --observations results/validation/observations.json --output results/validation/box-audit.json +python -m bdon_vision.scenes generate --data DATA --output scenes/v2.1-train --count 16000 --seed 20261111 --profile train --workers 48 +python -m bdon_vision.scenes generate --data DATA --output scenes/v2.1-validation --count 2000 --seed 20261112 --profile validation --workers 48 +python -m bdon_vision.scenes generate --data DATA --output scenes/v2.1-stress --count 1000 --seed 20261113 --profile stress --workers 48 +python -m bdon_vision.scenes generate --data DATA --output scenes/v2.1-final --count 2000 --seed 20261114 --profile final --workers 48 +python -m bdon_vision.scenes verify scenes/v2.1-train +python -m bdon_vision.scenes stats scenes/v2.1-train --output scenes/v2.1-train-stats.json ``` -数据生成最后才将 `complete` 置为 true。训练裁块构建检查全部截图哈希。评估要求身份及 bbox IoU≥0.5，记录漏卡、多报、字段正确／错误／未知、隐藏字段误报，以及按语言、卡片尺寸、压缩和分辨率分层。`audit_merge` 额外检查 box 合并后的字段、凭空出现的值、虚假冲突和真实冲突丢失。 +真值格式为 `bdon-synthetic/2`，新增字段列在 `truth.json` 的 `schema_extension` 中。`visible_fraction` 同时扣除屏外部分和界面遮挡；`foreign_cards` 记录图库外卡面的卡框和字段，不含 `id`。train 档排除 `id % 7 == 0` 的身份，图库外卡面来源和卡面背景也排除；编号是 5 的倍数的场景背景只用于非训练档。`final` 档的分辨率和压缩质量与其他档都不同，只在模型冻结后使用。图片可存为无损 WebP，像素与施加采集损伤后的结果一致。每张图由生成器版本、profile、seed 和序号单独决定，与进程数和分块方式无关。 -## 列表整屏场景 +当前源码的生成器标识为 `scenes-v2.1`。训练和测试还使用了一组上一生成器版本 `scenes-v2` 的数据（种子 20261101–20261104），它们只能以真值哈希核对，不能由当前源码逐字节重建。 -`bdon_vision.scenes` 按列表组件的原生几何放置原生卡块：1920×1080 参考画布按 `min(W/1920, H/1080)` 缩放，成员每行 6 张、留影每行 4 张并水平居中，顶部留白 180 个参考单位，再加滚动偏移。画面另含带雾感的模糊背景、顶栏标题与排序／视图按钮、少量界面干扰、图库外美术卡块和无卡负样本。可选的场景背景放在 `DATA/backgrounds/adv-stage/*.webp`，由使用者自行准备；缺失时只用卡面模糊、噪声和渐变背景。 +## 卡框定位器 ```bash -python -m bdon_vision.scenes generate --data DATA --output scenes/train --count 16000 --seed 1 --profile train --workers 48 -python -m bdon_vision.scenes verify scenes/train -python -m bdon_vision.scenes stats scenes/train --output scenes/train-stats.json +python -m bdon_vision.train_locator --train scenes/v2-train "synthetic/train-*" --validation scenes/v2-validation --validation-limit 600 --backbone mobilenet_v3_large-5c1a4163.pth --output runs/locator-a --steps 15000 --lr 0.002 --warmup 500 --seed 1 +python -m bdon_vision.train_locator --train scenes/v2.1-train scenes/v2-train "synthetic/train-*" --validation scenes/v2.1-validation --validation-limit 600 --initial runs/locator-a/best.pt --output runs/locator-b --steps 4000 --lr 0.0006 --warmup 100 --eval-every 500 --seed 3 +python -m bdon_vision.export_locator --weights runs/locator-b/best.pt --output models/locator.onnx +python -m bdon_vision.evaluate_locator --model models/locator.onnx --set v2.1-validation=scenes/v2.1-validation --output locator-validation.json ``` -真值仍是 `bdon-synthetic/2`：`cards`、`ignored_cards` 语义不变，新增字段列在 `truth.json` 的 `schema_extension` 中。`visible_fraction` 同时扣除屏外部分和界面遮挡；`foreign_cards` 记录图库外美术的卡框和字段，不含 `id`。train 档排除 `id % 7 == 0` 的身份，外来卡来源和卡面背景也排除；编号是 5 的倍数的场景背景只用于非训练档。`final` 采集档的分辨率和压缩质量与其他档都不同，只在模型冻结后使用。图片可存为无损 WebP，像素与施加采集损伤后的结果一致。每张图由 profile、seed 和序号单独决定，与进程数和分块方式无关。 +`synthetic/train-*` 是 `bdon_vision.synthetic generate --profile train` 生成的 16 组、每组 40 张单类卡片截图（种子 300000–315000）。训练按 512 像素随机裁块，以中心热图 focal loss、尺寸 L1 和 GIoU 为目标，按开发验证 AP50 选用检查点。 + +## 卡面编码器 + +```bash +python -m bdon_vision.kind_corpus --data DATA --datasets scenes/v2.1-train --output banks/encoder/train --arrays context --stride 2 +python -m bdon_vision.kind_corpus --data DATA --datasets scenes/v2.1-validation --output banks/encoder/validation --arrays art baseline +python -m bdon_vision.kind_corpus --data DATA --datasets scenes/v2.1-stress --output banks/encoder/stress --arrays art baseline +python -m bdon_vision.train_kind_encoder --kind member --data DATA --train banks/encoder/train --validation banks/encoder/validation --stress banks/encoder/stress --initial encoder.pt --output runs/member-encoder --steps 3000 --lr 0.00012 --head-lr 0.00012 --margin 0 --open-weight 0 --same-name-margin 0 --kd-weight 1 --ref-mode frozen --cross-kind +python -m bdon_vision.export_kind_encoder --kind member --data DATA --model runs/member-encoder/step-001750.pt --output models --thresholds runs/member-encoder/selection.json +``` -## 场景微调 +留影把 `--kind` 换成 `snap`。训练查询来自整屏卡框裁块，参考来自图库卡面；`id % 7 == 0` 的身份在两个分支中都被排除。每 250 步在开发验证集上搜索接受阈值（训练外身份留一误接受与图库外卡面误接受均不超过 1%，接受精度不低于 99.5%），取已知与训练外身份平均接受正确率最高的检查点；并列时比较压力集。导出时检查 PyTorch 与 ONNX 输出差异、批量与单张一致性和向量范数。 -GPU 环境使用已验证的 PyTorch／ONNX 依赖；不要把 GPU 包加入普通 CPU 使用者的运行依赖。实际训练脚本及模型的 JSON 报告保存参数、步数、样本量和数据指纹。 +## 参数与卡阶模型 ```bash -python -m bdon_vision.corpus --data DATA --datasets synthetic/train --output corpora/scenes-train -python -m bdon_vision.corpus --data DATA --datasets synthetic/validation --output corpora/scenes-validation -python -m bdon_vision.train_scene_encoder --data DATA --train corpora/scenes-train --validation corpora/scenes-validation --initial models/previous/encoder.pt --output models/candidate-encoder --steps 2400 --batch 128 -python -m bdon_vision.train_fields --train corpora/scenes-train --validation corpora/scenes-validation --initial models/previous/fields.pt --replay corpora/native-fields-train --output models/candidate-fields --steps 2400 --batch 512 +python -m bdon_vision.kind_fields --datasets scenes/v2.1-train --output banks/fields/v2.1-train +python -m bdon_vision.kind_fields --datasets scenes/v2.1-validation --output banks/fields/v2.1-validation +python -m bdon_vision.kind_fields --datasets scenes/v2.1-stress --output banks/fields/v2.1-stress +python -m bdon_vision.kind_hardcrops --data DATA --kind member --output banks/hard-g-member --count 60000 --seed 190001 --split train --occlusion-rate .08 +python -m bdon_vision.kind_hardcrops --data DATA --kind member --output banks/hard-n-member --count 60000 --seed 180001 --split train --occlusion-rate 0 +python -m bdon_vision.kind_hardcrops --data DATA --kind member --output banks/hard-l-member --count 16000 --seed 170001 --split validation +python -m bdon_vision.train_kind_fields --kind member --train banks/fields/v2.1-train --hard banks/hard-g-member --dev validation:normal:banks/fields/v2.1-validation stress:stress:banks/fields/v2.1-stress hard:hard:banks/hard-l-member --initial fields.pt --output runs/member-fields --steps 8000 --lr 0.0003 +python -m bdon_vision.train_kind_fields --kind member --train banks/fields/v2.1-train --hard banks/hard-g-member --dev validation:normal:banks/fields/v2.1-validation stress:stress:banks/fields/v2.1-stress hard:hard:banks/hard-l-member --initial runs/member-fields/selected.pt --output runs/member-fields-2 --steps 3000 --eval-every 250 --lr 0.0001 --jitter-shift 4 7 --jitter-scale .035 +python -m bdon_vision.train_kind_rank --kind member --train banks/fields/v2.1-train --hard banks/hard-n-member --dev validation:normal:banks/fields/v2.1-validation stress:stress:banks/fields/v2.1-stress hard:hard:banks/hard-l-member --output runs/member-rank --steps 6000 ``` -字段共有 106 类：0 表示其他／隐藏参数；1–100 表示等级；101–105 表示可见特训次数。图库模型训练排除指定身份的两个分支，验证时仅添加图像向量，不更新权重。 +`kind_hardcrops` 每个样本渲染一张原生卡块（卡面、卡框、卡阶图标）和一层本地化参数文字，贴到类似列表的背景上，再对整张卡块模拟屏幕采集：缩小到屏幕尺度、模糊、随机编码块相位、1–4 次 JPEG 或 WebP 压缩，最后按部署时的裁切几何取参数区和卡阶图标。`--occlusion-rate` 控制带界面浮层的比例；文字或图标像素被遮住至少 2% 的样本记为类别 0。留影把 `--kind` 换成 `snap`、种子加 2；成员第二条命令是成员参数的第 2 阶段。各模型使用的裁块库见报告。 + +参数模型分为成员 106 类、留影 101 类，从共享参数模型初始化；卡阶模型 6 类，随机初始化。训练时对裁切位置和尺度做随机抖动，使模型适应定位误差。每个检查点都在开发集上求出“所有开发裁块零误读”的最低置信度阈值（不低于 0.995，差值固定 0.5），再取该阈值下普通与压力整屏召回最高的检查点；选定阈值就是部署阈值。误读包括可见值读错，以及对不可见目标输出了与卡片状态不同的值。`select_kind_models` 可对已有检查点目录重新执行同一选择。 + +## 端到端评估 + +```bash +python -m bdon_vision.evaluate_engine --data DATA --set v2.1-final=scenes/v2.1-final --set v2-final=scenes/v2-final --output engine-final.json +python -m bdon_vision.audit_merge --dataset scenes/v2.1-validation --observations observations.json --output box-audit.json +``` -卡阶 1–5 均包含在整屏合成中，实际可见训练计数分别为 7208／7089／7371／7357／7372。卡阶推理使用原生精灵与局部背景的模板匹配，没有单独训练卡阶分类网络。它与等级／特训字段神经网络是不同模块，不能把卡阶模板验证描述成神经网络训练准确率。 +`evaluate_engine` 的计分规则见 [EVALUATION.md](EVALUATION.md)。`evaluate_kind_fields` 与 `evaluate_kind_encoder` 在开发集上分别评估参数／卡阶和编码器，只用于开发阶段。`audit_merge` 额外检查 box 合并后的字段、凭空出现的值、虚假冲突和真实冲突丢失。 -`scripts/verify_rank_matrix.py --data DATA --output rank-matrix` 遍历当前图库每个身份、masterdata 中每个合法已拥有卡阶及四种分辨率。为隔离卡阶变量，该矩阵固定等级为 1；等级泛化另看混合等级整屏数据和字段训练分布。现行整屏拼接保留原生透明边距，避免把卡框外伸出的属性／卡阶图标截掉；历史复现使用冻结的旧生成器，不能用新图冒充原始训练输入。 +候选 ONNX 先放入单独数据目录验收，不覆盖已使用的版本。裁块分类准确率不等于端到端整屏准确率，压力集错读也不能用普通集高分掩盖。 -候选 ONNX 先放入单独数据目录验收，不覆盖已使用的版本。保留真实回归、完整合成验证、压力集、未知卡、空背景、跨图冲突、图库更新集成和有限 CPU 性能证据。裁块分类准确率不等于端到端整屏准确率，压力集错读也不能用普通集高分掩盖。 +GPU 环境使用可用的 PyTorch／torchvision 与 ONNX；不要把 GPU 包加入普通 CPU 使用者的运行依赖。 diff --git a/reports/README.md b/reports/README.md index b869746..f38f9b4 100644 --- a/reports/README.md +++ b/reports/README.md @@ -1,9 +1,10 @@ # Published reports -This directory contains the compact evidence needed to understand the released training coverage, final acceptance, and CPU performance. Reports are grouped by their purpose rather than by internal experiment version. +This directory contains the compact evidence behind the released models: training-data coverage, model selection on development sets, and acceptance on frozen test sets. -- `training/` records card/rank coverage, level and rank sample counts, hard-field learning curves, and the selected checkpoints. -- `acceptance/` records the clear rank matrix, stratified full-screen synthetic results, data audit, open-set check, and a sanitized real-sample regression summary. -- `performance/` contains each measured CPU benchmark run. +- `training/scene-coverage.json` counts screenshots, cards, identities, display modes, resolutions and visible field values of every full-screen set. +- `training/model-selection.json` records, for each of the seven models, its architecture, initialisation, training settings, selected checkpoint with its development metrics, deployment thresholds and weight hashes. +- `acceptance/holdout-full-screen.json` holds the stratified end-to-end results of `bdon_vision.evaluate_engine` on the two final-profile sets, scored after all models and thresholds were frozen. +- `acceptance/real-sample-regression.json` summarises the real-screenshot regression set without the images. -The repository omits intermediate smoke outputs, superseded per-run reports, per-screenshot traces, screenshots, game assets, and private calibration details. The full-screen aggregate preserves counts and test strata; its raw source report SHA-256 values remain in that summary. Scope limits and denominators are documented in [EVALUATION.md](../docs/EVALUATION.md), [DATA_CARD.md](../docs/DATA_CARD.md), and [TRAINING.md](../docs/TRAINING.md). +Per-screenshot traces, screenshots and game assets are not included. Scope limits and denominators are documented in [EVALUATION.md](../docs/EVALUATION.md), [DATA_CARD.md](../docs/DATA_CARD.md) and [TRAINING.md](../docs/TRAINING.md). diff --git a/reports/acceptance/clear-rank-matrix.json b/reports/acceptance/clear-rank-matrix.json deleted file mode 100644 index 4cfc265..0000000 --- a/reports/acceptance/clear-rank-matrix.json +++ /dev/null @@ -1,421 +0,0 @@ -{ - "schema": "ournotes-boxlens.clear-rank-matrix/1", - "composition": "native-padding-preserved-v2", - "screenshots": 220, - "groups": { - "overall": { - "expected": 2440, - "identified": 2440, - "correct": 2440, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "rank/1": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity2/rank1": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "[1280, 720]": { - "expected": 610, - "identified": 610, - "correct": 610, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "ja": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "en": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "zh-Hant": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity3/rank1": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "zh-Hans": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "ko": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity4/rank1": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity2/rank1": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity3/rank1": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity4/rank1": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity10/rank1": { - "expected": 8, - "identified": 8, - "correct": 8, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "rank/2": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity2/rank2": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity3/rank2": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity4/rank2": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity2/rank2": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity3/rank2": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity4/rank2": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity10/rank2": { - "expected": 8, - "identified": 8, - "correct": 8, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "rank/3": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity2/rank3": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity3/rank3": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity4/rank3": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity2/rank3": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity3/rank3": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity4/rank3": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity10/rank3": { - "expected": 8, - "identified": 8, - "correct": 8, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "rank/4": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity2/rank4": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity3/rank4": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity4/rank4": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity2/rank4": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity3/rank4": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity4/rank4": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity10/rank4": { - "expected": 8, - "identified": 8, - "correct": 8, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "rank/5": { - "expected": 488, - "identified": 488, - "correct": 488, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity2/rank5": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity3/rank5": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "member/rarity4/rank5": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity2/rank5": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity3/rank5": { - "expected": 100, - "identified": 100, - "correct": 100, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity4/rank5": { - "expected": 40, - "identified": 40, - "correct": 40, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "snap/rarity10/rank5": { - "expected": 8, - "identified": 8, - "correct": 8, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "[1440, 1080]": { - "expected": 610, - "identified": 610, - "correct": 610, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "[1920, 1080]": { - "expected": 610, - "identified": 610, - "correct": 610, - "wrong": 0, - "unknown": 0, - "missed": 0 - }, - "[2560, 1440]": { - "expected": 610, - "identified": 610, - "correct": 610, - "wrong": 0, - "unknown": 0, - "missed": 0 - } - }, - "level_fixed_to": 1, - "scope": "Every catalog identity at every master-defined owned rank at four resolutions; complete visible lists, JPEG95, no blur or occlusion. Rank0 is not an owned rank.", - "recognition_provenance": { - "schema": "ournotes-boxlens.recognition-provenance/1", - "files": { - "catalog.json": "902b59ed37ea50e601187b4ccf387cacfbae61c714f52f524f42d6d320df7818", - "index.npz": "3a8d7e8f1e3d4d9ae14682a10a85476daeb7c912ad5c4763ca35abf9dac6e50f", - "models/encoder.onnx": "a0a64115241a0eb22e2aab3b151ee3a3f57dc317d88524c027aaa443d051efa1", - "models/fields.onnx": "768889db7b475dce28ed949299d2e7a624faa3ee18f3c28bb22deac2bcdd1a8d" - }, - "inference_core_sha256": "bf95bcd9579ae61c7dceedcf883befaa1cff81dd929d592e069249683c86dbfe", - "field_confidence_threshold": 0.995, - "onnxruntime_version": "1.20.1" - } -} \ No newline at end of file diff --git a/reports/acceptance/holdout-full-screen.json b/reports/acceptance/holdout-full-screen.json new file mode 100644 index 0000000..1c8795c --- /dev/null +++ b/reports/acceptance/holdout-full-screen.json @@ -0,0 +1,1893 @@ +{ + "schema": "ournotes-boxlens.engine-evaluation-summary/1", + "release": "0.2.0", + "scope": "End-to-end evaluation of ``Engine.scan`` on scene sets with tile truth.", + "matching": "one-to-one tile pairing at IoU >= 0.5; cards and unidentified outputs both pair", + "model_manifest_sha256": "bd726ca5a5f247828e80c9aba9d0a5c1a32b5a5ae5c063c3417802e15b77b716", + "models_onnx_sha256": { + "locator": "f160f2a5022a9ff1295426014bd03a76815d42a0bba8826dda93d8dc54120ec6", + "member-encoder": "e2110efcbef161a7f0783841d07be6f7da742ccd6a5406361e59ae728a837b9a", + "snap-encoder": "ef3db30237d3d840727c298c7100b86c56dfec77a83aff6faa7712bfcee33b93", + "member-fields": "bc6ff2c7221d0ea197661dd6670fb12d3bd8d0e6811c00b8caa904a793b59264", + "snap-fields": "fa35b8a80688e3c1f92b3dbd6c567364b842a3a8f25387fac0879b7f740591b4", + "member-rank": "e8e8daa2a28def53b7d1baca281980db118fc9a3364cfde02f4bd8b3f8af68fc", + "snap-rank": "38c5eb44cecc3380bcc2adf3c0f59049e904b5618593950b27c2d552412883f1" + }, + "catalog_sha256": "c6995ccac52fa98438eeca3cd82e375e6e417e64b5b0f3e91ecacfb5fee8647e", + "inference_core_sha256": "47f8bdc7f3dfd2dcada58c2a198d1373466153827e0823803d84592dc6428316", + "selection": "Models and thresholds were frozen on development sets before these sets were scored; no value here was used to select a checkpoint or threshold.", + "identity_split": "identity:heldout covers gallery cards with id % 7 == 0, whose artwork never entered encoder training; the gallery still contains them.", + "field_visibility": "A field target is visible when the list view shows it and its crop lies fully inside the image; values emitted for hidden targets are scored against the card state.", + "sets": { + "v2.1-final": { + "generator": "scenes-v2.1", + "profile": "final", + "seed": 20261114, + "truth_sha256": "83dda3e94d5ce63e6df93e957f5e405a444ee9b9ce74aa377a4465d9a48ddfbc", + "screens": 2000, + "strata": { + "card_width_bin:120": { + "cards": 10510, + "located": 10506, + "kind_correct": 10506, + "identity_correct": 8795, + "card_rank_visible": 8084, + "card_rank_correct": 8034, + "card_rank_unknown": 50, + "foreign_rejected": 1365, + "identity_unknown": 326, + "level_visible": 1879, + "level_correct": 1801, + "level_unknown": 77, + "awake_count_visible": 917, + "awake_count_correct": 887, + "awake_count_unknown": 30, + "foreign_false_accept": 20, + "level_wrong": 1, + "card_rank_hidden_emitted": 3, + "card_rank_hidden_latent_match": 3, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.963836, + "identity_precision": 1.0, + "level_coverage": 0.958489, + "level_misreads": 1, + "awake_count_coverage": 0.967285, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993815, + "card_rank_misreads": 0 + }, + "card_width_bin:180": { + "cards": 8220, + "located": 8215, + "kind_correct": 8215, + "identity_correct": 6895, + "identity_unknown": 211, + "foreign_rejected": 1095, + "foreign_false_accept": 14, + "card_rank_visible": 6392, + "card_rank_correct": 6352, + "card_rank_unknown": 40, + "awake_count_visible": 482, + "awake_count_correct": 467, + "awake_count_unknown": 15, + "level_visible": 1350, + "level_correct": 1300, + "level_unknown": 50, + "level_hidden_emitted": 2, + "level_hidden_latent_match": 2, + "identity_recall": 0.969625, + "identity_precision": 1.0, + "level_coverage": 0.962963, + "level_misreads": 0, + "awake_count_coverage": 0.96888, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993742, + "card_rank_misreads": 0 + }, + "card_width_bin:240": { + "cards": 3906, + "located": 3900, + "kind_correct": 3900, + "identity_correct": 3285, + "foreign_rejected": 533, + "card_rank_visible": 2867, + "card_rank_correct": 2863, + "card_rank_unknown": 4, + "level_visible": 594, + "level_unknown": 29, + "identity_unknown": 75, + "level_correct": 565, + "foreign_false_accept": 7, + "awake_count_visible": 55, + "awake_count_correct": 46, + "awake_count_unknown": 9, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "card_rank_hidden_emitted": 1, + "card_rank_hidden_latent_match": 1, + "identity_recall": 0.975936, + "identity_precision": 1.0, + "level_coverage": 0.951178, + "level_misreads": 0, + "awake_count_coverage": 0.836364, + "awake_count_misreads": 0, + "card_rank_coverage": 0.998605, + "card_rank_misreads": 0 + }, + "card_width_bin:300": { + "cards": 1540, + "located": 1536, + "kind_correct": 1536, + "identity_correct": 1341, + "card_rank_visible": 1122, + "card_rank_correct": 1117, + "card_rank_unknown": 5, + "foreign_rejected": 160, + "identity_unknown": 33, + "level_visible": 269, + "level_correct": 249, + "level_unknown": 20, + "foreign_false_accept": 2, + "identity_recall": 0.973149, + "identity_precision": 1.0, + "level_coverage": 0.925651, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995544, + "card_rank_misreads": 0 + }, + "card_width_bin:360": { + "cards": 121, + "located": 121, + "kind_correct": 121, + "identity_correct": 103, + "level_visible": 59, + "level_correct": 54, + "level_unknown": 5, + "card_rank_visible": 107, + "card_rank_correct": 106, + "card_rank_unknown": 1, + "foreign_rejected": 15, + "identity_unknown": 3, + "identity_recall": 0.971698, + "identity_precision": 1.0, + "level_coverage": 0.915254, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.990654, + "card_rank_misreads": 0 + }, + "card_width_bin:60": { + "cards": 4032, + "located": 4030, + "kind_correct": 4030, + "identity_correct": 3398, + "card_rank_visible": 3108, + "card_rank_correct": 3101, + "card_rank_unknown": 7, + "foreign_rejected": 500, + "level_visible": 707, + "level_correct": 658, + "level_unknown": 49, + "awake_count_visible": 321, + "awake_count_correct": 299, + "awake_count_unknown": 22, + "identity_unknown": 126, + "foreign_false_accept": 6, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "identity_recall": 0.963698, + "identity_precision": 1.0, + "level_coverage": 0.930693, + "level_misreads": 0, + "awake_count_coverage": 0.931464, + "awake_count_misreads": 0, + "card_rank_coverage": 0.997748, + "card_rank_misreads": 0 + }, + "foreign:False": { + "cards": 24604, + "located": 24591, + "kind_correct": 24591, + "identity_correct": 23817, + "card_rank_visible": 18817, + "card_rank_correct": 18736, + "card_rank_unknown": 81, + "identity_unknown": 774, + "level_visible": 4181, + "level_correct": 4008, + "level_unknown": 173, + "awake_count_visible": 1560, + "awake_count_correct": 1497, + "awake_count_unknown": 63, + "level_hidden_emitted": 3, + "level_hidden_latent_match": 3, + "card_rank_hidden_emitted": 4, + "card_rank_hidden_latent_match": 4, + "identity_recall": 0.968013, + "identity_precision": 1.0, + "level_coverage": 0.958622, + "level_misreads": 0, + "awake_count_coverage": 0.959615, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995695, + "card_rank_misreads": 0 + }, + "foreign:True": { + "cards": 3725, + "located": 3717, + "kind_correct": 3717, + "foreign_rejected": 3668, + "card_rank_visible": 2863, + "card_rank_correct": 2837, + "card_rank_unknown": 26, + "foreign_false_accept": 49, + "level_visible": 677, + "level_correct": 619, + "level_unknown": 57, + "awake_count_visible": 215, + "awake_count_correct": 202, + "awake_count_unknown": 13, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "level_wrong": 1, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.0, + "identity_precision": null, + "level_coverage": 0.914328, + "level_misreads": 1, + "awake_count_coverage": 0.939535, + "awake_count_misreads": 0, + "card_rank_coverage": 0.990919, + "card_rank_misreads": 0 + }, + "identity:heldout": { + "cards": 3649, + "located": 3646, + "kind_correct": 3646, + "identity_correct": 3353, + "card_rank_visible": 2760, + "card_rank_correct": 2736, + "card_rank_unknown": 24, + "identity_unknown": 293, + "level_visible": 642, + "level_correct": 601, + "level_unknown": 41, + "awake_count_visible": 231, + "awake_count_correct": 210, + "awake_count_unknown": 21, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "identity_recall": 0.918882, + "identity_precision": 1.0, + "level_coverage": 0.936137, + "level_misreads": 0, + "awake_count_coverage": 0.909091, + "awake_count_misreads": 0, + "card_rank_coverage": 0.991304, + "card_rank_misreads": 0 + }, + "identity:seen": { + "cards": 20955, + "located": 20945, + "kind_correct": 20945, + "identity_correct": 20464, + "card_rank_visible": 16057, + "card_rank_correct": 16000, + "card_rank_unknown": 57, + "identity_unknown": 481, + "level_visible": 3539, + "level_correct": 3407, + "level_unknown": 132, + "awake_count_visible": 1329, + "awake_count_correct": 1287, + "awake_count_unknown": 42, + "card_rank_hidden_emitted": 4, + "card_rank_hidden_latent_match": 4, + "level_hidden_emitted": 2, + "level_hidden_latent_match": 2, + "identity_recall": 0.976569, + "identity_precision": 1.0, + "level_coverage": 0.962701, + "level_misreads": 0, + "awake_count_coverage": 0.968397, + "awake_count_misreads": 0, + "card_rank_coverage": 0.99645, + "card_rank_misreads": 0 + }, + "kind:member": { + "cards": 14388, + "located": 14380, + "kind_correct": 14380, + "identity_correct": 12156, + "card_rank_visible": 11301, + "card_rank_correct": 11232, + "card_rank_unknown": 69, + "foreign_rejected": 1761, + "identity_unknown": 435, + "awake_count_visible": 1775, + "awake_count_correct": 1699, + "awake_count_unknown": 76, + "level_visible": 2206, + "level_correct": 2090, + "level_unknown": 116, + "foreign_false_accept": 28, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.964838, + "identity_precision": 1.0, + "level_coverage": 0.947416, + "level_misreads": 0, + "awake_count_coverage": 0.957183, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993894, + "card_rank_misreads": 0 + }, + "kind:snap": { + "cards": 13941, + "located": 13928, + "kind_correct": 13928, + "identity_correct": 11661, + "identity_unknown": 339, + "foreign_rejected": 1907, + "foreign_false_accept": 21, + "level_visible": 2652, + "level_correct": 2537, + "level_unknown": 114, + "card_rank_visible": 10379, + "card_rank_correct": 10341, + "card_rank_unknown": 38, + "level_hidden_emitted": 4, + "level_hidden_latent_match": 4, + "level_wrong": 1, + "card_rank_hidden_emitted": 4, + "card_rank_hidden_latent_match": 4, + "identity_recall": 0.970698, + "identity_precision": 1.0, + "level_coverage": 0.956637, + "level_misreads": 1, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996339, + "card_rank_misreads": 0 + }, + "locale:en": { + "cards": 5786, + "located": 5783, + "kind_correct": 5783, + "identity_correct": 4899, + "card_rank_visible": 4443, + "card_rank_correct": 4426, + "card_rank_unknown": 17, + "screens": 411, + "negative_screens": 16, + "extra_identified": 0, + "extra_unidentified": 1, + "foreign_rejected": 719, + "level_visible": 1205, + "level_correct": 1149, + "level_unknown": 56, + "awake_count_visible": 362, + "awake_count_correct": 319, + "awake_count_unknown": 43, + "identity_unknown": 156, + "foreign_false_accept": 9, + "identity_recall": 0.968565, + "identity_precision": 1.0, + "level_coverage": 0.953527, + "level_misreads": 0, + "awake_count_coverage": 0.881215, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996174, + "card_rank_misreads": 0 + }, + "locale:ja": { + "cards": 5491, + "located": 5484, + "kind_correct": 5484, + "identity_correct": 4604, + "level_visible": 698, + "level_correct": 670, + "level_unknown": 28, + "card_rank_visible": 4115, + "card_rank_correct": 4081, + "card_rank_unknown": 34, + "screens": 396, + "negative_screens": 18, + "extra_identified": 0, + "extra_unidentified": 0, + "foreign_rejected": 740, + "awake_count_visible": 304, + "awake_count_correct": 300, + "awake_count_unknown": 4, + "identity_unknown": 130, + "foreign_false_accept": 10, + "identity_recall": 0.971103, + "identity_precision": 1.0, + "level_coverage": 0.959885, + "level_misreads": 0, + "awake_count_coverage": 0.986842, + "awake_count_misreads": 0, + "card_rank_coverage": 0.991738, + "card_rank_misreads": 0 + }, + "locale:ko": { + "cards": 5103, + "located": 5099, + "kind_correct": 5099, + "identity_correct": 4331, + "card_rank_visible": 3893, + "card_rank_correct": 3877, + "card_rank_unknown": 16, + "foreign_rejected": 613, + "screens": 375, + "negative_screens": 20, + "extra_identified": 0, + "extra_unidentified": 0, + "identity_unknown": 147, + "foreign_false_accept": 8, + "level_visible": 807, + "level_correct": 775, + "level_unknown": 32, + "awake_count_visible": 235, + "awake_count_correct": 231, + "awake_count_unknown": 4, + "identity_recall": 0.96631, + "identity_precision": 1.0, + "level_coverage": 0.960347, + "level_misreads": 0, + "awake_count_coverage": 0.982979, + "awake_count_misreads": 0, + "card_rank_coverage": 0.99589, + "card_rank_misreads": 0 + }, + "locale:zh-Hans": { + "cards": 6131, + "located": 6125, + "kind_correct": 6125, + "identity_correct": 5099, + "card_rank_visible": 4828, + "card_rank_correct": 4807, + "card_rank_unknown": 21, + "identity_unknown": 172, + "screens": 417, + "negative_screens": 21, + "extra_identified": 0, + "extra_unidentified": 2, + "level_visible": 1056, + "level_correct": 994, + "level_unknown": 62, + "foreign_rejected": 844, + "awake_count_visible": 407, + "awake_count_correct": 393, + "awake_count_unknown": 14, + "foreign_false_accept": 10, + "level_hidden_emitted": 3, + "level_hidden_latent_match": 3, + "card_rank_hidden_emitted": 4, + "card_rank_hidden_latent_match": 4, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.966269, + "identity_precision": 1.0, + "level_coverage": 0.941288, + "level_misreads": 0, + "awake_count_coverage": 0.965602, + "awake_count_misreads": 0, + "card_rank_coverage": 0.99565, + "card_rank_misreads": 0 + }, + "locale:zh-Hant": { + "cards": 5818, + "located": 5817, + "kind_correct": 5817, + "identity_correct": 4884, + "card_rank_visible": 4401, + "card_rank_correct": 4382, + "card_rank_unknown": 19, + "foreign_rejected": 752, + "screens": 401, + "negative_screens": 24, + "extra_identified": 0, + "extra_unidentified": 3, + "level_visible": 1092, + "level_correct": 1039, + "level_unknown": 52, + "identity_unknown": 169, + "foreign_false_accept": 12, + "awake_count_visible": 467, + "awake_count_correct": 456, + "awake_count_unknown": 11, + "level_wrong": 1, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "identity_recall": 0.966363, + "identity_precision": 1.0, + "level_coverage": 0.951465, + "level_misreads": 1, + "awake_count_coverage": 0.976445, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995683, + "card_rank_misreads": 0 + }, + "mode:hide": { + "cards": 4700, + "located": 4691, + "kind_correct": 4691, + "identity_correct": 4003, + "identity_unknown": 94, + "foreign_rejected": 586, + "foreign_false_accept": 8, + "level_visible": 0, + "level_unknown": 0, + "card_rank_visible": 0, + "card_rank_unknown": 0, + "awake_count_visible": 0, + "awake_count_unknown": 0, + "identity_recall": 0.974915, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": null, + "card_rank_misreads": 0 + }, + "mode:level": { + "cards": 5325, + "located": 5325, + "kind_correct": 5325, + "identity_correct": 4436, + "level_visible": 4858, + "level_correct": 4627, + "level_unknown": 230, + "card_rank_visible": 4844, + "card_rank_correct": 4817, + "card_rank_unknown": 27, + "foreign_rejected": 740, + "identity_unknown": 142, + "level_hidden_emitted": 4, + "level_hidden_latent_match": 4, + "level_wrong": 1, + "foreign_false_accept": 7, + "card_rank_hidden_emitted": 1, + "card_rank_hidden_latent_match": 1, + "identity_recall": 0.968982, + "identity_precision": 1.0, + "level_coverage": 0.95245, + "level_misreads": 1, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.994426, + "card_rank_misreads": 0 + }, + "mode:performance": { + "cards": 4606, + "located": 4598, + "kind_correct": 4598, + "identity_correct": 3844, + "card_rank_visible": 4241, + "card_rank_correct": 4224, + "card_rank_unknown": 17, + "foreign_rejected": 607, + "level_visible": 0, + "level_unknown": 0, + "identity_unknown": 139, + "foreign_false_accept": 8, + "identity_recall": 0.963167, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995992, + "card_rank_misreads": 0 + }, + "mode:technic": { + "cards": 5276, + "located": 5275, + "kind_correct": 5275, + "identity_correct": 4482, + "card_rank_visible": 4869, + "card_rank_correct": 4845, + "card_rank_unknown": 24, + "foreign_rejected": 657, + "identity_unknown": 125, + "foreign_false_accept": 11, + "level_visible": 0, + "level_unknown": 0, + "card_rank_hidden_emitted": 3, + "card_rank_hidden_latent_match": 3, + "identity_recall": 0.972656, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995071, + "card_rank_misreads": 0 + }, + "mode:total": { + "cards": 1832, + "located": 1832, + "kind_correct": 1832, + "identity_correct": 1540, + "card_rank_visible": 1661, + "card_rank_correct": 1654, + "card_rank_unknown": 7, + "identity_unknown": 80, + "foreign_rejected": 208, + "foreign_false_accept": 4, + "identity_recall": 0.950617, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995786, + "card_rank_misreads": 0 + }, + "mode:training": { + "cards": 1927, + "located": 1927, + "kind_correct": 1927, + "identity_correct": 1636, + "card_rank_visible": 1763, + "card_rank_correct": 1749, + "card_rank_unknown": 14, + "awake_count_visible": 1775, + "awake_count_correct": 1699, + "awake_count_unknown": 76, + "foreign_rejected": 237, + "identity_unknown": 50, + "foreign_false_accept": 4, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.970344, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": 0.957183, + "awake_count_misreads": 0, + "card_rank_coverage": 0.992059, + "card_rank_misreads": 0 + }, + "mode:visual": { + "cards": 4663, + "located": 4660, + "kind_correct": 4660, + "identity_correct": 3876, + "card_rank_visible": 4302, + "card_rank_correct": 4284, + "card_rank_unknown": 18, + "foreign_rejected": 633, + "identity_unknown": 144, + "foreign_false_accept": 7, + "level_visible": 0, + "level_unknown": 0, + "awake_count_visible": 0, + "awake_count_unknown": 0, + "identity_recall": 0.96346, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995816, + "card_rank_misreads": 0 + }, + "occluded": { + "cards": 1101, + "located": 1101, + "kind_correct": 1101, + "identity_correct": 726, + "card_rank_visible": 837, + "card_rank_correct": 818, + "card_rank_unknown": 19, + "level_visible": 187, + "level_correct": 175, + "level_unknown": 12, + "foreign_rejected": 148, + "identity_unknown": 226, + "awake_count_visible": 67, + "awake_count_correct": 65, + "awake_count_unknown": 2, + "level_hidden_emitted": 2, + "level_hidden_latent_match": 2, + "foreign_false_accept": 1, + "identity_recall": 0.762605, + "identity_precision": 1.0, + "level_coverage": 0.935829, + "level_misreads": 0, + "awake_count_coverage": 0.970149, + "awake_count_misreads": 0, + "card_rank_coverage": 0.9773, + "card_rank_misreads": 0 + }, + "overall": { + "cards": 28329, + "located": 28308, + "kind_correct": 28308, + "identity_correct": 23817, + "card_rank_visible": 21680, + "card_rank_correct": 21573, + "card_rank_unknown": 107, + "foreign_rejected": 3668, + "screens": 2000, + "negative_screens": 99, + "extra_identified": 0, + "extra_unidentified": 6, + "identity_unknown": 774, + "foreign_false_accept": 49, + "level_visible": 4858, + "level_correct": 4627, + "level_unknown": 230, + "awake_count_visible": 1775, + "awake_count_correct": 1699, + "awake_count_unknown": 76, + "level_hidden_emitted": 4, + "level_hidden_latent_match": 4, + "level_wrong": 1, + "card_rank_hidden_emitted": 4, + "card_rank_hidden_latent_match": 4, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.967699, + "identity_precision": 1.0, + "level_coverage": 0.95245, + "level_misreads": 1, + "awake_count_coverage": 0.957183, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995065, + "card_rank_misreads": 0 + }, + "partly_outside": { + "cards": 3669, + "located": 3648, + "kind_correct": 3648, + "identity_correct": 2436, + "card_rank_visible": 1223, + "card_rank_correct": 1199, + "card_rank_unknown": 24, + "identity_unknown": 716, + "foreign_false_accept": 5, + "foreign_rejected": 491, + "level_visible": 285, + "level_unknown": 18, + "level_correct": 267, + "awake_count_visible": 102, + "awake_count_correct": 97, + "awake_count_unknown": 5, + "level_hidden_emitted": 2, + "level_hidden_latent_match": 2, + "identity_recall": 0.767728, + "identity_precision": 1.0, + "level_coverage": 0.936842, + "level_misreads": 0, + "awake_count_coverage": 0.95098, + "awake_count_misreads": 0, + "card_rank_coverage": 0.980376, + "card_rank_misreads": 0 + }, + "profile:final": { + "cards": 28329, + "located": 28308, + "kind_correct": 28308, + "identity_correct": 23817, + "card_rank_visible": 21680, + "card_rank_correct": 21573, + "card_rank_unknown": 107, + "foreign_rejected": 3668, + "screens": 2000, + "negative_screens": 99, + "extra_identified": 0, + "extra_unidentified": 6, + "identity_unknown": 774, + "foreign_false_accept": 49, + "level_visible": 4858, + "level_correct": 4627, + "level_unknown": 230, + "awake_count_visible": 1775, + "awake_count_correct": 1699, + "awake_count_unknown": 76, + "level_hidden_emitted": 4, + "level_hidden_latent_match": 4, + "level_wrong": 1, + "card_rank_hidden_emitted": 4, + "card_rank_hidden_latent_match": 4, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.967699, + "identity_precision": 1.0, + "level_coverage": 0.95245, + "level_misreads": 1, + "awake_count_coverage": 0.957183, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995065, + "card_rank_misreads": 0 + }, + "source_resolution:1334x750": { + "cards": 4732, + "located": 4732, + "kind_correct": 4732, + "identity_correct": 3936, + "level_visible": 873, + "level_correct": 820, + "level_unknown": 53, + "card_rank_visible": 3577, + "card_rank_correct": 3564, + "card_rank_unknown": 13, + "foreign_rejected": 629, + "awake_count_visible": 223, + "awake_count_correct": 212, + "awake_count_unknown": 11, + "identity_unknown": 162, + "foreign_false_accept": 5, + "identity_recall": 0.960469, + "identity_precision": 1.0, + "level_coverage": 0.93929, + "level_misreads": 0, + "awake_count_coverage": 0.950673, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996366, + "card_rank_misreads": 0 + }, + "source_resolution:1792x828": { + "cards": 5538, + "located": 5533, + "kind_correct": 5533, + "identity_correct": 4601, + "level_visible": 1137, + "level_correct": 1097, + "level_unknown": 39, + "card_rank_visible": 4162, + "card_rank_correct": 4143, + "card_rank_unknown": 19, + "foreign_rejected": 747, + "awake_count_visible": 259, + "awake_count_correct": 247, + "awake_count_unknown": 12, + "identity_unknown": 176, + "level_hidden_emitted": 3, + "level_hidden_latent_match": 3, + "foreign_false_accept": 9, + "level_wrong": 1, + "card_rank_hidden_emitted": 3, + "card_rank_hidden_latent_match": 3, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.96215, + "identity_precision": 1.0, + "level_coverage": 0.96482, + "level_misreads": 1, + "awake_count_coverage": 0.953668, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995435, + "card_rank_misreads": 0 + }, + "source_resolution:2224x1668": { + "cards": 6604, + "located": 6595, + "kind_correct": 6595, + "identity_correct": 5624, + "card_rank_visible": 5005, + "card_rank_correct": 4982, + "card_rank_unknown": 23, + "foreign_rejected": 804, + "level_visible": 1057, + "level_unknown": 46, + "identity_unknown": 154, + "awake_count_visible": 522, + "awake_count_correct": 501, + "awake_count_unknown": 21, + "level_correct": 1011, + "foreign_false_accept": 13, + "identity_recall": 0.971833, + "identity_precision": 1.0, + "level_coverage": 0.956481, + "level_misreads": 0, + "awake_count_coverage": 0.95977, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995405, + "card_rank_misreads": 0 + }, + "source_resolution:2388x1668": { + "cards": 6387, + "located": 6381, + "kind_correct": 6381, + "identity_correct": 5410, + "card_rank_visible": 4972, + "card_rank_correct": 4946, + "card_rank_unknown": 26, + "foreign_rejected": 815, + "awake_count_visible": 361, + "awake_count_correct": 343, + "awake_count_unknown": 18, + "level_visible": 1008, + "level_correct": 948, + "level_unknown": 60, + "identity_unknown": 142, + "foreign_false_accept": 14, + "identity_recall": 0.973372, + "identity_precision": 1.0, + "level_coverage": 0.940476, + "level_misreads": 0, + "awake_count_coverage": 0.950139, + "awake_count_misreads": 0, + "card_rank_coverage": 0.994771, + "card_rank_misreads": 0 + }, + "source_resolution:2400x1080": { + "cards": 5068, + "located": 5067, + "kind_correct": 5067, + "identity_correct": 4246, + "identity_unknown": 140, + "foreign_rejected": 673, + "foreign_false_accept": 8, + "card_rank_visible": 3964, + "card_rank_correct": 3938, + "card_rank_unknown": 26, + "awake_count_visible": 410, + "awake_count_correct": 396, + "awake_count_unknown": 14, + "level_visible": 783, + "level_correct": 751, + "level_unknown": 32, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "card_rank_hidden_emitted": 1, + "card_rank_hidden_latent_match": 1, + "identity_recall": 0.96786, + "identity_precision": 1.0, + "level_coverage": 0.959132, + "level_misreads": 0, + "awake_count_coverage": 0.965854, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993441, + "card_rank_misreads": 0 + } + }, + "errors": [ + { + "file": "000345.webp", + "card": 15, + "error": "level", + "expected": 6, + "actual": 61 + } + ], + "error_count": 1 + }, + "v2-final": { + "generator": "scenes-v2", + "profile": "final", + "seed": 20261104, + "truth_sha256": "6f78126d3708361bb4e79f458b89da8cff7b0f6cbfb07e51bdfd975353365ab3", + "screens": 2000, + "strata": { + "card_width_bin:120": { + "cards": 10327, + "located": 10318, + "kind_correct": 10318, + "identity_correct": 8732, + "card_rank_visible": 7935, + "card_rank_correct": 7891, + "card_rank_unknown": 44, + "foreign_rejected": 1300, + "level_visible": 1436, + "level_correct": 1382, + "level_unknown": 54, + "identity_unknown": 271, + "awake_count_visible": 912, + "awake_count_correct": 889, + "awake_count_unknown": 23, + "level_hidden_emitted": 2, + "level_hidden_latent_match": 2, + "foreign_false_accept": 15, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.96893, + "identity_precision": 1.0, + "level_coverage": 0.962396, + "level_misreads": 0, + "awake_count_coverage": 0.974781, + "awake_count_misreads": 0, + "card_rank_coverage": 0.994455, + "card_rank_misreads": 0 + }, + "card_width_bin:180": { + "cards": 8029, + "located": 8025, + "kind_correct": 8025, + "identity_correct": 6811, + "level_visible": 1349, + "level_correct": 1300, + "level_unknown": 49, + "card_rank_visible": 6201, + "card_rank_correct": 6171, + "card_rank_unknown": 30, + "foreign_rejected": 1005, + "foreign_false_accept": 7, + "identity_unknown": 202, + "awake_count_visible": 634, + "awake_count_correct": 610, + "awake_count_unknown": 24, + "level_hidden_emitted": 7, + "level_hidden_latent_match": 7, + "identity_recall": 0.970643, + "identity_precision": 1.0, + "level_coverage": 0.963677, + "level_misreads": 0, + "awake_count_coverage": 0.962145, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995162, + "card_rank_misreads": 0 + }, + "card_width_bin:240": { + "cards": 3969, + "located": 3966, + "kind_correct": 3966, + "identity_correct": 3427, + "card_rank_visible": 3234, + "card_rank_correct": 3223, + "card_rank_unknown": 11, + "identity_unknown": 65, + "foreign_rejected": 470, + "level_visible": 912, + "level_correct": 877, + "level_unknown": 35, + "awake_count_visible": 145, + "awake_count_correct": 142, + "awake_count_unknown": 3, + "foreign_false_accept": 4, + "identity_recall": 0.980544, + "identity_precision": 1.0, + "level_coverage": 0.961623, + "level_misreads": 0, + "awake_count_coverage": 0.97931, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996599, + "card_rank_misreads": 0 + }, + "card_width_bin:300": { + "cards": 1525, + "located": 1520, + "kind_correct": 1520, + "identity_correct": 1308, + "level_visible": 413, + "level_correct": 390, + "level_unknown": 23, + "card_rank_visible": 1112, + "card_rank_correct": 1110, + "card_rank_unknown": 2, + "foreign_rejected": 186, + "identity_unknown": 23, + "foreign_false_accept": 3, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "identity_recall": 0.979042, + "identity_precision": 1.0, + "level_coverage": 0.94431, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.998201, + "card_rank_misreads": 0 + }, + "card_width_bin:360": { + "cards": 153, + "located": 153, + "kind_correct": 153, + "identity_unknown": 8, + "level_visible": 24, + "level_correct": 21, + "level_unknown": 3, + "card_rank_visible": 104, + "card_rank_correct": 104, + "card_rank_unknown": 0, + "identity_correct": 126, + "foreign_rejected": 19, + "identity_recall": 0.940299, + "identity_precision": 1.0, + "level_coverage": 0.875, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "card_width_bin:60": { + "cards": 3895, + "located": 3893, + "kind_correct": 3893, + "identity_correct": 3273, + "card_rank_visible": 3001, + "card_rank_correct": 2981, + "card_rank_unknown": 20, + "foreign_rejected": 478, + "identity_unknown": 139, + "level_visible": 577, + "level_correct": 515, + "level_unknown": 62, + "awake_count_visible": 372, + "awake_count_correct": 319, + "awake_count_unknown": 53, + "foreign_false_accept": 3, + "identity_recall": 0.958699, + "identity_precision": 1.0, + "level_coverage": 0.892548, + "level_misreads": 0, + "awake_count_coverage": 0.857527, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993336, + "card_rank_misreads": 0 + }, + "foreign:False": { + "cards": 24405, + "located": 24385, + "kind_correct": 24385, + "identity_correct": 23677, + "card_rank_visible": 18895, + "card_rank_correct": 18817, + "card_rank_unknown": 78, + "level_visible": 4100, + "level_correct": 3946, + "level_unknown": 154, + "identity_unknown": 708, + "awake_count_visible": 1789, + "awake_count_correct": 1718, + "awake_count_unknown": 71, + "level_hidden_emitted": 10, + "level_hidden_latent_match": 10, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.97017, + "identity_precision": 1.0, + "level_coverage": 0.962439, + "level_misreads": 0, + "awake_count_coverage": 0.960313, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995872, + "card_rank_misreads": 0 + }, + "foreign:True": { + "cards": 3493, + "located": 3490, + "kind_correct": 3490, + "foreign_rejected": 3458, + "card_rank_visible": 2692, + "card_rank_correct": 2663, + "card_rank_unknown": 29, + "level_visible": 611, + "level_correct": 539, + "level_unknown": 72, + "foreign_false_accept": 32, + "awake_count_visible": 274, + "awake_count_correct": 242, + "awake_count_unknown": 32, + "identity_recall": 0.0, + "identity_precision": null, + "level_coverage": 0.88216, + "level_misreads": 0, + "awake_count_coverage": 0.883212, + "awake_count_misreads": 0, + "card_rank_coverage": 0.989227, + "card_rank_misreads": 0 + }, + "identity:heldout": { + "cards": 3573, + "located": 3571, + "kind_correct": 3571, + "identity_correct": 3321, + "card_rank_visible": 2804, + "card_rank_correct": 2791, + "card_rank_unknown": 13, + "level_visible": 617, + "level_correct": 582, + "level_unknown": 35, + "identity_unknown": 250, + "awake_count_visible": 242, + "awake_count_correct": 220, + "awake_count_unknown": 22, + "level_hidden_emitted": 3, + "level_hidden_latent_match": 3, + "identity_recall": 0.929471, + "identity_precision": 1.0, + "level_coverage": 0.943274, + "level_misreads": 0, + "awake_count_coverage": 0.909091, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995364, + "card_rank_misreads": 0 + }, + "identity:seen": { + "cards": 20832, + "located": 20814, + "kind_correct": 20814, + "identity_correct": 20356, + "card_rank_visible": 16091, + "card_rank_correct": 16026, + "card_rank_unknown": 65, + "level_visible": 3483, + "level_correct": 3364, + "level_unknown": 119, + "identity_unknown": 458, + "awake_count_visible": 1547, + "awake_count_correct": 1498, + "awake_count_unknown": 49, + "level_hidden_emitted": 7, + "level_hidden_latent_match": 7, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.977151, + "identity_precision": 1.0, + "level_coverage": 0.965834, + "level_misreads": 0, + "awake_count_coverage": 0.968326, + "awake_count_misreads": 0, + "card_rank_coverage": 0.99596, + "card_rank_misreads": 0 + }, + "kind:member": { + "cards": 14068, + "located": 14058, + "kind_correct": 14058, + "identity_correct": 11819, + "card_rank_visible": 11149, + "card_rank_correct": 11082, + "card_rank_unknown": 67, + "foreign_rejected": 1802, + "identity_unknown": 418, + "level_visible": 1919, + "level_correct": 1794, + "level_unknown": 125, + "awake_count_visible": 2063, + "awake_count_correct": 1960, + "awake_count_unknown": 103, + "foreign_false_accept": 19, + "level_hidden_emitted": 3, + "level_hidden_latent_match": 3, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.965053, + "identity_precision": 1.0, + "level_coverage": 0.934862, + "level_misreads": 0, + "awake_count_coverage": 0.950073, + "awake_count_misreads": 0, + "card_rank_coverage": 0.99399, + "card_rank_misreads": 0 + }, + "kind:snap": { + "cards": 13830, + "located": 13817, + "kind_correct": 13817, + "identity_correct": 11858, + "level_visible": 2792, + "level_correct": 2691, + "level_unknown": 101, + "card_rank_visible": 10438, + "card_rank_correct": 10398, + "card_rank_unknown": 40, + "foreign_rejected": 1656, + "foreign_false_accept": 13, + "identity_unknown": 290, + "level_hidden_emitted": 7, + "level_hidden_latent_match": 7, + "identity_recall": 0.975084, + "identity_precision": 1.0, + "level_coverage": 0.963825, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996168, + "card_rank_misreads": 0 + }, + "locale:en": { + "cards": 5858, + "located": 5846, + "kind_correct": 5846, + "identity_correct": 4933, + "card_rank_visible": 4570, + "card_rank_correct": 4541, + "card_rank_unknown": 29, + "foreign_rejected": 766, + "screens": 413, + "negative_screens": 20, + "extra_identified": 0, + "extra_unidentified": 0, + "level_visible": 982, + "level_correct": 904, + "level_unknown": 78, + "identity_unknown": 142, + "awake_count_visible": 420, + "awake_count_correct": 342, + "awake_count_unknown": 78, + "foreign_false_accept": 5, + "identity_recall": 0.969727, + "identity_precision": 1.0, + "level_coverage": 0.92057, + "level_misreads": 0, + "awake_count_coverage": 0.814286, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993654, + "card_rank_misreads": 0 + }, + "locale:ja": { + "cards": 5183, + "located": 5178, + "kind_correct": 5178, + "identity_correct": 4384, + "level_visible": 890, + "level_correct": 855, + "level_unknown": 35, + "card_rank_visible": 3940, + "card_rank_correct": 3925, + "card_rank_unknown": 15, + "foreign_rejected": 636, + "foreign_false_accept": 7, + "screens": 387, + "negative_screens": 28, + "extra_identified": 0, + "extra_unidentified": 1, + "awake_count_visible": 294, + "awake_count_correct": 289, + "awake_count_unknown": 5, + "identity_unknown": 151, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "identity_recall": 0.965639, + "identity_precision": 1.0, + "level_coverage": 0.960674, + "level_misreads": 0, + "awake_count_coverage": 0.982993, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996193, + "card_rank_misreads": 0 + }, + "locale:ko": { + "cards": 5733, + "located": 5732, + "kind_correct": 5732, + "identity_correct": 4815, + "card_rank_visible": 4505, + "card_rank_correct": 4480, + "card_rank_unknown": 25, + "identity_unknown": 137, + "foreign_rejected": 770, + "screens": 388, + "negative_screens": 22, + "extra_identified": 0, + "extra_unidentified": 1, + "awake_count_visible": 454, + "awake_count_correct": 445, + "awake_count_unknown": 9, + "level_visible": 929, + "level_correct": 887, + "level_unknown": 42, + "foreign_false_accept": 10, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.972138, + "identity_precision": 1.0, + "level_coverage": 0.95479, + "level_misreads": 0, + "awake_count_coverage": 0.980176, + "awake_count_misreads": 0, + "card_rank_coverage": 0.994451, + "card_rank_misreads": 0 + }, + "locale:zh-Hans": { + "cards": 5953, + "located": 5949, + "kind_correct": 5949, + "identity_correct": 5110, + "card_rank_visible": 4550, + "card_rank_correct": 4538, + "card_rank_unknown": 12, + "screens": 420, + "negative_screens": 15, + "extra_identified": 0, + "extra_unidentified": 0, + "foreign_rejected": 681, + "level_visible": 1077, + "level_correct": 1039, + "level_unknown": 38, + "identity_unknown": 154, + "awake_count_visible": 454, + "awake_count_correct": 446, + "awake_count_unknown": 8, + "foreign_false_accept": 4, + "level_hidden_emitted": 7, + "level_hidden_latent_match": 7, + "identity_recall": 0.970008, + "identity_precision": 1.0, + "level_coverage": 0.964717, + "level_misreads": 0, + "awake_count_coverage": 0.982379, + "awake_count_misreads": 0, + "card_rank_coverage": 0.997363, + "card_rank_misreads": 0 + }, + "locale:zh-Hant": { + "cards": 5171, + "located": 5170, + "kind_correct": 5170, + "foreign_rejected": 605, + "card_rank_visible": 4022, + "card_rank_correct": 3996, + "card_rank_unknown": 26, + "screens": 392, + "negative_screens": 21, + "extra_identified": 0, + "extra_unidentified": 0, + "identity_correct": 4435, + "identity_unknown": 124, + "awake_count_visible": 441, + "awake_count_correct": 438, + "awake_count_unknown": 3, + "level_visible": 833, + "level_correct": 800, + "level_unknown": 33, + "foreign_false_accept": 6, + "level_hidden_emitted": 2, + "level_hidden_latent_match": 2, + "identity_recall": 0.972588, + "identity_precision": 1.0, + "level_coverage": 0.960384, + "level_misreads": 0, + "awake_count_coverage": 0.993197, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993536, + "card_rank_misreads": 0 + }, + "mode:hide": { + "cards": 4556, + "located": 4554, + "kind_correct": 4554, + "identity_correct": 3881, + "identity_unknown": 88, + "foreign_rejected": 578, + "foreign_false_accept": 7, + "level_visible": 0, + "level_unknown": 0, + "card_rank_visible": 0, + "card_rank_unknown": 0, + "awake_count_visible": 0, + "awake_count_unknown": 0, + "identity_recall": 0.977336, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": null, + "card_rank_misreads": 0 + }, + "mode:level": { + "cards": 5069, + "located": 5068, + "kind_correct": 5068, + "identity_correct": 4296, + "level_visible": 4711, + "level_correct": 4485, + "level_unknown": 226, + "card_rank_visible": 4711, + "card_rank_correct": 4682, + "card_rank_unknown": 29, + "foreign_rejected": 633, + "foreign_false_accept": 8, + "identity_unknown": 131, + "level_hidden_emitted": 10, + "level_hidden_latent_match": 10, + "awake_count_visible": 0, + "awake_count_unknown": 0, + "identity_recall": 0.97019, + "identity_precision": 1.0, + "level_coverage": 0.952027, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993844, + "card_rank_misreads": 0 + }, + "mode:performance": { + "cards": 4511, + "located": 4511, + "kind_correct": 4511, + "identity_correct": 3811, + "card_rank_visible": 4176, + "card_rank_correct": 4162, + "card_rank_unknown": 14, + "foreign_rejected": 567, + "identity_unknown": 130, + "foreign_false_accept": 3, + "identity_recall": 0.967013, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996648, + "card_rank_misreads": 0 + }, + "mode:technic": { + "cards": 4891, + "located": 4884, + "kind_correct": 4884, + "identity_correct": 4151, + "card_rank_visible": 4513, + "card_rank_correct": 4504, + "card_rank_unknown": 9, + "identity_unknown": 115, + "foreign_rejected": 614, + "foreign_false_accept": 4, + "level_visible": 0, + "level_unknown": 0, + "awake_count_visible": 0, + "awake_count_unknown": 0, + "identity_recall": 0.971449, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.998006, + "card_rank_misreads": 0 + }, + "mode:total": { + "cards": 2107, + "located": 2103, + "kind_correct": 2103, + "identity_correct": 1777, + "card_rank_visible": 1936, + "card_rank_correct": 1927, + "card_rank_unknown": 9, + "foreign_rejected": 250, + "identity_unknown": 72, + "foreign_false_accept": 4, + "level_visible": 0, + "level_unknown": 0, + "awake_count_visible": 0, + "awake_count_unknown": 0, + "identity_recall": 0.958985, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995351, + "card_rank_misreads": 0 + }, + "mode:training": { + "cards": 2240, + "located": 2238, + "kind_correct": 2238, + "identity_correct": 1884, + "card_rank_visible": 2064, + "card_rank_correct": 2046, + "card_rank_unknown": 18, + "awake_count_visible": 2063, + "awake_count_correct": 1960, + "awake_count_unknown": 103, + "foreign_rejected": 301, + "identity_unknown": 52, + "foreign_false_accept": 1, + "level_visible": 0, + "level_unknown": 0, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.972136, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": 0.950073, + "awake_count_misreads": 0, + "card_rank_coverage": 0.991279, + "card_rank_misreads": 0 + }, + "mode:visual": { + "cards": 4524, + "located": 4517, + "kind_correct": 4517, + "identity_correct": 3877, + "card_rank_visible": 4187, + "card_rank_correct": 4159, + "card_rank_unknown": 28, + "foreign_rejected": 515, + "identity_unknown": 120, + "level_visible": 0, + "level_unknown": 0, + "foreign_false_accept": 5, + "awake_count_visible": 0, + "awake_count_unknown": 0, + "identity_recall": 0.968282, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 0.993313, + "card_rank_misreads": 0 + }, + "occluded": { + "cards": 1068, + "located": 1068, + "kind_correct": 1068, + "identity_correct": 704, + "card_rank_visible": 824, + "card_rank_correct": 809, + "card_rank_unknown": 15, + "foreign_rejected": 121, + "identity_unknown": 242, + "level_visible": 160, + "level_correct": 153, + "level_unknown": 7, + "awake_count_visible": 87, + "awake_count_correct": 82, + "awake_count_unknown": 5, + "level_hidden_emitted": 4, + "level_hidden_latent_match": 4, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "foreign_false_accept": 1, + "identity_recall": 0.744186, + "identity_precision": 1.0, + "level_coverage": 0.95625, + "level_misreads": 0, + "awake_count_coverage": 0.942529, + "awake_count_misreads": 0, + "card_rank_coverage": 0.981796, + "card_rank_misreads": 0 + }, + "overall": { + "cards": 27898, + "located": 27875, + "kind_correct": 27875, + "identity_correct": 23677, + "card_rank_visible": 21587, + "card_rank_correct": 21480, + "card_rank_unknown": 107, + "foreign_rejected": 3458, + "screens": 2000, + "negative_screens": 106, + "extra_identified": 0, + "extra_unidentified": 2, + "level_visible": 4711, + "level_correct": 4485, + "level_unknown": 226, + "foreign_false_accept": 32, + "identity_unknown": 708, + "awake_count_visible": 2063, + "awake_count_correct": 1960, + "awake_count_unknown": 103, + "level_hidden_emitted": 10, + "level_hidden_latent_match": 10, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.970051, + "identity_precision": 1.0, + "level_coverage": 0.952027, + "level_misreads": 0, + "awake_count_coverage": 0.950073, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995043, + "card_rank_misreads": 0 + }, + "partly_outside": { + "cards": 3431, + "located": 3408, + "kind_correct": 3408, + "identity_correct": 2325, + "card_rank_visible": 1219, + "card_rank_correct": 1193, + "card_rank_unknown": 26, + "foreign_rejected": 407, + "identity_unknown": 674, + "level_visible": 218, + "level_correct": 207, + "level_unknown": 11, + "awake_count_visible": 138, + "awake_count_correct": 129, + "awake_count_unknown": 9, + "level_hidden_emitted": 6, + "level_hidden_latent_match": 6, + "foreign_false_accept": 2, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.769358, + "identity_precision": 1.0, + "level_coverage": 0.949541, + "level_misreads": 0, + "awake_count_coverage": 0.934783, + "awake_count_misreads": 0, + "card_rank_coverage": 0.978671, + "card_rank_misreads": 0 + }, + "profile:final": { + "cards": 27898, + "located": 27875, + "kind_correct": 27875, + "identity_correct": 23677, + "card_rank_visible": 21587, + "card_rank_correct": 21480, + "card_rank_unknown": 107, + "foreign_rejected": 3458, + "screens": 2000, + "negative_screens": 106, + "extra_identified": 0, + "extra_unidentified": 2, + "level_visible": 4711, + "level_correct": 4485, + "level_unknown": 226, + "foreign_false_accept": 32, + "identity_unknown": 708, + "awake_count_visible": 2063, + "awake_count_correct": 1960, + "awake_count_unknown": 103, + "level_hidden_emitted": 10, + "level_hidden_latent_match": 10, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.970051, + "identity_precision": 1.0, + "level_coverage": 0.952027, + "level_misreads": 0, + "awake_count_coverage": 0.950073, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995043, + "card_rank_misreads": 0 + }, + "source_resolution:1334x750": { + "cards": 4743, + "located": 4743, + "kind_correct": 4743, + "identity_correct": 3987, + "card_rank_visible": 3595, + "card_rank_correct": 3566, + "card_rank_unknown": 29, + "identity_unknown": 158, + "foreign_rejected": 594, + "level_visible": 827, + "level_correct": 758, + "level_unknown": 69, + "awake_count_visible": 250, + "awake_count_correct": 205, + "awake_count_unknown": 45, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "foreign_false_accept": 4, + "identity_recall": 0.961882, + "identity_precision": 1.0, + "level_coverage": 0.916566, + "level_misreads": 0, + "awake_count_coverage": 0.82, + "awake_count_misreads": 0, + "card_rank_coverage": 0.991933, + "card_rank_misreads": 0 + }, + "source_resolution:1792x828": { + "cards": 5065, + "located": 5058, + "kind_correct": 5058, + "identity_correct": 4345, + "card_rank_visible": 3787, + "card_rank_correct": 3771, + "card_rank_unknown": 16, + "foreign_rejected": 570, + "level_visible": 735, + "level_correct": 717, + "level_unknown": 18, + "identity_unknown": 140, + "awake_count_visible": 272, + "awake_count_correct": 257, + "awake_count_unknown": 15, + "foreign_false_accept": 3, + "level_hidden_emitted": 4, + "level_hidden_latent_match": 4, + "identity_recall": 0.967275, + "identity_precision": 1.0, + "level_coverage": 0.97551, + "level_misreads": 0, + "awake_count_coverage": 0.944853, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995775, + "card_rank_misreads": 0 + }, + "source_resolution:2224x1668": { + "cards": 7109, + "located": 7102, + "kind_correct": 7102, + "identity_correct": 6072, + "level_visible": 1365, + "level_correct": 1308, + "level_unknown": 57, + "card_rank_visible": 5708, + "card_rank_correct": 5687, + "card_rank_unknown": 21, + "foreign_rejected": 886, + "foreign_false_accept": 9, + "identity_unknown": 135, + "awake_count_visible": 636, + "awake_count_correct": 619, + "awake_count_unknown": 17, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "identity_recall": 0.977148, + "identity_precision": 1.0, + "level_coverage": 0.958242, + "level_misreads": 0, + "awake_count_coverage": 0.97327, + "awake_count_misreads": 0, + "card_rank_coverage": 0.996321, + "card_rank_misreads": 0 + }, + "source_resolution:2388x1668": { + "cards": 5982, + "located": 5977, + "kind_correct": 5977, + "identity_correct": 5069, + "card_rank_visible": 4598, + "card_rank_correct": 4576, + "card_rank_unknown": 22, + "identity_unknown": 141, + "foreign_rejected": 759, + "level_visible": 1048, + "level_correct": 999, + "level_unknown": 49, + "awake_count_visible": 604, + "awake_count_correct": 584, + "awake_count_unknown": 20, + "foreign_false_accept": 8, + "level_hidden_emitted": 3, + "level_hidden_latent_match": 3, + "awake_count_hidden_emitted": 1, + "awake_count_hidden_latent_match": 1, + "identity_recall": 0.972004, + "identity_precision": 1.0, + "level_coverage": 0.953244, + "level_misreads": 0, + "awake_count_coverage": 0.966887, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995215, + "card_rank_misreads": 0 + }, + "source_resolution:2400x1080": { + "cards": 4999, + "located": 4995, + "kind_correct": 4995, + "identity_correct": 4204, + "card_rank_visible": 3899, + "card_rank_correct": 3880, + "card_rank_unknown": 19, + "foreign_rejected": 649, + "identity_unknown": 134, + "awake_count_visible": 301, + "awake_count_correct": 295, + "awake_count_unknown": 6, + "level_visible": 736, + "level_correct": 703, + "level_unknown": 33, + "foreign_false_accept": 8, + "level_hidden_emitted": 1, + "level_hidden_latent_match": 1, + "identity_recall": 0.968217, + "identity_precision": 1.0, + "level_coverage": 0.955163, + "level_misreads": 0, + "awake_count_coverage": 0.980066, + "awake_count_misreads": 0, + "card_rank_coverage": 0.995127, + "card_rank_misreads": 0 + } + }, + "errors": [], + "error_count": 0 + } + } +} diff --git a/reports/acceptance/real-sample-regression.json b/reports/acceptance/real-sample-regression.json index f7f8a8f..cf6fca6 100644 --- a/reports/acceptance/real-sample-regression.json +++ b/reports/acceptance/real-sample-regression.json @@ -1,226 +1,200 @@ -{ - "screenshots": 10, - "expected_cards": 156, - "identified": 156, - "extra_predictions": 0, - "level": { - "correct": 20, - "total": 21 - }, - "card_rank": { - "correct": 120, - "total": 120 - }, - "awake_count": { - "correct": 18, - "total": 18 - }, - "hidden_field_predictions": 0, - "latency_ms": { - "median": 1806.395, - "p95": 1953.9015 - }, - "opencv_threads": 2, - "cpu_affinity": null, - "platform": "Windows-10-10.0.19044-SP0", - "matching": "identity and bbox IoU >= 0.5; ID-only when calibration truth lacks boxes", - "strata": { - "overall": { - "screenshots": 10, - "extra_predictions": 0, - "expected_cards": 156, - "identified": 156, - "card_rank_total": 120, - "card_rank_correct": 120, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 21, - "level_correct": 20, - "level_wrong": 0, - "level_unknown": 1 - }, - "profile:legacy": { - "screenshots": 10, - "extra_predictions": 0, - "expected_cards": 156, - "identified": 156, - "card_rank_total": 120, - "card_rank_correct": 120, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 21, - "level_correct": 20, - "level_wrong": 0, - "level_unknown": 1 - }, - "locale:zh-Hans": { - "screenshots": 10, - "extra_predictions": 0, - "expected_cards": 156, - "identified": 156, - "card_rank_total": 120, - "card_rank_correct": 120, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 21, - "level_correct": 20, - "level_wrong": 0, - "level_unknown": 1 - }, - "screen_resolution:[1441, 1080]": { - "screenshots": 10, - "extra_predictions": 0, - "expected_cards": 156, - "identified": 156, - "card_rank_total": 120, - "card_rank_correct": 120, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 21, - "level_correct": 20, - "level_wrong": 0, - "level_unknown": 1 - }, - "upload_long_edge_bin:1440": { - "screenshots": 10, - "extra_predictions": 0, - "expected_cards": 156, - "identified": 156, - "card_rank_total": 120, - "card_rank_correct": 120, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 21, - "level_correct": 20, - "level_wrong": 0, - "level_unknown": 1 - }, - "compression_rounds:0": { - "screenshots": 10, - "extra_predictions": 0, - "expected_cards": 156, - "identified": 156, - "card_rank_total": 120, - "card_rank_correct": 120, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 21, - "level_correct": 20, - "level_wrong": 0, - "level_unknown": 1 - }, - "kind:member": { - "expected_cards": 144, - "identified": 144, - "card_rank_total": 108, - "card_rank_correct": 108, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 18, - "level_correct": 17, - "level_wrong": 0, - "level_unknown": 1 - }, - "identity_split:seen": { - "expected_cards": 134, - "identified": 134, - "card_rank_total": 104, - "card_rank_correct": 104, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 16, - "awake_count_correct": 16, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 18, - "level_correct": 17, - "level_wrong": 0, - "level_unknown": 1 - }, - "card_width_bin:150": { - "expected_cards": 144, - "identified": 144, - "card_rank_total": 108, - "card_rank_correct": 108, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 18, - "awake_count_correct": 18, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 18, - "level_correct": 17, - "level_wrong": 0, - "level_unknown": 1 - }, - "identity_split:heldout": { - "expected_cards": 22, - "identified": 22, - "card_rank_total": 16, - "card_rank_correct": 16, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "awake_count_total": 2, - "awake_count_correct": 2, - "awake_count_wrong": 0, - "awake_count_unknown": 0, - "level_total": 3, - "level_correct": 3, - "level_wrong": 0, - "level_unknown": 0 - }, - "kind:snap": { - "expected_cards": 12, - "identified": 12, - "card_rank_total": 12, - "card_rank_correct": 12, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "level_total": 3, - "level_correct": 3, - "level_wrong": 0, - "level_unknown": 0 - }, - "card_width_bin:240": { - "expected_cards": 12, - "identified": 12, - "card_rank_total": 12, - "card_rank_correct": 12, - "card_rank_wrong": 0, - "card_rank_unknown": 0, - "level_total": 3, - "level_correct": 3, - "level_wrong": 0, - "level_unknown": 0 - } - }, - "scope": "Ten calibration/regression images of one card collection, not independent real-world generalization." -} \ No newline at end of file +{ + "schema": "ournotes-boxlens.engine-evaluation-summary/1", + "release": "0.2.0", + "scope": "Eight screenshots of one player collection on one device (member and Snap lists, level and training views). Used for regression, not as an independent accuracy estimate; the images are not distributed.", + "matching": "one-to-one tile pairing at IoU >= 0.5; cards and unidentified outputs both pair", + "model_manifest_sha256": "bd726ca5a5f247828e80c9aba9d0a5c1a32b5a5ae5c063c3417802e15b77b716", + "screens": 8, + "strata": { + "identity:heldout": { + "cards": 18, + "located": 18, + "kind_correct": 18, + "identity_correct": 18, + "card_rank_visible": 13, + "card_rank_correct": 13, + "card_rank_unknown": 0, + "awake_count_visible": 2, + "awake_count_correct": 2, + "awake_count_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": 1.0, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "identity:seen": { + "cards": 111, + "located": 111, + "kind_correct": 111, + "identity_correct": 111, + "card_rank_visible": 86, + "card_rank_correct": 86, + "card_rank_unknown": 0, + "awake_count_visible": 16, + "awake_count_correct": 16, + "awake_count_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": 1.0, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "kind:member": { + "cards": 120, + "located": 120, + "kind_correct": 120, + "identity_correct": 120, + "card_rank_visible": 90, + "card_rank_correct": 90, + "card_rank_unknown": 0, + "awake_count_visible": 18, + "awake_count_correct": 18, + "awake_count_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": 1.0, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "kind:snap": { + "cards": 9, + "located": 9, + "kind_correct": 9, + "identity_correct": 9, + "card_rank_visible": 9, + "card_rank_correct": 9, + "card_rank_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "mode:performance": { + "cards": 27, + "located": 27, + "kind_correct": 27, + "identity_correct": 27, + "card_rank_visible": 21, + "card_rank_correct": 21, + "card_rank_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "mode:technic": { + "cards": 27, + "located": 27, + "kind_correct": 27, + "identity_correct": 27, + "card_rank_visible": 21, + "card_rank_correct": 21, + "card_rank_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "mode:total": { + "cards": 24, + "located": 24, + "kind_correct": 24, + "identity_correct": 24, + "card_rank_visible": 18, + "card_rank_correct": 18, + "card_rank_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "mode:training": { + "cards": 24, + "located": 24, + "kind_correct": 24, + "identity_correct": 24, + "card_rank_visible": 18, + "card_rank_correct": 18, + "card_rank_unknown": 0, + "awake_count_visible": 18, + "awake_count_correct": 18, + "awake_count_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": 1.0, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "mode:visual": { + "cards": 27, + "located": 27, + "kind_correct": 27, + "identity_correct": 27, + "card_rank_visible": 21, + "card_rank_correct": 21, + "card_rank_unknown": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": null, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + }, + "overall": { + "cards": 129, + "located": 129, + "kind_correct": 129, + "identity_correct": 129, + "card_rank_visible": 99, + "card_rank_correct": 99, + "card_rank_unknown": 0, + "awake_count_visible": 18, + "awake_count_correct": 18, + "awake_count_unknown": 0, + "screens": 8, + "negative_screens": 0, + "extra_identified": 0, + "extra_unidentified": 0, + "identity_recall": 1.0, + "identity_precision": 1.0, + "level_coverage": null, + "level_misreads": 0, + "awake_count_coverage": 1.0, + "awake_count_misreads": 0, + "card_rank_coverage": 1.0, + "card_rank_misreads": 0 + } + }, + "error_count": 0 +} diff --git a/reports/acceptance/synthetic-data-audit.json b/reports/acceptance/synthetic-data-audit.json deleted file mode 100644 index 1900876..0000000 --- a/reports/acceptance/synthetic-data-audit.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "schema": "bdon-data-audit/1", - "passed": true, - "datasets": [ - { - "dataset": "synthetic/scale-v6/train", - "profile": "train", - "seed": 50021, - "manifest_sha256": "bbb1b2aea52b1bd04f6bd6b09f6130f82111cd825c7b94493f339716826d353e", - "counts": { - "screenshots": 2048, - "legacy_hash_computed_from_image": 2048, - "cards": 35483, - "mode:level": 35483 - }, - "locales": { - "zh-Hans": 2048 - }, - "identities": 105 - }, - { - "dataset": "synthetic/scale-v6/validation", - "profile": "validation", - "seed": 60017, - "manifest_sha256": "ec7ca137870ebbd47c4bd4c988ae3a1809712ed9b8f8c08f0b160d158b58d8c4", - "counts": { - "screenshots": 512, - "legacy_hash_computed_from_image": 512, - "cards": 8887, - "mode:level": 8887 - }, - "locales": { - "zh-Hans": 512 - }, - "identities": 122 - }, - { - "dataset": "synthetic/scale-v6/stress", - "profile": "stress", - "seed": 70019, - "manifest_sha256": "92349a86c6ad6420249ba9e538c4deae7264801b2f0d77ed5622ba88c6bf54c0", - "counts": { - "screenshots": 256, - "legacy_hash_computed_from_image": 256, - "cards": 4273, - "mode:level": 4273 - }, - "locales": { - "zh-Hans": 256 - }, - "identities": 122 - }, - { - "dataset": "synthetic/modes-v7/train", - "profile": "train", - "seed": 91703, - "manifest_sha256": "ceb7bc7c1fe2e634de8633455bc80867fa14007bcb0d3e60201cf8a87004228d", - "counts": { - "screenshots": 350, - "legacy_hash_computed_from_image": 350, - "cards": 5935, - "mode:level": 1057, - "mode:training": 456, - "mode:performance": 939, - "mode:total": 458, - "mode:technic": 984, - "mode:visual": 1003, - "mode:hide": 1038 - }, - "locales": { - "ja": 70, - "en": 70, - "zh-Hant": 70, - "zh-Hans": 70, - "ko": 70 - }, - "identities": 105 - }, - { - "dataset": "synthetic/modes-v7/validation", - "profile": "validation", - "seed": 92711, - "manifest_sha256": "521b9bd19836ed2333cfc115018a35b3f2642e63ecec39a5e257f64406fe7138", - "counts": { - "screenshots": 140, - "legacy_hash_computed_from_image": 140, - "cards": 2344, - "mode:level": 435, - "mode:training": 176, - "mode:performance": 370, - "mode:total": 173, - "mode:technic": 421, - "mode:visual": 395, - "mode:hide": 374 - }, - "locales": { - "ja": 28, - "en": 28, - "zh-Hant": 28, - "zh-Hans": 28, - "ko": 28 - }, - "identities": 122 - }, - { - "dataset": "synthetic/modes-v7/stress", - "profile": "stress", - "seed": 93713, - "manifest_sha256": "1540ced231e83c2c4b716b52312d9ad2e20367416182473be6c2e0e4fce8553e", - "counts": { - "screenshots": 70, - "legacy_hash_computed_from_image": 70, - "cards": 1126, - "mode:level": 162, - "mode:training": 82, - "mode:performance": 209, - "mode:total": 62, - "mode:technic": 203, - "mode:visual": 217, - "mode:hide": 191 - }, - "locales": { - "ja": 14, - "en": 14, - "zh-Hant": 14, - "zh-Hans": 14, - "ko": 14 - }, - "identities": 122 - }, - { - "dataset": "synthetic/final-v9/validation", - "profile": "validation", - "seed": 908271, - "manifest_sha256": "7cef333204cac03514a6390ed3aca374995d771d73fe751b69038fc4648712d3", - "counts": { - "screenshots": 560, - "cards": 9723, - "mode:level": 1689, - "mode:training": 655, - "mode:performance": 1642, - "mode:total": 694, - "mode:technic": 1654, - "mode:visual": 1714, - "mode:hide": 1675 - }, - "locales": { - "ja": 112, - "en": 112, - "zh-Hant": 112, - "zh-Hans": 112, - "ko": 112 - }, - "identities": 122 - }, - { - "dataset": "synthetic/final-v9/stress", - "profile": "stress", - "seed": 708289, - "manifest_sha256": "ed1e80cbc13ac5bc41de0396deae6270bc3a638dfb8deaee766bcdd953fa0f31", - "counts": { - "screenshots": 280, - "cards": 4673, - "mode:level": 812, - "mode:training": 283, - "mode:performance": 848, - "mode:total": 352, - "mode:technic": 768, - "mode:visual": 817, - "mode:hide": 793 - }, - "locales": { - "ja": 56, - "en": 56, - "zh-Hant": 56, - "zh-Hans": 56, - "ko": 56 - }, - "identities": 122 - }, - { - "dataset": "synthetic/final-v15/validation", - "profile": "validation", - "seed": 915271, - "manifest_sha256": "cff8de062b3422a874885ec9cca8c57d43aa86a218cbccc57245fd6546504eb2", - "counts": { - "screenshots": 560, - "cards": 9779, - "mode:level": 1676, - "mode:training": 697, - "mode:performance": 1646, - "mode:total": 713, - "mode:technic": 1727, - "mode:visual": 1628, - "mode:hide": 1692 - }, - "locales": { - "ja": 112, - "en": 112, - "zh-Hant": 112, - "zh-Hans": 112, - "ko": 112 - }, - "identities": 122 - }, - { - "dataset": "synthetic/final-v15/stress", - "profile": "stress", - "seed": 715289, - "manifest_sha256": "87bcb54891fcef6f2552345d79fc86e1a72f4f76e4248a26bc9eee476d47f7c1", - "counts": { - "screenshots": 280, - "cards": 4531, - "mode:level": 749, - "mode:training": 299, - "mode:performance": 776, - "mode:total": 289, - "mode:technic": 785, - "mode:visual": 813, - "mode:hide": 820 - }, - "locales": { - "ja": 56, - "en": 56, - "zh-Hant": 56, - "zh-Hans": 56, - "ko": 56 - }, - "identities": 122 - } - ], - "errors": [], - "scope": "Manifest hashes, exact screenshot duplicates, training ID exclusions, progression and visible labels. Does not certify pixel parity or exact randomized power values." -} \ No newline at end of file diff --git a/reports/acceptance/synthetic-full-screen.json b/reports/acceptance/synthetic-full-screen.json deleted file mode 100644 index 0b207bc..0000000 --- a/reports/acceptance/synthetic-full-screen.json +++ /dev/null @@ -1,4089 +0,0 @@ -{ - "schema": "ournotes-boxlens.metrics/1", - "runs": { - "validation": { - "screenshots": 560, - "identity": { - "expected": 9779, - "correct": 9673, - "extra": 0, - "missed": 106, - "recall": 0.9891604458533593, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 1522, - "correct": 1518, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 1, - "end_to_end_accuracy": 0.9973718791064389, - "accepted_precision": 1.0, - "coverage": 0.9973718791064389 - }, - "card_rank": { - "visible_truth": 7407, - "correct": 7249, - "wrong": 0, - "unknown_after_localization": 124, - "not_localized": 34, - "end_to_end_accuracy": 0.9786688267854732, - "accepted_precision": 1.0, - "coverage": 0.9786688267854732 - }, - "awake_count": { - "visible_truth": 637, - "correct": 637, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0, - "hidden_claims_by_field": {}, - "latency_ms": { - "median": 918.02, - "p95": 1341.7854999999993 - }, - "cpu_affinity": [ - 0, - 1 - ], - "platform": "Linux-5.10.134-008.14.kangaroo.al8.x86_64-x86_64-with-glibc2.35", - "opencv_threads": 2, - "matching": "identity and bbox IoU >= 0.5; ID-only when calibration truth lacks boxes", - "raw_report_sha256": "4ee2b46853cefe0aeff767e6b9234633e85172430b20d7ac2b387c95a43ff02a", - "strata": { - "overall": { - "identity": { - "expected": 9779, - "correct": 9673, - "extra": 0, - "missed": 106, - "recall": 0.9891604458533593, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 1522, - "correct": 1518, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 1, - "end_to_end_accuracy": 0.9973718791064389, - "accepted_precision": 1.0, - "coverage": 0.9973718791064389 - }, - "card_rank": { - "visible_truth": 7407, - "correct": 7249, - "wrong": 0, - "unknown_after_localization": 124, - "not_localized": 34, - "end_to_end_accuracy": 0.9786688267854732, - "accepted_precision": 1.0, - "coverage": 0.9786688267854732 - }, - "awake_count": { - "visible_truth": 637, - "correct": 637, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "profile:validation": { - "identity": { - "expected": 9779, - "correct": 9673, - "extra": 0, - "missed": 106, - "recall": 0.9891604458533593, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 1522, - "correct": 1518, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 1, - "end_to_end_accuracy": 0.9973718791064389, - "accepted_precision": 1.0, - "coverage": 0.9973718791064389 - }, - "card_rank": { - "visible_truth": 7407, - "correct": 7249, - "wrong": 0, - "unknown_after_localization": 124, - "not_localized": 34, - "end_to_end_accuracy": 0.9786688267854732, - "accepted_precision": 1.0, - "coverage": 0.9786688267854732 - }, - "awake_count": { - "visible_truth": 637, - "correct": 637, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "locale:ja": { - "identity": { - "expected": 1968, - "correct": 1948, - "extra": 0, - "missed": 20, - "recall": 0.9898373983739838, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 352, - "correct": 350, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 1, - "end_to_end_accuracy": 0.9943181818181818, - "accepted_precision": 1.0, - "coverage": 0.9943181818181818 - }, - "card_rank": { - "visible_truth": 1488, - "correct": 1467, - "wrong": 0, - "unknown_after_localization": 11, - "not_localized": 10, - "end_to_end_accuracy": 0.9858870967741935, - "accepted_precision": 1.0, - "coverage": 0.9858870967741935 - }, - "awake_count": { - "visible_truth": 105, - "correct": 105, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[1920, 1200]": { - "identity": { - "expected": 2106, - "correct": 2080, - "extra": 0, - "missed": 26, - "recall": 0.9876543209876543, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 259, - "correct": 258, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9961389961389961, - "accepted_precision": 1.0, - "coverage": 0.9961389961389961 - }, - "card_rank": { - "visible_truth": 1577, - "correct": 1559, - "wrong": 0, - "unknown_after_localization": 10, - "not_localized": 8, - "end_to_end_accuracy": 0.9885859226379201, - "accepted_precision": 1.0, - "coverage": 0.9885859226379201 - }, - "awake_count": { - "visible_truth": 144, - "correct": 144, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:1080": { - "identity": { - "expected": 2228, - "correct": 2199, - "extra": 0, - "missed": 29, - "recall": 0.9869838420107719, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 340, - "correct": 339, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9970588235294118, - "accepted_precision": 1.0, - "coverage": 0.9970588235294118 - }, - "card_rank": { - "visible_truth": 1715, - "correct": 1677, - "wrong": 0, - "unknown_after_localization": 27, - "not_localized": 11, - "end_to_end_accuracy": 0.9778425655976677, - "accepted_precision": 1.0, - "coverage": 0.9778425655976677 - }, - "awake_count": { - "visible_truth": 72, - "correct": 72, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "compression_rounds:2": { - "identity": { - "expected": 4613, - "correct": 4567, - "extra": 0, - "missed": 46, - "recall": 0.9900281812269672, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 753, - "correct": 752, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.99867197875166, - "accepted_precision": 1.0, - "coverage": 0.99867197875166 - }, - "card_rank": { - "visible_truth": 3398, - "correct": 3351, - "wrong": 0, - "unknown_after_localization": 36, - "not_localized": 11, - "end_to_end_accuracy": 0.9861683343143025, - "accepted_precision": 1.0, - "coverage": 0.9861683343143025 - }, - "awake_count": { - "visible_truth": 234, - "correct": 234, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "minimum_quality_bin:80": { - "identity": { - "expected": 1885, - "correct": 1869, - "extra": 0, - "missed": 16, - "recall": 0.9915119363395225, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 298, - "correct": 298, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1374, - "correct": 1362, - "wrong": 0, - "unknown_after_localization": 9, - "not_localized": 3, - "end_to_end_accuracy": 0.9912663755458515, - "accepted_precision": 1.0, - "coverage": 0.9912663755458515 - }, - "awake_count": { - "visible_truth": 87, - "correct": 87, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "codecs:JPEG": { - "identity": { - "expected": 1567, - "correct": 1556, - "extra": 0, - "missed": 11, - "recall": 0.9929802169751116, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 234, - "correct": 234, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1200, - "correct": 1195, - "wrong": 0, - "unknown_after_localization": 2, - "not_localized": 3, - "end_to_end_accuracy": 0.9958333333333333, - "accepted_precision": 1.0, - "coverage": 0.9958333333333333 - }, - "awake_count": { - "visible_truth": 67, - "correct": 67, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "kind:member": { - "identity": { - "expected": 4713, - "correct": 4706, - "extra": 0, - "missed": 7, - "recall": 0.9985147464460005, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 613, - "correct": 610, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 0, - "end_to_end_accuracy": 0.9951060358890701, - "accepted_precision": 1.0, - "coverage": 0.9951060358890701 - }, - "card_rank": { - "visible_truth": 3736, - "correct": 3635, - "wrong": 0, - "unknown_after_localization": 101, - "not_localized": 0, - "end_to_end_accuracy": 0.9729657387580299, - "accepted_precision": 1.0, - "coverage": 0.9729657387580299 - }, - "awake_count": { - "visible_truth": 637, - "correct": 637, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "identity_split:seen": { - "identity": { - "expected": 8421, - "correct": 8317, - "extra": 0, - "missed": 104, - "recall": 0.9876499228120176, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 1314, - "correct": 1310, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 1, - "end_to_end_accuracy": 0.9969558599695586, - "accepted_precision": 1.0, - "coverage": 0.9969558599695586 - }, - "card_rank": { - "visible_truth": 6369, - "correct": 6221, - "wrong": 0, - "unknown_after_localization": 114, - "not_localized": 34, - "end_to_end_accuracy": 0.9767624430836866, - "accepted_precision": 1.0, - "coverage": 0.9767624430836866 - }, - "awake_count": { - "visible_truth": 550, - "correct": 550, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:150": { - "identity": { - "expected": 1618, - "correct": 1602, - "extra": 0, - "missed": 16, - "recall": 0.9901112484548825, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 221, - "correct": 221, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1255, - "correct": 1243, - "wrong": 0, - "unknown_after_localization": 7, - "not_localized": 5, - "end_to_end_accuracy": 0.9904382470119522, - "accepted_precision": 1.0, - "coverage": 0.9904382470119522 - }, - "awake_count": { - "visible_truth": 226, - "correct": 226, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "identity_split:heldout": { - "identity": { - "expected": 1358, - "correct": 1356, - "extra": 0, - "missed": 2, - "recall": 0.9985272459499264, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 208, - "correct": 208, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1038, - "correct": 1028, - "wrong": 0, - "unknown_after_localization": 10, - "not_localized": 0, - "end_to_end_accuracy": 0.9903660886319846, - "accepted_precision": 1.0, - "coverage": 0.9903660886319846 - }, - "awake_count": { - "visible_truth": 87, - "correct": 87, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[1366, 768]": { - "identity": { - "expected": 1864, - "correct": 1838, - "extra": 0, - "missed": 26, - "recall": 0.9860515021459227, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 357, - "correct": 357, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1473, - "correct": 1382, - "wrong": 0, - "unknown_after_localization": 80, - "not_localized": 11, - "end_to_end_accuracy": 0.9382213170400543, - "accepted_precision": 1.0, - "coverage": 0.9382213170400543 - }, - "awake_count": { - "visible_truth": 133, - "correct": 133, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "compression_rounds:3": { - "identity": { - "expected": 5166, - "correct": 5106, - "extra": 0, - "missed": 60, - "recall": 0.9883855981416957, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 769, - "correct": 766, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 0, - "end_to_end_accuracy": 0.9960988296488946, - "accepted_precision": 1.0, - "coverage": 0.9960988296488946 - }, - "card_rank": { - "visible_truth": 4009, - "correct": 3898, - "wrong": 0, - "unknown_after_localization": 88, - "not_localized": 23, - "end_to_end_accuracy": 0.9723122973310052, - "accepted_precision": 1.0, - "coverage": 0.9723122973310052 - }, - "awake_count": { - "visible_truth": 403, - "correct": 403, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "minimum_quality_bin:40": { - "identity": { - "expected": 4774, - "correct": 4725, - "extra": 0, - "missed": 49, - "recall": 0.9897360703812317, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 756, - "correct": 753, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 0, - "end_to_end_accuracy": 0.996031746031746, - "accepted_precision": 1.0, - "coverage": 0.996031746031746 - }, - "card_rank": { - "visible_truth": 3658, - "correct": 3561, - "wrong": 0, - "unknown_after_localization": 81, - "not_localized": 16, - "end_to_end_accuracy": 0.9734827774740296, - "accepted_precision": 1.0, - "coverage": 0.9734827774740296 - }, - "awake_count": { - "visible_truth": 380, - "correct": 380, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "codecs:JPEG,WEBP": { - "identity": { - "expected": 6125, - "correct": 6056, - "extra": 0, - "missed": 69, - "recall": 0.988734693877551, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 1007, - "correct": 1003, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 1, - "end_to_end_accuracy": 0.9960278053624627, - "accepted_precision": 1.0, - "coverage": 0.9960278053624627 - }, - "card_rank": { - "visible_truth": 4638, - "correct": 4512, - "wrong": 0, - "unknown_after_localization": 99, - "not_localized": 27, - "end_to_end_accuracy": 0.9728331177231565, - "accepted_precision": 1.0, - "coverage": 0.9728331177231565 - }, - "awake_count": { - "visible_truth": 335, - "correct": 335, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "kind:snap": { - "identity": { - "expected": 5066, - "correct": 4967, - "extra": 0, - "missed": 99, - "recall": 0.9804579549940782, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 909, - "correct": 908, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9988998899889989, - "accepted_precision": 1.0, - "coverage": 0.9988998899889989 - }, - "card_rank": { - "visible_truth": 3671, - "correct": 3614, - "wrong": 0, - "unknown_after_localization": 23, - "not_localized": 34, - "end_to_end_accuracy": 0.9844728956687551, - "accepted_precision": 1.0, - "coverage": 0.9844728956687551 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:180": { - "identity": { - "expected": 2041, - "correct": 2023, - "extra": 0, - "missed": 18, - "recall": 0.9911807937285644, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 348, - "correct": 347, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9971264367816092, - "accepted_precision": 1.0, - "coverage": 0.9971264367816092 - }, - "card_rank": { - "visible_truth": 1631, - "correct": 1618, - "wrong": 0, - "unknown_after_localization": 8, - "not_localized": 5, - "end_to_end_accuracy": 0.9920294297976702, - "accepted_precision": 1.0, - "coverage": 0.9920294297976702 - }, - "awake_count": { - "visible_truth": 175, - "correct": 175, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:720": { - "identity": { - "expected": 1076, - "correct": 1057, - "extra": 0, - "missed": 19, - "recall": 0.9823420074349443, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 189, - "correct": 189, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 839, - "correct": 772, - "wrong": 0, - "unknown_after_localization": 59, - "not_localized": 8, - "end_to_end_accuracy": 0.9201430274135876, - "accepted_precision": 1.0, - "coverage": 0.9201430274135876 - }, - "awake_count": { - "visible_truth": 133, - "correct": 133, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:120": { - "identity": { - "expected": 876, - "correct": 870, - "extra": 0, - "missed": 6, - "recall": 0.9931506849315068, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 103, - "correct": 102, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9902912621359223, - "accepted_precision": 1.0, - "coverage": 0.9902912621359223 - }, - "card_rank": { - "visible_truth": 681, - "correct": 654, - "wrong": 0, - "unknown_after_localization": 23, - "not_localized": 4, - "end_to_end_accuracy": 0.960352422907489, - "accepted_precision": 1.0, - "coverage": 0.960352422907489 - }, - "awake_count": { - "visible_truth": 48, - "correct": 48, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[2048, 1536]": { - "identity": { - "expected": 2820, - "correct": 2796, - "extra": 0, - "missed": 24, - "recall": 0.9914893617021276, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 426, - "correct": 425, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9976525821596244, - "accepted_precision": 1.0, - "coverage": 0.9976525821596244 - }, - "card_rank": { - "visible_truth": 2248, - "correct": 2219, - "wrong": 0, - "unknown_after_localization": 18, - "not_localized": 11, - "end_to_end_accuracy": 0.9870996441281139, - "accepted_precision": 1.0, - "coverage": 0.9870996441281139 - }, - "awake_count": { - "visible_truth": 224, - "correct": 224, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:1440": { - "identity": { - "expected": 4111, - "correct": 4074, - "extra": 0, - "missed": 37, - "recall": 0.9909997567501825, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 572, - "correct": 570, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 1, - "end_to_end_accuracy": 0.9965034965034965, - "accepted_precision": 1.0, - "coverage": 0.9965034965034965 - }, - "card_rank": { - "visible_truth": 3151, - "correct": 3113, - "wrong": 0, - "unknown_after_localization": 24, - "not_localized": 14, - "end_to_end_accuracy": 0.9879403364011425, - "accepted_precision": 1.0, - "coverage": 0.9879403364011425 - }, - "awake_count": { - "visible_truth": 353, - "correct": 353, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "minimum_quality_bin:60": { - "identity": { - "expected": 3120, - "correct": 3079, - "extra": 0, - "missed": 41, - "recall": 0.9868589743589744, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 468, - "correct": 467, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9978632478632479, - "accepted_precision": 1.0, - "coverage": 0.9978632478632479 - }, - "card_rank": { - "visible_truth": 2375, - "correct": 2326, - "wrong": 0, - "unknown_after_localization": 34, - "not_localized": 15, - "end_to_end_accuracy": 0.9793684210526316, - "accepted_precision": 1.0, - "coverage": 0.9793684210526316 - }, - "awake_count": { - "visible_truth": 170, - "correct": 170, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:270": { - "identity": { - "expected": 961, - "correct": 944, - "extra": 0, - "missed": 17, - "recall": 0.9823100936524454, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 144, - "correct": 143, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9930555555555556, - "accepted_precision": 1.0, - "coverage": 0.9930555555555556 - }, - "card_rank": { - "visible_truth": 610, - "correct": 606, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 3, - "end_to_end_accuracy": 0.9934426229508196, - "accepted_precision": 1.0, - "coverage": 0.9934426229508196 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[2340, 1080]": { - "identity": { - "expected": 1530, - "correct": 1510, - "extra": 0, - "missed": 20, - "recall": 0.9869281045751634, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 266, - "correct": 266, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1028, - "correct": 1019, - "wrong": 0, - "unknown_after_localization": 6, - "not_localized": 3, - "end_to_end_accuracy": 0.9912451361867705, - "accepted_precision": 1.0, - "coverage": 0.9912451361867705 - }, - "awake_count": { - "visible_truth": 72, - "correct": 72, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:1800": { - "identity": { - "expected": 1790, - "correct": 1773, - "extra": 0, - "missed": 17, - "recall": 0.9905027932960894, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 337, - "correct": 336, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9970326409495549, - "accepted_precision": 1.0, - "coverage": 0.9970326409495549 - }, - "card_rank": { - "visible_truth": 1312, - "correct": 1298, - "wrong": 0, - "unknown_after_localization": 13, - "not_localized": 1, - "end_to_end_accuracy": 0.989329268292683, - "accepted_precision": 1.0, - "coverage": 0.989329268292683 - }, - "awake_count": { - "visible_truth": 48, - "correct": 48, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:300": { - "identity": { - "expected": 628, - "correct": 622, - "extra": 0, - "missed": 6, - "recall": 0.9904458598726115, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 113, - "correct": 113, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 474, - "correct": 467, - "wrong": 0, - "unknown_after_localization": 6, - "not_localized": 1, - "end_to_end_accuracy": 0.9852320675105485, - "accepted_precision": 1.0, - "coverage": 0.9852320675105485 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:360": { - "identity": { - "expected": 205, - "correct": 204, - "extra": 0, - "missed": 1, - "recall": 0.9951219512195122, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 60, - "correct": 60, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 137, - "correct": 137, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "codecs:WEBP": { - "identity": { - "expected": 2087, - "correct": 2061, - "extra": 0, - "missed": 26, - "recall": 0.9875419262098706, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 281, - "correct": 281, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1569, - "correct": 1542, - "wrong": 0, - "unknown_after_localization": 23, - "not_localized": 4, - "end_to_end_accuracy": 0.982791586998088, - "accepted_precision": 1.0, - "coverage": 0.982791586998088 - }, - "awake_count": { - "visible_truth": 235, - "correct": 235, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:210": { - "identity": { - "expected": 1419, - "correct": 1404, - "extra": 0, - "missed": 15, - "recall": 0.9894291754756871, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 238, - "correct": 237, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9957983193277311, - "accepted_precision": 1.0, - "coverage": 0.9957983193277311 - }, - "card_rank": { - "visible_truth": 1084, - "correct": 1075, - "wrong": 0, - "unknown_after_localization": 4, - "not_localized": 5, - "end_to_end_accuracy": 0.9916974169741697, - "accepted_precision": 1.0, - "coverage": 0.9916974169741697 - }, - "awake_count": { - "visible_truth": 55, - "correct": 55, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[2560, 1440]": { - "identity": { - "expected": 1459, - "correct": 1449, - "extra": 0, - "missed": 10, - "recall": 0.9931459904043866, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 214, - "correct": 212, - "wrong": 0, - "unknown_after_localization": 2, - "not_localized": 0, - "end_to_end_accuracy": 0.9906542056074766, - "accepted_precision": 1.0, - "coverage": 0.9906542056074766 - }, - "card_rank": { - "visible_truth": 1081, - "correct": 1070, - "wrong": 0, - "unknown_after_localization": 10, - "not_localized": 1, - "end_to_end_accuracy": 0.9898242368177613, - "accepted_precision": 1.0, - "coverage": 0.9898242368177613 - }, - "awake_count": { - "visible_truth": 64, - "correct": 64, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:2160": { - "identity": { - "expected": 574, - "correct": 570, - "extra": 0, - "missed": 4, - "recall": 0.9930313588850174, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 84, - "correct": 84, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 390, - "correct": 389, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9974358974358974, - "accepted_precision": 1.0, - "coverage": 0.9974358974358974 - }, - "awake_count": { - "visible_truth": 31, - "correct": 31, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "locale:en": { - "identity": { - "expected": 1977, - "correct": 1955, - "extra": 0, - "missed": 22, - "recall": 0.988872028325746, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 262, - "correct": 262, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1550, - "correct": 1523, - "wrong": 0, - "unknown_after_localization": 21, - "not_localized": 6, - "end_to_end_accuracy": 0.9825806451612903, - "accepted_precision": 1.0, - "coverage": 0.9825806451612903 - }, - "awake_count": { - "visible_truth": 124, - "correct": 124, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:240": { - "identity": { - "expected": 1107, - "correct": 1088, - "extra": 0, - "missed": 19, - "recall": 0.982836495031617, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 189, - "correct": 189, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 824, - "correct": 798, - "wrong": 0, - "unknown_after_localization": 15, - "not_localized": 11, - "end_to_end_accuracy": 0.9684466019417476, - "accepted_precision": 1.0, - "coverage": 0.9684466019417476 - }, - "awake_count": { - "visible_truth": 12, - "correct": 12, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:90": { - "identity": { - "expected": 502, - "correct": 501, - "extra": 0, - "missed": 1, - "recall": 0.99800796812749, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 58, - "correct": 58, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 417, - "correct": 358, - "wrong": 0, - "unknown_after_localization": 59, - "not_localized": 0, - "end_to_end_accuracy": 0.8585131894484412, - "accepted_precision": 1.0, - "coverage": 0.8585131894484412 - }, - "awake_count": { - "visible_truth": 121, - "correct": 121, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "locale:zh-Hant": { - "identity": { - "expected": 1932, - "correct": 1914, - "extra": 0, - "missed": 18, - "recall": 0.9906832298136646, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 380, - "correct": 379, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9973684210526316, - "accepted_precision": 1.0, - "coverage": 0.9973684210526316 - }, - "card_rank": { - "visible_truth": 1382, - "correct": 1339, - "wrong": 0, - "unknown_after_localization": 38, - "not_localized": 5, - "end_to_end_accuracy": 0.9688856729377714, - "accepted_precision": 1.0, - "coverage": 0.9688856729377714 - }, - "awake_count": { - "visible_truth": 138, - "correct": 138, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "locale:zh-Hans": { - "identity": { - "expected": 1921, - "correct": 1898, - "extra": 0, - "missed": 23, - "recall": 0.9880270692347736, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 256, - "correct": 255, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.99609375, - "accepted_precision": 1.0, - "coverage": 0.99609375 - }, - "card_rank": { - "visible_truth": 1532, - "correct": 1503, - "wrong": 0, - "unknown_after_localization": 23, - "not_localized": 6, - "end_to_end_accuracy": 0.9810704960835509, - "accepted_precision": 1.0, - "coverage": 0.9810704960835509 - }, - "awake_count": { - "visible_truth": 122, - "correct": 122, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "locale:ko": { - "identity": { - "expected": 1981, - "correct": 1958, - "extra": 0, - "missed": 23, - "recall": 0.9883897021706209, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 272, - "correct": 272, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 1455, - "correct": 1417, - "wrong": 0, - "unknown_after_localization": 31, - "not_localized": 7, - "end_to_end_accuracy": 0.9738831615120275, - "accepted_precision": 1.0, - "coverage": 0.9738831615120275 - }, - "awake_count": { - "visible_truth": 148, - "correct": 148, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:330": { - "identity": { - "expected": 322, - "correct": 319, - "extra": 0, - "missed": 3, - "recall": 0.9906832298136646, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 32, - "correct": 32, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 230, - "correct": 229, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9956521739130435, - "accepted_precision": 1.0, - "coverage": 0.9956521739130435 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:390": { - "identity": { - "expected": 100, - "correct": 96, - "extra": 0, - "missed": 4, - "recall": 0.96, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 16, - "correct": 16, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 64, - "correct": 64, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - } - }, - "merged_boxes": { - "count": 140, - "duplicate_invariant": true, - "fields": { - "level": { - "observable": 1522, - "correct": 1518, - "not_observed": 6910, - "unknown": 4 - }, - "card_rank": { - "observable": 6663, - "correct": 6542, - "not_observed": 1769, - "unknown": 121 - }, - "awake_count": { - "not_observed": 7795, - "observable": 637, - "correct": 637 - } - } - } - }, - "stress": { - "screenshots": 280, - "identity": { - "expected": 4531, - "correct": 3956, - "extra": 0, - "missed": 575, - "recall": 0.8730964467005076, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 688, - "correct": 597, - "wrong": 0, - "unknown_after_localization": 50, - "not_localized": 41, - "end_to_end_accuracy": 0.8677325581395349, - "accepted_precision": 1.0, - "coverage": 0.8677325581395349 - }, - "card_rank": { - "visible_truth": 3281, - "correct": 2391, - "wrong": 0, - "unknown_after_localization": 481, - "not_localized": 409, - "end_to_end_accuracy": 0.7287412374276135, - "accepted_precision": 1.0, - "coverage": 0.7287412374276135 - }, - "awake_count": { - "visible_truth": 279, - "correct": 238, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 40, - "end_to_end_accuracy": 0.8530465949820788, - "accepted_precision": 1.0, - "coverage": 0.8530465949820788 - } - }, - "hidden_field_predictions": 0, - "hidden_claims_by_field": {}, - "latency_ms": { - "median": 763.46, - "p95": 1283.4970000000005 - }, - "cpu_affinity": [ - 0, - 1 - ], - "platform": "Linux-5.10.134-008.14.kangaroo.al8.x86_64-x86_64-with-glibc2.35", - "opencv_threads": 2, - "matching": "identity and bbox IoU >= 0.5; ID-only when calibration truth lacks boxes", - "raw_report_sha256": "9f224be84758a04457dd2a5a031ee4e2e4351f26d97f6298b8122cf64f1d4d6e", - "strata": { - "overall": { - "identity": { - "expected": 4531, - "correct": 3956, - "extra": 0, - "missed": 575, - "recall": 0.8730964467005076, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 688, - "correct": 597, - "wrong": 0, - "unknown_after_localization": 50, - "not_localized": 41, - "end_to_end_accuracy": 0.8677325581395349, - "accepted_precision": 1.0, - "coverage": 0.8677325581395349 - }, - "card_rank": { - "visible_truth": 3281, - "correct": 2391, - "wrong": 0, - "unknown_after_localization": 481, - "not_localized": 409, - "end_to_end_accuracy": 0.7287412374276135, - "accepted_precision": 1.0, - "coverage": 0.7287412374276135 - }, - "awake_count": { - "visible_truth": 279, - "correct": 238, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 40, - "end_to_end_accuracy": 0.8530465949820788, - "accepted_precision": 1.0, - "coverage": 0.8530465949820788 - } - }, - "hidden_field_predictions": 0 - }, - "profile:stress": { - "identity": { - "expected": 4531, - "correct": 3956, - "extra": 0, - "missed": 575, - "recall": 0.8730964467005076, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 688, - "correct": 597, - "wrong": 0, - "unknown_after_localization": 50, - "not_localized": 41, - "end_to_end_accuracy": 0.8677325581395349, - "accepted_precision": 1.0, - "coverage": 0.8677325581395349 - }, - "card_rank": { - "visible_truth": 3281, - "correct": 2391, - "wrong": 0, - "unknown_after_localization": 481, - "not_localized": 409, - "end_to_end_accuracy": 0.7287412374276135, - "accepted_precision": 1.0, - "coverage": 0.7287412374276135 - }, - "awake_count": { - "visible_truth": 279, - "correct": 238, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 40, - "end_to_end_accuracy": 0.8530465949820788, - "accepted_precision": 1.0, - "coverage": 0.8530465949820788 - } - }, - "hidden_field_predictions": 0 - }, - "locale:ja": { - "identity": { - "expected": 947, - "correct": 859, - "extra": 0, - "missed": 88, - "recall": 0.9070749736008448, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 175, - "correct": 162, - "wrong": 0, - "unknown_after_localization": 6, - "not_localized": 7, - "end_to_end_accuracy": 0.9257142857142857, - "accepted_precision": 1.0, - "coverage": 0.9257142857142857 - }, - "card_rank": { - "visible_truth": 706, - "correct": 545, - "wrong": 0, - "unknown_after_localization": 119, - "not_localized": 42, - "end_to_end_accuracy": 0.7719546742209632, - "accepted_precision": 1.0, - "coverage": 0.7719546742209632 - }, - "awake_count": { - "visible_truth": 63, - "correct": 63, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[2532, 1170]": { - "identity": { - "expected": 821, - "correct": 794, - "extra": 0, - "missed": 27, - "recall": 0.9671132764920828, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 108, - "correct": 93, - "wrong": 0, - "unknown_after_localization": 13, - "not_localized": 2, - "end_to_end_accuracy": 0.8611111111111112, - "accepted_precision": 1.0, - "coverage": 0.8611111111111112 - }, - "card_rank": { - "visible_truth": 496, - "correct": 405, - "wrong": 0, - "unknown_after_localization": 78, - "not_localized": 13, - "end_to_end_accuracy": 0.8165322580645161, - "accepted_precision": 1.0, - "coverage": 0.8165322580645161 - }, - "awake_count": { - "visible_truth": 23, - "correct": 23, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:1080": { - "identity": { - "expected": 698, - "correct": 685, - "extra": 0, - "missed": 13, - "recall": 0.9813753581661891, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 111, - "correct": 111, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 499, - "correct": 445, - "wrong": 0, - "unknown_after_localization": 48, - "not_localized": 6, - "end_to_end_accuracy": 0.8917835671342685, - "accepted_precision": 1.0, - "coverage": 0.8917835671342685 - }, - "awake_count": { - "visible_truth": 17, - "correct": 17, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "compression_rounds:4": { - "identity": { - "expected": 1599, - "correct": 1414, - "extra": 0, - "missed": 185, - "recall": 0.884302689180738, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 355, - "correct": 310, - "wrong": 0, - "unknown_after_localization": 19, - "not_localized": 26, - "end_to_end_accuracy": 0.8732394366197183, - "accepted_precision": 1.0, - "coverage": 0.8732394366197183 - }, - "card_rank": { - "visible_truth": 1254, - "correct": 946, - "wrong": 0, - "unknown_after_localization": 174, - "not_localized": 134, - "end_to_end_accuracy": 0.7543859649122807, - "accepted_precision": 1.0, - "coverage": 0.7543859649122807 - }, - "awake_count": { - "visible_truth": 95, - "correct": 94, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9894736842105263, - "accepted_precision": 1.0, - "coverage": 0.9894736842105263 - } - }, - "hidden_field_predictions": 0 - }, - "minimum_quality_bin:20": { - "identity": { - "expected": 2528, - "correct": 2214, - "extra": 0, - "missed": 314, - "recall": 0.8757911392405063, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 434, - "correct": 379, - "wrong": 0, - "unknown_after_localization": 22, - "not_localized": 33, - "end_to_end_accuracy": 0.8732718894009217, - "accepted_precision": 1.0, - "coverage": 0.8732718894009217 - }, - "card_rank": { - "visible_truth": 1702, - "correct": 1222, - "wrong": 0, - "unknown_after_localization": 267, - "not_localized": 213, - "end_to_end_accuracy": 0.717978848413631, - "accepted_precision": 1.0, - "coverage": 0.717978848413631 - }, - "awake_count": { - "visible_truth": 101, - "correct": 79, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 22, - "end_to_end_accuracy": 0.7821782178217822, - "accepted_precision": 1.0, - "coverage": 0.7821782178217822 - } - }, - "hidden_field_predictions": 0 - }, - "codecs:JPEG,WEBP": { - "identity": { - "expected": 3120, - "correct": 2711, - "extra": 0, - "missed": 409, - "recall": 0.8689102564102564, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 528, - "correct": 449, - "wrong": 0, - "unknown_after_localization": 38, - "not_localized": 41, - "end_to_end_accuracy": 0.8503787878787878, - "accepted_precision": 1.0, - "coverage": 0.8503787878787878 - }, - "card_rank": { - "visible_truth": 2305, - "correct": 1666, - "wrong": 0, - "unknown_after_localization": 315, - "not_localized": 324, - "end_to_end_accuracy": 0.7227765726681128, - "accepted_precision": 1.0, - "coverage": 0.7227765726681128 - }, - "awake_count": { - "visible_truth": 220, - "correct": 191, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 28, - "end_to_end_accuracy": 0.8681818181818182, - "accepted_precision": 1.0, - "coverage": 0.8681818181818182 - } - }, - "hidden_field_predictions": 0 - }, - "kind:member": { - "identity": { - "expected": 2166, - "correct": 1897, - "extra": 0, - "missed": 269, - "recall": 0.8758079409048938, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 249, - "correct": 235, - "wrong": 0, - "unknown_after_localization": 14, - "not_localized": 0, - "end_to_end_accuracy": 0.9437751004016064, - "accepted_precision": 1.0, - "coverage": 0.9437751004016064 - }, - "card_rank": { - "visible_truth": 1594, - "correct": 1100, - "wrong": 0, - "unknown_after_localization": 312, - "not_localized": 182, - "end_to_end_accuracy": 0.6900878293601004, - "accepted_precision": 1.0, - "coverage": 0.6900878293601004 - }, - "awake_count": { - "visible_truth": 279, - "correct": 238, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 40, - "end_to_end_accuracy": 0.8530465949820788, - "accepted_precision": 1.0, - "coverage": 0.8530465949820788 - } - }, - "hidden_field_predictions": 0 - }, - "identity_split:seen": { - "identity": { - "expected": 3914, - "correct": 3410, - "extra": 0, - "missed": 504, - "recall": 0.8712314767501278, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 596, - "correct": 517, - "wrong": 0, - "unknown_after_localization": 44, - "not_localized": 35, - "end_to_end_accuracy": 0.8674496644295302, - "accepted_precision": 1.0, - "coverage": 0.8674496644295302 - }, - "card_rank": { - "visible_truth": 2852, - "correct": 2074, - "wrong": 0, - "unknown_after_localization": 419, - "not_localized": 359, - "end_to_end_accuracy": 0.7272089761570828, - "accepted_precision": 1.0, - "coverage": 0.7272089761570828 - }, - "awake_count": { - "visible_truth": 256, - "correct": 216, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 39, - "end_to_end_accuracy": 0.84375, - "accepted_precision": 1.0, - "coverage": 0.84375 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:90": { - "identity": { - "expected": 621, - "correct": 584, - "extra": 0, - "missed": 37, - "recall": 0.9404186795491143, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 63, - "correct": 47, - "wrong": 0, - "unknown_after_localization": 14, - "not_localized": 2, - "end_to_end_accuracy": 0.746031746031746, - "accepted_precision": 1.0, - "coverage": 0.746031746031746 - }, - "card_rank": { - "visible_truth": 409, - "correct": 268, - "wrong": 0, - "unknown_after_localization": 120, - "not_localized": 21, - "end_to_end_accuracy": 0.6552567237163814, - "accepted_precision": 1.0, - "coverage": 0.6552567237163814 - }, - "awake_count": { - "visible_truth": 10, - "correct": 10, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "identity_split:heldout": { - "identity": { - "expected": 617, - "correct": 546, - "extra": 0, - "missed": 71, - "recall": 0.8849270664505673, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 92, - "correct": 80, - "wrong": 0, - "unknown_after_localization": 6, - "not_localized": 6, - "end_to_end_accuracy": 0.8695652173913043, - "accepted_precision": 1.0, - "coverage": 0.8695652173913043 - }, - "card_rank": { - "visible_truth": 429, - "correct": 317, - "wrong": 0, - "unknown_after_localization": 62, - "not_localized": 50, - "end_to_end_accuracy": 0.7389277389277389, - "accepted_precision": 1.0, - "coverage": 0.7389277389277389 - }, - "awake_count": { - "visible_truth": 23, - "correct": 22, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9565217391304348, - "accepted_precision": 1.0, - "coverage": 0.9565217391304348 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[3840, 2160]": { - "identity": { - "expected": 995, - "correct": 980, - "extra": 0, - "missed": 15, - "recall": 0.9849246231155779, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 214, - "correct": 214, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 745, - "correct": 712, - "wrong": 0, - "unknown_after_localization": 28, - "not_localized": 5, - "end_to_end_accuracy": 0.9557046979865772, - "accepted_precision": 1.0, - "coverage": 0.9557046979865772 - }, - "awake_count": { - "visible_truth": 130, - "correct": 130, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:2160": { - "identity": { - "expected": 111, - "correct": 109, - "extra": 0, - "missed": 2, - "recall": 0.9819819819819819, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 36, - "correct": 36, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 58, - "correct": 58, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 1, - "correct": 1, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "kind:snap": { - "identity": { - "expected": 2365, - "correct": 2059, - "extra": 0, - "missed": 306, - "recall": 0.8706131078224102, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 439, - "correct": 362, - "wrong": 0, - "unknown_after_localization": 36, - "not_localized": 41, - "end_to_end_accuracy": 0.8246013667425968, - "accepted_precision": 1.0, - "coverage": 0.8246013667425968 - }, - "card_rank": { - "visible_truth": 1687, - "correct": 1291, - "wrong": 0, - "unknown_after_localization": 169, - "not_localized": 227, - "end_to_end_accuracy": 0.7652637818612922, - "accepted_precision": 1.0, - "coverage": 0.7652637818612922 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:300": { - "identity": { - "expected": 153, - "correct": 151, - "extra": 0, - "missed": 2, - "recall": 0.9869281045751634, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 52, - "correct": 52, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 95, - "correct": 95, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 1, - "correct": 1, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:2520": { - "identity": { - "expected": 196, - "correct": 196, - "extra": 0, - "missed": 0, - "recall": 1.0, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 8, - "correct": 8, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 141, - "correct": 139, - "wrong": 0, - "unknown_after_localization": 2, - "not_localized": 0, - "end_to_end_accuracy": 0.9858156028368794, - "accepted_precision": 1.0, - "coverage": 0.9858156028368794 - }, - "awake_count": { - "visible_truth": 69, - "correct": 69, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "compression_rounds:2": { - "identity": { - "expected": 1561, - "correct": 1337, - "extra": 0, - "missed": 224, - "recall": 0.8565022421524664, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 154, - "correct": 108, - "wrong": 0, - "unknown_after_localization": 31, - "not_localized": 15, - "end_to_end_accuracy": 0.7012987012987013, - "accepted_precision": 1.0, - "coverage": 0.7012987012987013 - }, - "card_rank": { - "visible_truth": 1031, - "correct": 746, - "wrong": 0, - "unknown_after_localization": 138, - "not_localized": 147, - "end_to_end_accuracy": 0.7235693501454898, - "accepted_precision": 1.0, - "coverage": 0.7235693501454898 - }, - "awake_count": { - "visible_truth": 63, - "correct": 51, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 12, - "end_to_end_accuracy": 0.8095238095238095, - "accepted_precision": 1.0, - "coverage": 0.8095238095238095 - } - }, - "hidden_field_predictions": 0 - }, - "minimum_quality_bin:60": { - "identity": { - "expected": 79, - "correct": 78, - "extra": 0, - "missed": 1, - "recall": 0.9873417721518988, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - }, - "card_rank": { - "visible_truth": 67, - "correct": 65, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 1, - "end_to_end_accuracy": 0.9701492537313433, - "accepted_precision": 1.0, - "coverage": 0.9701492537313433 - }, - "awake_count": { - "visible_truth": 36, - "correct": 36, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "codecs:JPEG": { - "identity": { - "expected": 714, - "correct": 622, - "extra": 0, - "missed": 92, - "recall": 0.8711484593837535, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 67, - "correct": 66, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9850746268656716, - "accepted_precision": 1.0, - "coverage": 0.9850746268656716 - }, - "card_rank": { - "visible_truth": 498, - "correct": 372, - "wrong": 0, - "unknown_after_localization": 62, - "not_localized": 64, - "end_to_end_accuracy": 0.7469879518072289, - "accepted_precision": 1.0, - "coverage": 0.7469879518072289 - }, - "awake_count": { - "visible_truth": 42, - "correct": 42, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:270": { - "identity": { - "expected": 281, - "correct": 276, - "extra": 0, - "missed": 5, - "recall": 0.9822064056939501, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 76, - "correct": 76, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 208, - "correct": 208, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 18, - "correct": 18, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "minimum_quality_bin:40": { - "identity": { - "expected": 1924, - "correct": 1664, - "extra": 0, - "missed": 260, - "recall": 0.8648648648648649, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 254, - "correct": 218, - "wrong": 0, - "unknown_after_localization": 28, - "not_localized": 8, - "end_to_end_accuracy": 0.8582677165354331, - "accepted_precision": 1.0, - "coverage": 0.8582677165354331 - }, - "card_rank": { - "visible_truth": 1512, - "correct": 1104, - "wrong": 0, - "unknown_after_localization": 213, - "not_localized": 195, - "end_to_end_accuracy": 0.7301587301587301, - "accepted_precision": 1.0, - "coverage": 0.7301587301587301 - }, - "awake_count": { - "visible_truth": 142, - "correct": 123, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 18, - "end_to_end_accuracy": 0.8661971830985915, - "accepted_precision": 1.0, - "coverage": 0.8661971830985915 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:150": { - "identity": { - "expected": 421, - "correct": 413, - "extra": 0, - "missed": 8, - "recall": 0.9809976247030879, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 66, - "correct": 66, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 357, - "correct": 310, - "wrong": 0, - "unknown_after_localization": 42, - "not_localized": 5, - "end_to_end_accuracy": 0.8683473389355743, - "accepted_precision": 1.0, - "coverage": 0.8683473389355743 - }, - "awake_count": { - "visible_truth": 55, - "correct": 54, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9818181818181818, - "accepted_precision": 1.0, - "coverage": 0.9818181818181818 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[2280, 1080]": { - "identity": { - "expected": 852, - "correct": 790, - "extra": 0, - "missed": 62, - "recall": 0.9272300469483568, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 129, - "correct": 117, - "wrong": 0, - "unknown_after_localization": 12, - "not_localized": 0, - "end_to_end_accuracy": 0.9069767441860465, - "accepted_precision": 1.0, - "coverage": 0.9069767441860465 - }, - "card_rank": { - "visible_truth": 663, - "correct": 498, - "wrong": 0, - "unknown_after_localization": 132, - "not_localized": 33, - "end_to_end_accuracy": 0.751131221719457, - "accepted_precision": 1.0, - "coverage": 0.751131221719457 - }, - "awake_count": { - "visible_truth": 15, - "correct": 15, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:1440": { - "identity": { - "expected": 1091, - "correct": 1068, - "extra": 0, - "missed": 23, - "recall": 0.9789184234647113, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 152, - "correct": 152, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 792, - "correct": 755, - "wrong": 0, - "unknown_after_localization": 31, - "not_localized": 6, - "end_to_end_accuracy": 0.9532828282828283, - "accepted_precision": 1.0, - "coverage": 0.9532828282828283 - }, - "awake_count": { - "visible_truth": 56, - "correct": 56, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:120": { - "identity": { - "expected": 481, - "correct": 469, - "extra": 0, - "missed": 12, - "recall": 0.975051975051975, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 34, - "correct": 32, - "wrong": 0, - "unknown_after_localization": 2, - "not_localized": 0, - "end_to_end_accuracy": 0.9411764705882353, - "accepted_precision": 1.0, - "coverage": 0.9411764705882353 - }, - "card_rank": { - "visible_truth": 335, - "correct": 303, - "wrong": 0, - "unknown_after_localization": 29, - "not_localized": 3, - "end_to_end_accuracy": 0.9044776119402985, - "accepted_precision": 1.0, - "coverage": 0.9044776119402985 - }, - "awake_count": { - "visible_truth": 42, - "correct": 42, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:1800": { - "identity": { - "expected": 651, - "correct": 644, - "extra": 0, - "missed": 7, - "recall": 0.989247311827957, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 179, - "correct": 179, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 507, - "correct": 489, - "wrong": 0, - "unknown_after_localization": 13, - "not_localized": 5, - "end_to_end_accuracy": 0.9644970414201184, - "accepted_precision": 1.0, - "coverage": 0.9644970414201184 - }, - "awake_count": { - "visible_truth": 78, - "correct": 77, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9871794871794872, - "accepted_precision": 1.0, - "coverage": 0.9871794871794872 - } - }, - "hidden_field_predictions": 0 - }, - "compression_rounds:3": { - "identity": { - "expected": 1371, - "correct": 1205, - "extra": 0, - "missed": 166, - "recall": 0.8789204959883297, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 179, - "correct": 179, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 996, - "correct": 699, - "wrong": 0, - "unknown_after_localization": 169, - "not_localized": 128, - "end_to_end_accuracy": 0.7018072289156626, - "accepted_precision": 1.0, - "coverage": 0.7018072289156626 - }, - "awake_count": { - "visible_truth": 121, - "correct": 93, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 28, - "end_to_end_accuracy": 0.768595041322314, - "accepted_precision": 1.0, - "coverage": 0.768595041322314 - } - }, - "hidden_field_predictions": 0 - }, - "codecs:WEBP": { - "identity": { - "expected": 697, - "correct": 623, - "extra": 0, - "missed": 74, - "recall": 0.8938307030129125, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 93, - "correct": 82, - "wrong": 0, - "unknown_after_localization": 11, - "not_localized": 0, - "end_to_end_accuracy": 0.8817204301075269, - "accepted_precision": 1.0, - "coverage": 0.8817204301075269 - }, - "card_rank": { - "visible_truth": 478, - "correct": 353, - "wrong": 0, - "unknown_after_localization": 104, - "not_localized": 21, - "end_to_end_accuracy": 0.7384937238493724, - "accepted_precision": 1.0, - "coverage": 0.7384937238493724 - }, - "awake_count": { - "visible_truth": 17, - "correct": 5, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 12, - "end_to_end_accuracy": 0.29411764705882354, - "accepted_precision": 1.0, - "coverage": 0.29411764705882354 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:720": { - "identity": { - "expected": 759, - "correct": 704, - "extra": 0, - "missed": 55, - "recall": 0.927536231884058, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 108, - "correct": 88, - "wrong": 0, - "unknown_after_localization": 18, - "not_localized": 2, - "end_to_end_accuracy": 0.8148148148148148, - "accepted_precision": 1.0, - "coverage": 0.8148148148148148 - }, - "card_rank": { - "visible_truth": 504, - "correct": 274, - "wrong": 0, - "unknown_after_localization": 203, - "not_localized": 27, - "end_to_end_accuracy": 0.5436507936507936, - "accepted_precision": 1.0, - "coverage": 0.5436507936507936 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:60": { - "identity": { - "expected": 663, - "correct": 505, - "extra": 0, - "missed": 158, - "recall": 0.7616892911010558, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 80, - "correct": 31, - "wrong": 0, - "unknown_after_localization": 34, - "not_localized": 15, - "end_to_end_accuracy": 0.3875, - "accepted_precision": 1.0, - "coverage": 0.3875 - }, - "card_rank": { - "visible_truth": 501, - "correct": 126, - "wrong": 0, - "unknown_after_localization": 269, - "not_localized": 106, - "end_to_end_accuracy": 0.25149700598802394, - "accepted_precision": 1.0, - "coverage": 0.25149700598802394 - }, - "awake_count": { - "visible_truth": 18, - "correct": 18, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[2800, 1752]": { - "identity": { - "expected": 879, - "correct": 860, - "extra": 0, - "missed": 19, - "recall": 0.9783845278725825, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 161, - "correct": 158, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 0, - "end_to_end_accuracy": 0.9813664596273292, - "accepted_precision": 1.0, - "coverage": 0.9813664596273292 - }, - "card_rank": { - "visible_truth": 652, - "correct": 580, - "wrong": 0, - "unknown_after_localization": 64, - "not_localized": 8, - "end_to_end_accuracy": 0.8895705521472392, - "accepted_precision": 1.0, - "coverage": 0.8895705521472392 - }, - "awake_count": { - "visible_truth": 53, - "correct": 52, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9811320754716981, - "accepted_precision": 1.0, - "coverage": 0.9811320754716981 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:180": { - "identity": { - "expected": 414, - "correct": 406, - "extra": 0, - "missed": 8, - "recall": 0.9806763285024155, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 93, - "correct": 93, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 289, - "correct": 276, - "wrong": 0, - "unknown_after_localization": 8, - "not_localized": 5, - "end_to_end_accuracy": 0.9550173010380623, - "accepted_precision": 1.0, - "coverage": 0.9550173010380623 - }, - "awake_count": { - "visible_truth": 16, - "correct": 16, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:240": { - "identity": { - "expected": 363, - "correct": 354, - "extra": 0, - "missed": 9, - "recall": 0.9752066115702479, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 61, - "correct": 61, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 235, - "correct": 229, - "wrong": 0, - "unknown_after_localization": 3, - "not_localized": 3, - "end_to_end_accuracy": 0.9744680851063829, - "accepted_precision": 1.0, - "coverage": 0.9744680851063829 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:450": { - "identity": { - "expected": 20, - "correct": 20, - "extra": 0, - "missed": 0, - "recall": 1.0, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - }, - "card_rank": { - "visible_truth": 16, - "correct": 16, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "locale:en": { - "identity": { - "expected": 917, - "correct": 742, - "extra": 0, - "missed": 175, - "recall": 0.8091603053435115, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 112, - "correct": 102, - "wrong": 0, - "unknown_after_localization": 10, - "not_localized": 0, - "end_to_end_accuracy": 0.9107142857142857, - "accepted_precision": 1.0, - "coverage": 0.9107142857142857 - }, - "card_rank": { - "visible_truth": 687, - "correct": 431, - "wrong": 0, - "unknown_after_localization": 120, - "not_localized": 136, - "end_to_end_accuracy": 0.6273653566229985, - "accepted_precision": 1.0, - "coverage": 0.6273653566229985 - }, - "awake_count": { - "visible_truth": 77, - "correct": 76, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.987012987012987, - "accepted_precision": 1.0, - "coverage": 0.987012987012987 - } - }, - "hidden_field_predictions": 0 - }, - "screen_resolution:[960, 540]": { - "identity": { - "expected": 984, - "correct": 532, - "extra": 0, - "missed": 452, - "recall": 0.540650406504065, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 76, - "correct": 15, - "wrong": 0, - "unknown_after_localization": 22, - "not_localized": 39, - "end_to_end_accuracy": 0.19736842105263158, - "accepted_precision": 1.0, - "coverage": 0.19736842105263158 - }, - "card_rank": { - "visible_truth": 725, - "correct": 196, - "wrong": 0, - "unknown_after_localization": 179, - "not_localized": 350, - "end_to_end_accuracy": 0.27034482758620687, - "accepted_precision": 1.0, - "coverage": 0.27034482758620687 - }, - "awake_count": { - "visible_truth": 58, - "correct": 18, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 40, - "end_to_end_accuracy": 0.3103448275862069, - "accepted_precision": 1.0, - "coverage": 0.3103448275862069 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:360": { - "identity": { - "expected": 911, - "correct": 508, - "extra": 0, - "missed": 403, - "recall": 0.557628979143798, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 94, - "correct": 23, - "wrong": 0, - "unknown_after_localization": 32, - "not_localized": 39, - "end_to_end_accuracy": 0.24468085106382978, - "accepted_precision": 1.0, - "coverage": 0.24468085106382978 - }, - "card_rank": { - "visible_truth": 701, - "correct": 193, - "wrong": 0, - "unknown_after_localization": 184, - "not_localized": 324, - "end_to_end_accuracy": 0.275320970042796, - "accepted_precision": 1.0, - "coverage": 0.275320970042796 - }, - "awake_count": { - "visible_truth": 36, - "correct": 18, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 18, - "end_to_end_accuracy": 0.5, - "accepted_precision": 1.0, - "coverage": 0.5 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:360": { - "identity": { - "expected": 98, - "correct": 98, - "extra": 0, - "missed": 0, - "recall": 1.0, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 23, - "correct": 23, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 57, - "correct": 56, - "wrong": 0, - "unknown_after_localization": 1, - "not_localized": 0, - "end_to_end_accuracy": 0.9824561403508771, - "accepted_precision": 1.0, - "coverage": 0.9824561403508771 - }, - "awake_count": { - "visible_truth": 3, - "correct": 3, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:30": { - "identity": { - "expected": 319, - "correct": 0, - "extra": 0, - "missed": 319, - "recall": 0.0, - "precision_on_scored_regions": null - }, - "fields": { - "level": { - "visible_truth": 24, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 24, - "end_to_end_accuracy": 0.0, - "accepted_precision": null, - "coverage": 0.0 - }, - "card_rank": { - "visible_truth": 259, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 259, - "end_to_end_accuracy": 0.0, - "accepted_precision": null, - "coverage": 0.0 - }, - "awake_count": { - "visible_truth": 40, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 40, - "end_to_end_accuracy": 0.0, - "accepted_precision": null, - "coverage": 0.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:210": { - "identity": { - "expected": 435, - "correct": 426, - "extra": 0, - "missed": 9, - "recall": 0.9793103448275862, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 52, - "correct": 52, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 324, - "correct": 311, - "wrong": 0, - "unknown_after_localization": 9, - "not_localized": 4, - "end_to_end_accuracy": 0.9598765432098766, - "accepted_precision": 1.0, - "coverage": 0.9598765432098766 - }, - "awake_count": { - "visible_truth": 76, - "correct": 76, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:540": { - "identity": { - "expected": 17, - "correct": 17, - "extra": 0, - "missed": 0, - "recall": 1.0, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - }, - "card_rank": { - "visible_truth": 17, - "correct": 17, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "locale:zh-Hant": { - "identity": { - "expected": 880, - "correct": 741, - "extra": 0, - "missed": 139, - "recall": 0.8420454545454545, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 154, - "correct": 154, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 600, - "correct": 460, - "wrong": 0, - "unknown_after_localization": 44, - "not_localized": 96, - "end_to_end_accuracy": 0.7666666666666667, - "accepted_precision": 1.0, - "coverage": 0.7666666666666667 - }, - "awake_count": { - "visible_truth": 36, - "correct": 24, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 12, - "end_to_end_accuracy": 0.6666666666666666, - "accepted_precision": 1.0, - "coverage": 0.6666666666666666 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:2880": { - "identity": { - "expected": 44, - "correct": 42, - "extra": 0, - "missed": 2, - "recall": 0.9545454545454546, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - }, - "card_rank": { - "visible_truth": 38, - "correct": 38, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:390": { - "identity": { - "expected": 77, - "correct": 76, - "extra": 0, - "missed": 1, - "recall": 0.987012987012987, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 10, - "correct": 10, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 54, - "correct": 53, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9814814814814815, - "accepted_precision": 1.0, - "coverage": 0.9814814814814815 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "locale:zh-Hans": { - "identity": { - "expected": 893, - "correct": 807, - "extra": 0, - "missed": 86, - "recall": 0.9036954087346024, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 113, - "correct": 69, - "wrong": 0, - "unknown_after_localization": 18, - "not_localized": 26, - "end_to_end_accuracy": 0.6106194690265486, - "accepted_precision": 1.0, - "coverage": 0.6106194690265486 - }, - "card_rank": { - "visible_truth": 674, - "correct": 481, - "wrong": 0, - "unknown_after_localization": 120, - "not_localized": 73, - "end_to_end_accuracy": 0.7136498516320475, - "accepted_precision": 1.0, - "coverage": 0.7136498516320475 - }, - "awake_count": { - "visible_truth": 56, - "correct": 56, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:420": { - "identity": { - "expected": 44, - "correct": 43, - "extra": 0, - "missed": 1, - "recall": 0.9772727272727273, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - }, - "card_rank": { - "visible_truth": 21, - "correct": 20, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9523809523809523, - "accepted_precision": 1.0, - "coverage": 0.9523809523809523 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:330": { - "identity": { - "expected": 88, - "correct": 86, - "extra": 0, - "missed": 2, - "recall": 0.9772727272727273, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 42, - "correct": 42, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 79, - "correct": 78, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 1, - "end_to_end_accuracy": 0.9873417721518988, - "accepted_precision": 1.0, - "coverage": 0.9873417721518988 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - }, - "locale:ko": { - "identity": { - "expected": 894, - "correct": 807, - "extra": 0, - "missed": 87, - "recall": 0.9026845637583892, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 134, - "correct": 110, - "wrong": 0, - "unknown_after_localization": 16, - "not_localized": 8, - "end_to_end_accuracy": 0.8208955223880597, - "accepted_precision": 1.0, - "coverage": 0.8208955223880597 - }, - "card_rank": { - "visible_truth": 614, - "correct": 474, - "wrong": 0, - "unknown_after_localization": 78, - "not_localized": 62, - "end_to_end_accuracy": 0.7719869706840391, - "accepted_precision": 1.0, - "coverage": 0.7719869706840391 - }, - "awake_count": { - "visible_truth": 47, - "correct": 19, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 28, - "end_to_end_accuracy": 0.40425531914893614, - "accepted_precision": 1.0, - "coverage": 0.40425531914893614 - } - }, - "hidden_field_predictions": 0 - }, - "upload_long_edge_bin:0": { - "identity": { - "expected": 70, - "correct": 0, - "extra": 0, - "missed": 70, - "recall": 0.0, - "precision_on_scored_regions": null - }, - "fields": { - "level": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - }, - "card_rank": { - "visible_truth": 41, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 41, - "end_to_end_accuracy": 0.0, - "accepted_precision": null, - "coverage": 0.0 - }, - "awake_count": { - "visible_truth": 22, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 22, - "end_to_end_accuracy": 0.0, - "accepted_precision": null, - "coverage": 0.0 - } - }, - "hidden_field_predictions": 0 - }, - "card_width_bin:480": { - "identity": { - "expected": 36, - "correct": 32, - "extra": 0, - "missed": 4, - "recall": 0.8888888888888888, - "precision_on_scored_regions": 1.0 - }, - "fields": { - "level": { - "visible_truth": 12, - "correct": 12, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "card_rank": { - "visible_truth": 25, - "correct": 25, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": 1.0, - "accepted_precision": 1.0, - "coverage": 1.0 - }, - "awake_count": { - "visible_truth": 0, - "correct": 0, - "wrong": 0, - "unknown_after_localization": 0, - "not_localized": 0, - "end_to_end_accuracy": null, - "accepted_precision": null, - "coverage": null - } - }, - "hidden_field_predictions": 0 - } - }, - "merged_boxes": { - "count": 70, - "duplicate_invariant": true, - "fields": { - "level": { - "observable": 688, - "correct": 597, - "not_observed": 3288, - "unknown": 91 - }, - "card_rank": { - "observable": 3021, - "correct": 2246, - "unknown": 775, - "not_observed": 955 - }, - "awake_count": { - "observable": 279, - "correct": 238, - "not_observed": 3697, - "unknown": 41 - } - } - } - } - }, - "limits": [ - "Synthetic distribution metrics do not estimate arbitrary deployment accuracy.", - "Cards and views within a player box are correlated.", - "Confidence scores are not calibrated probabilities.", - "Latency excludes file decoding and HTTP; other host workload may be present." - ] -} \ No newline at end of file diff --git a/reports/acceptance/unknown-card-results.json b/reports/acceptance/unknown-card-results.json deleted file mode 100644 index dd12c3a..0000000 --- a/reports/acceptance/unknown-card-results.json +++ /dev/null @@ -1,1735 +0,0 @@ -{ - "excluded_ids": [ - [ - "member", - 7 - ], - [ - "member", - 14 - ], - [ - "member", - 21 - ], - [ - "member", - 28 - ], - [ - "member", - 35 - ], - [ - "member", - 42 - ], - [ - "member", - 49 - ], - [ - "member", - 56 - ], - [ - "snap", - 7 - ], - [ - "snap", - 14 - ], - [ - "snap", - 21 - ], - [ - "snap", - 28 - ], - [ - "snap", - 35 - ], - [ - "snap", - 42 - ], - [ - "snap", - 49 - ], - [ - "snap", - 56 - ], - [ - "snap", - 70 - ] - ], - "unknown_visible_instances": 1258, - "false_known_claims": 0, - "false_known_examples": [], - "known_visible_instances": 7629, - "known_correct": 7579, - "unmatched_predictions": [ - { - "file": "000008.png", - "predicted": [ - "member", - 57 - ] - }, - { - "file": "000008.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000010.png", - "predicted": [ - "member", - 41 - ] - }, - { - "file": "000010.png", - "predicted": [ - "member", - 37 - ] - }, - { - "file": "000010.png", - "predicted": [ - "member", - 33 - ] - }, - { - "file": "000010.png", - "predicted": [ - "member", - 19 - ] - }, - { - "file": "000011.png", - "predicted": [ - "snap", - 59 - ] - }, - { - "file": "000011.png", - "predicted": [ - "snap", - 6 - ] - }, - { - "file": "000023.png", - "predicted": [ - "snap", - 47 - ] - }, - { - "file": "000023.png", - "predicted": [ - "snap", - 34 - ] - }, - { - "file": "000024.png", - "predicted": [ - "member", - 55 - ] - }, - { - "file": "000024.png", - "predicted": [ - "member", - 59 - ] - }, - { - "file": "000028.png", - "predicted": [ - "member", - 50 - ] - }, - { - "file": "000028.png", - "predicted": [ - "member", - 54 - ] - }, - { - "file": "000028.png", - "predicted": [ - "member", - 12 - ] - }, - { - "file": "000028.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000028.png", - "predicted": [ - "member", - 47 - ] - }, - { - "file": "000028.png", - "predicted": [ - "member", - 24 - ] - }, - { - "file": "000032.png", - "predicted": [ - "member", - 31 - ] - }, - { - "file": "000032.png", - "predicted": [ - "member", - 40 - ] - }, - { - "file": "000032.png", - "predicted": [ - "member", - 55 - ] - }, - { - "file": "000034.png", - "predicted": [ - "member", - 34 - ] - }, - { - "file": "000034.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000034.png", - "predicted": [ - "member", - 45 - ] - }, - { - "file": "000034.png", - "predicted": [ - "member", - 33 - ] - }, - { - "file": "000034.png", - "predicted": [ - "member", - 47 - ] - }, - { - "file": "000038.png", - "predicted": [ - "member", - 59 - ] - }, - { - "file": "000038.png", - "predicted": [ - "member", - 54 - ] - }, - { - "file": "000040.png", - "predicted": [ - "member", - 8 - ] - }, - { - "file": "000040.png", - "predicted": [ - "member", - 9 - ] - }, - { - "file": "000040.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000040.png", - "predicted": [ - "member", - 47 - ] - }, - { - "file": "000040.png", - "predicted": [ - "member", - 18 - ] - }, - { - "file": "000040.png", - "predicted": [ - "member", - 2 - ] - }, - { - "file": "000041.png", - "predicted": [ - "snap", - 51 - ] - }, - { - "file": "000041.png", - "predicted": [ - "snap", - 37 - ] - }, - { - "file": "000042.png", - "predicted": [ - "member", - 36 - ] - }, - { - "file": "000042.png", - "predicted": [ - "member", - 13 - ] - }, - { - "file": "000042.png", - "predicted": [ - "member", - 8 - ] - }, - { - "file": "000042.png", - "predicted": [ - "member", - 31 - ] - }, - { - "file": "000055.png", - "predicted": [ - "snap", - 18 - ] - }, - { - "file": "000055.png", - "predicted": [ - "snap", - 38 - ] - }, - { - "file": "000055.png", - "predicted": [ - "snap", - 26 - ] - }, - { - "file": "000058.png", - "predicted": [ - "member", - 53 - ] - }, - { - "file": "000058.png", - "predicted": [ - "member", - 44 - ] - }, - { - "file": "000070.png", - "predicted": [ - "member", - 50 - ] - }, - { - "file": "000070.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000070.png", - "predicted": [ - "member", - 44 - ] - }, - { - "file": "000070.png", - "predicted": [ - "member", - 43 - ] - }, - { - "file": "000071.png", - "predicted": [ - "snap", - 46 - ] - }, - { - "file": "000076.png", - "predicted": [ - "member", - 46 - ] - }, - { - "file": "000094.png", - "predicted": [ - "member", - 10 - ] - }, - { - "file": "000094.png", - "predicted": [ - "member", - 19 - ] - }, - { - "file": "000094.png", - "predicted": [ - "member", - 46 - ] - }, - { - "file": "000101.png", - "predicted": [ - "snap", - 38 - ] - }, - { - "file": "000101.png", - "predicted": [ - "snap", - 41 - ] - }, - { - "file": "000106.png", - "predicted": [ - "member", - 13 - ] - }, - { - "file": "000106.png", - "predicted": [ - "member", - 43 - ] - }, - { - "file": "000106.png", - "predicted": [ - "member", - 37 - ] - }, - { - "file": "000106.png", - "predicted": [ - "member", - 58 - ] - }, - { - "file": "000116.png", - "predicted": [ - "member", - 54 - ] - }, - { - "file": "000118.png", - "predicted": [ - "member", - 39 - ] - }, - { - "file": "000135.png", - "predicted": [ - "snap", - 27 - ] - }, - { - "file": "000135.png", - "predicted": [ - "snap", - 50 - ] - }, - { - "file": "000136.png", - "predicted": [ - "member", - 5 - ] - }, - { - "file": "000136.png", - "predicted": [ - "member", - 4 - ] - }, - { - "file": "000136.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000136.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000136.png", - "predicted": [ - "member", - 39 - ] - }, - { - "file": "000138.png", - "predicted": [ - "member", - 41 - ] - }, - { - "file": "000143.png", - "predicted": [ - "snap", - 37 - ] - }, - { - "file": "000149.png", - "predicted": [ - "snap", - 27 - ] - }, - { - "file": "000149.png", - "predicted": [ - "snap", - 34 - ] - }, - { - "file": "000154.png", - "predicted": [ - "member", - 33 - ] - }, - { - "file": "000154.png", - "predicted": [ - "member", - 43 - ] - }, - { - "file": "000154.png", - "predicted": [ - "member", - 37 - ] - }, - { - "file": "000154.png", - "predicted": [ - "member", - 44 - ] - }, - { - "file": "000154.png", - "predicted": [ - "member", - 24 - ] - }, - { - "file": "000158.png", - "predicted": [ - "member", - 55 - ] - }, - { - "file": "000158.png", - "predicted": [ - "member", - 51 - ] - }, - { - "file": "000158.png", - "predicted": [ - "member", - 57 - ] - }, - { - "file": "000164.png", - "predicted": [ - "member", - 55 - ] - }, - { - "file": "000164.png", - "predicted": [ - "member", - 11 - ] - }, - { - "file": "000164.png", - "predicted": [ - "member", - 10 - ] - }, - { - "file": "000164.png", - "predicted": [ - "member", - 2 - ] - }, - { - "file": "000172.png", - "predicted": [ - "member", - 51 - ] - }, - { - "file": "000172.png", - "predicted": [ - "member", - 53 - ] - }, - { - "file": "000190.png", - "predicted": [ - "member", - 57 - ] - }, - { - "file": "000190.png", - "predicted": [ - "member", - 46 - ] - }, - { - "file": "000190.png", - "predicted": [ - "member", - 58 - ] - }, - { - "file": "000194.png", - "predicted": [ - "member", - 51 - ] - }, - { - "file": "000194.png", - "predicted": [ - "member", - 50 - ] - }, - { - "file": "000194.png", - "predicted": [ - "member", - 27 - ] - }, - { - "file": "000194.png", - "predicted": [ - "member", - 43 - ] - }, - { - "file": "000194.png", - "predicted": [ - "member", - 8 - ] - }, - { - "file": "000194.png", - "predicted": [ - "member", - 26 - ] - }, - { - "file": "000202.png", - "predicted": [ - "member", - 29 - ] - }, - { - "file": "000202.png", - "predicted": [ - "member", - 43 - ] - }, - { - "file": "000202.png", - "predicted": [ - "member", - 10 - ] - }, - { - "file": "000202.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000202.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000203.png", - "predicted": [ - "snap", - 39 - ] - }, - { - "file": "000203.png", - "predicted": [ - "snap", - 43 - ] - }, - { - "file": "000215.png", - "predicted": [ - "snap", - 46 - ] - }, - { - "file": "000215.png", - "predicted": [ - "snap", - 44 - ] - }, - { - "file": "000217.png", - "predicted": [ - "snap", - 54 - ] - }, - { - "file": "000217.png", - "predicted": [ - "snap", - 52 - ] - }, - { - "file": "000217.png", - "predicted": [ - "snap", - 60 - ] - }, - { - "file": "000220.png", - "predicted": [ - "member", - 1 - ] - }, - { - "file": "000220.png", - "predicted": [ - "member", - 19 - ] - }, - { - "file": "000220.png", - "predicted": [ - "member", - 54 - ] - }, - { - "file": "000220.png", - "predicted": [ - "member", - 5 - ] - }, - { - "file": "000220.png", - "predicted": [ - "member", - 17 - ] - }, - { - "file": "000227.png", - "predicted": [ - "snap", - 60 - ] - }, - { - "file": "000227.png", - "predicted": [ - "snap", - 27 - ] - }, - { - "file": "000229.png", - "predicted": [ - "snap", - 45 - ] - }, - { - "file": "000229.png", - "predicted": [ - "snap", - 48 - ] - }, - { - "file": "000232.png", - "predicted": [ - "member", - 43 - ] - }, - { - "file": "000232.png", - "predicted": [ - "member", - 18 - ] - }, - { - "file": "000241.png", - "predicted": [ - "snap", - 10 - ] - }, - { - "file": "000241.png", - "predicted": [ - "snap", - 37 - ] - }, - { - "file": "000248.png", - "predicted": [ - "member", - 11 - ] - }, - { - "file": "000248.png", - "predicted": [ - "member", - 9 - ] - }, - { - "file": "000248.png", - "predicted": [ - "member", - 4 - ] - }, - { - "file": "000248.png", - "predicted": [ - "member", - 25 - ] - }, - { - "file": "000248.png", - "predicted": [ - "member", - 59 - ] - }, - { - "file": "000248.png", - "predicted": [ - "member", - 1 - ] - }, - { - "file": "000268.png", - "predicted": [ - "member", - 39 - ] - }, - { - "file": "000268.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000272.png", - "predicted": [ - "member", - 45 - ] - }, - { - "file": "000272.png", - "predicted": [ - "member", - 22 - ] - }, - { - "file": "000272.png", - "predicted": [ - "member", - 6 - ] - }, - { - "file": "000272.png", - "predicted": [ - "member", - 33 - ] - }, - { - "file": "000272.png", - "predicted": [ - "member", - 18 - ] - }, - { - "file": "000272.png", - "predicted": [ - "member", - 1 - ] - }, - { - "file": "000278.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000278.png", - "predicted": [ - "member", - 31 - ] - }, - { - "file": "000293.png", - "predicted": [ - "snap", - 44 - ] - }, - { - "file": "000299.png", - "predicted": [ - "snap", - 39 - ] - }, - { - "file": "000299.png", - "predicted": [ - "snap", - 55 - ] - }, - { - "file": "000304.png", - "predicted": [ - "member", - 32 - ] - }, - { - "file": "000304.png", - "predicted": [ - "member", - 27 - ] - }, - { - "file": "000304.png", - "predicted": [ - "member", - 1 - ] - }, - { - "file": "000304.png", - "predicted": [ - "member", - 40 - ] - }, - { - "file": "000328.png", - "predicted": [ - "member", - 33 - ] - }, - { - "file": "000328.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000332.png", - "predicted": [ - "member", - 12 - ] - }, - { - "file": "000332.png", - "predicted": [ - "member", - 40 - ] - }, - { - "file": "000332.png", - "predicted": [ - "member", - 57 - ] - }, - { - "file": "000332.png", - "predicted": [ - "member", - 31 - ] - }, - { - "file": "000332.png", - "predicted": [ - "member", - 5 - ] - }, - { - "file": "000338.png", - "predicted": [ - "member", - 51 - ] - }, - { - "file": "000338.png", - "predicted": [ - "member", - 59 - ] - }, - { - "file": "000340.png", - "predicted": [ - "member", - 47 - ] - }, - { - "file": "000340.png", - "predicted": [ - "member", - 34 - ] - }, - { - "file": "000340.png", - "predicted": [ - "member", - 43 - ] - }, - { - "file": "000340.png", - "predicted": [ - "member", - 23 - ] - }, - { - "file": "000356.png", - "predicted": [ - "member", - 20 - ] - }, - { - "file": "000356.png", - "predicted": [ - "member", - 58 - ] - }, - { - "file": "000356.png", - "predicted": [ - "member", - 40 - ] - }, - { - "file": "000356.png", - "predicted": [ - "member", - 15 - ] - }, - { - "file": "000356.png", - "predicted": [ - "member", - 2 - ] - }, - { - "file": "000362.png", - "predicted": [ - "member", - 5 - ] - }, - { - "file": "000362.png", - "predicted": [ - "member", - 19 - ] - }, - { - "file": "000362.png", - "predicted": [ - "member", - 59 - ] - }, - { - "file": "000362.png", - "predicted": [ - "member", - 31 - ] - }, - { - "file": "000362.png", - "predicted": [ - "member", - 39 - ] - }, - { - "file": "000364.png", - "predicted": [ - "member", - 32 - ] - }, - { - "file": "000364.png", - "predicted": [ - "member", - 12 - ] - }, - { - "file": "000364.png", - "predicted": [ - "member", - 46 - ] - }, - { - "file": "000364.png", - "predicted": [ - "member", - 10 - ] - }, - { - "file": "000367.png", - "predicted": [ - "snap", - 23 - ] - }, - { - "file": "000367.png", - "predicted": [ - "snap", - 52 - ] - }, - { - "file": "000370.png", - "predicted": [ - "member", - 33 - ] - }, - { - "file": "000370.png", - "predicted": [ - "member", - 58 - ] - }, - { - "file": "000371.png", - "predicted": [ - "snap", - 51 - ] - }, - { - "file": "000371.png", - "predicted": [ - "snap", - 31 - ] - }, - { - "file": "000371.png", - "predicted": [ - "snap", - 58 - ] - }, - { - "file": "000374.png", - "predicted": [ - "member", - 58 - ] - }, - { - "file": "000374.png", - "predicted": [ - "member", - 26 - ] - }, - { - "file": "000374.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000388.png", - "predicted": [ - "member", - 53 - ] - }, - { - "file": "000388.png", - "predicted": [ - "member", - 39 - ] - }, - { - "file": "000388.png", - "predicted": [ - "member", - 58 - ] - }, - { - "file": "000388.png", - "predicted": [ - "member", - 46 - ] - }, - { - "file": "000394.png", - "predicted": [ - "member", - 59 - ] - }, - { - "file": "000394.png", - "predicted": [ - "member", - 33 - ] - }, - { - "file": "000398.png", - "predicted": [ - "member", - 34 - ] - }, - { - "file": "000401.png", - "predicted": [ - "snap", - 19 - ] - }, - { - "file": "000402.png", - "predicted": [ - "member", - 18 - ] - }, - { - "file": "000402.png", - "predicted": [ - "member", - 15 - ] - }, - { - "file": "000402.png", - "predicted": [ - "member", - 30 - ] - }, - { - "file": "000402.png", - "predicted": [ - "member", - 19 - ] - }, - { - "file": "000402.png", - "predicted": [ - "member", - 1 - ] - }, - { - "file": "000402.png", - "predicted": [ - "member", - 23 - ] - }, - { - "file": "000404.png", - "predicted": [ - "member", - 55 - ] - }, - { - "file": "000404.png", - "predicted": [ - "member", - 46 - ] - }, - { - "file": "000424.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000434.png", - "predicted": [ - "member", - 53 - ] - }, - { - "file": "000434.png", - "predicted": [ - "member", - 13 - ] - }, - { - "file": "000436.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000436.png", - "predicted": [ - "member", - 52 - ] - }, - { - "file": "000448.png", - "predicted": [ - "member", - 46 - ] - }, - { - "file": "000448.png", - "predicted": [ - "member", - 48 - ] - }, - { - "file": "000448.png", - "predicted": [ - "member", - 11 - ] - }, - { - "file": "000448.png", - "predicted": [ - "member", - 50 - ] - }, - { - "file": "000448.png", - "predicted": [ - "member", - 47 - ] - }, - { - "file": "000454.png", - "predicted": [ - "member", - 31 - ] - }, - { - "file": "000454.png", - "predicted": [ - "member", - 32 - ] - }, - { - "file": "000454.png", - "predicted": [ - "member", - 13 - ] - }, - { - "file": "000454.png", - "predicted": [ - "member", - 59 - ] - }, - { - "file": "000454.png", - "predicted": [ - "member", - 53 - ] - }, - { - "file": "000456.png", - "predicted": [ - "member", - 47 - ] - }, - { - "file": "000460.png", - "predicted": [ - "member", - 58 - ] - }, - { - "file": "000460.png", - "predicted": [ - "member", - 48 - ] - }, - { - "file": "000472.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000474.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000474.png", - "predicted": [ - "member", - 40 - ] - }, - { - "file": "000482.png", - "predicted": [ - "member", - 60 - ] - }, - { - "file": "000482.png", - "predicted": [ - "member", - 45 - ] - }, - { - "file": "000482.png", - "predicted": [ - "member", - 51 - ] - }, - { - "file": "000485.png", - "predicted": [ - "snap", - 61 - ] - }, - { - "file": "000485.png", - "predicted": [ - "snap", - 32 - ] - }, - { - "file": "000485.png", - "predicted": [ - "snap", - 37 - ] - }, - { - "file": "000488.png", - "predicted": [ - "member", - 54 - ] - }, - { - "file": "000496.png", - "predicted": [ - "member", - 31 - ] - }, - { - "file": "000496.png", - "predicted": [ - "member", - 54 - ] - }, - { - "file": "000508.png", - "predicted": [ - "member", - 39 - ] - }, - { - "file": "000508.png", - "predicted": [ - "member", - 53 - ] - }, - { - "file": "000508.png", - "predicted": [ - "member", - 45 - ] - }, - { - "file": "000508.png", - "predicted": [ - "member", - 27 - ] - }, - { - "file": "000508.png", - "predicted": [ - "member", - 11 - ] - }, - { - "file": "000509.png", - "predicted": [ - "snap", - 41 - ] - }, - { - "file": "000509.png", - "predicted": [ - "snap", - 55 - ] - }, - { - "file": "000509.png", - "predicted": [ - "snap", - 45 - ] - } - ], - "blank_scene_predictions": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ], - "scope": "same-artwork holdout removed from gallery; identity+IoU matching; no false-field claims inferred from this test" -} \ No newline at end of file diff --git a/reports/performance/cpu-2-logical-cores.json b/reports/performance/cpu-2-logical-cores.json deleted file mode 100644 index cbbce64..0000000 --- a/reports/performance/cpu-2-logical-cores.json +++ /dev/null @@ -1,603 +0,0 @@ -{ - "logical_cpu_affinity": [ - 0, - 1 - ], - "threads": 2, - "platform": "Windows-10-10.0.19044-SP0", - "processor": "Intel64 Family 6 Model 151 Stepping 2, GenuineIntel", - "host_logical_cpus": 12, - "model_sha256": { - "encoder.onnx": "a0a64115241a0eb22e2aab3b151ee3a3f57dc317d88524c027aaa443d051efa1", - "fields.onnx": "768889db7b475dce28ed949299d2e7a624faa3ee18f3c28bb22deac2bcdd1a8d" - }, - "startup_ms": 962.49, - "startup_memory": { - "rss_bytes": 162549760, - "peak_rss_bytes": 203296768 - }, - "final_memory": { - "rss_bytes": 266162176, - "peak_rss_bytes": 628568064 - }, - "input_sha256": { - "member.jpg": "94762016747673434bb8c749b067029419c929d6f98961f606db97755ff17e90", - "snap.jpg": "b4a62b4dc1fe09fa691e8893d1ae62832c8a26f2a8a26fd58043df23c009b4fc", - "member-performance.jpg": "419aa52b9911efc6a8ce0538f43f5dc8ed964d4d5c97d9703fa3851160c244ca", - "member-technic.jpg": "0a4cbdbfbb332cb6a6566ac6dcd92f5a0d6c9ac6f4161753e09a2923c23105e8", - "member-total.jpg": "07296f65c564e92998d48f57b6580361dc30072239f7b77e63d797757d391f40", - "member-training.jpg": "bac4cb7896d318fa9004a628c694d38da5558e4cef1161d2efbf8fe46ec9a1fc", - "member-visual.jpg": "3e25579f69e3000e753f4e95d6428d580ae972563194968b47321906ccb8202c", - "snap-performance.jpg": "ac472e94e10e92aa21042ee706dd75e1d610c2b5caecce16171945a6569a77e8", - "snap-technic.jpg": "4b0d3acabd475328b0cd59338e51280de6f82ae7b59a6d1816114f6f88ac6d9e", - "snap-visual.jpg": "9b9c9824cc2dde88a067d3a430c8b6374c00769f80124b3558d65111873a161f" - }, - "warmup": "one pass of all images", - "repeats": 5, - "median_ms": 1821.565, - "p95_ms": 2427.3035, - "min_ms": 564.7, - "max_ms": 2528.2, - "measurements": [ - { - "file": "member.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2419.51, - "process_cpu_ms": 2671.88 - }, - { - "file": "snap.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 688.38, - "process_cpu_ms": 968.75 - }, - { - "file": "member-performance.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2241.73, - "process_cpu_ms": 2515.62 - }, - { - "file": "member-technic.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2296.64, - "process_cpu_ms": 2578.12 - }, - { - "file": "member-total.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2138.92, - "process_cpu_ms": 2437.5 - }, - { - "file": "member-training.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1999.03, - "process_cpu_ms": 2328.12 - }, - { - "file": "member-visual.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1935.56, - "process_cpu_ms": 2281.25 - }, - { - "file": "snap-performance.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 924.66, - "process_cpu_ms": 1109.38 - }, - { - "file": "snap-technic.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 936.73, - "process_cpu_ms": 984.38 - }, - { - "file": "snap-visual.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 739.13, - "process_cpu_ms": 1000.0 - }, - { - "file": "member.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2434.92, - "process_cpu_ms": 2500.0 - }, - { - "file": "snap.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 982.07, - "process_cpu_ms": 1187.5 - }, - { - "file": "member-performance.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2310.4, - "process_cpu_ms": 2468.75 - }, - { - "file": "member-technic.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2528.2, - "process_cpu_ms": 2593.75 - }, - { - "file": "member-total.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1983.29, - "process_cpu_ms": 2421.88 - }, - { - "file": "member-training.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2346.72, - "process_cpu_ms": 2515.62 - }, - { - "file": "member-visual.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1883.31, - "process_cpu_ms": 2203.12 - }, - { - "file": "snap-performance.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 583.65, - "process_cpu_ms": 859.38 - }, - { - "file": "snap-technic.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 687.98, - "process_cpu_ms": 1031.25 - }, - { - "file": "snap-visual.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 633.24, - "process_cpu_ms": 906.25 - }, - { - "file": "member.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1847.0, - "process_cpu_ms": 2187.5 - }, - { - "file": "snap.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 621.79, - "process_cpu_ms": 843.75 - }, - { - "file": "member-performance.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2066.12, - "process_cpu_ms": 2484.38 - }, - { - "file": "member-technic.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2186.2, - "process_cpu_ms": 2421.88 - }, - { - "file": "member-total.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1950.88, - "process_cpu_ms": 2328.12 - }, - { - "file": "member-training.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2034.19, - "process_cpu_ms": 2156.25 - }, - { - "file": "member-visual.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2433.68, - "process_cpu_ms": 2531.25 - }, - { - "file": "snap-performance.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 697.71, - "process_cpu_ms": 968.75 - }, - { - "file": "snap-technic.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 593.83, - "process_cpu_ms": 921.88 - }, - { - "file": "snap-visual.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 699.51, - "process_cpu_ms": 953.12 - }, - { - "file": "member.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1959.67, - "process_cpu_ms": 2406.25 - }, - { - "file": "snap.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 662.18, - "process_cpu_ms": 890.62 - }, - { - "file": "member-performance.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1905.66, - "process_cpu_ms": 2265.62 - }, - { - "file": "member-technic.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1817.68, - "process_cpu_ms": 2250.0 - }, - { - "file": "member-total.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1759.39, - "process_cpu_ms": 2046.88 - }, - { - "file": "member-training.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2085.4, - "process_cpu_ms": 2453.12 - }, - { - "file": "member-visual.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1959.83, - "process_cpu_ms": 2171.88 - }, - { - "file": "snap-performance.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 591.24, - "process_cpu_ms": 921.88 - }, - { - "file": "snap-technic.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 564.7, - "process_cpu_ms": 859.38 - }, - { - "file": "snap-visual.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 569.43, - "process_cpu_ms": 890.62 - }, - { - "file": "member.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1795.17, - "process_cpu_ms": 2203.12 - }, - { - "file": "snap.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 726.54, - "process_cpu_ms": 953.12 - }, - { - "file": "member-performance.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1734.4, - "process_cpu_ms": 2171.88 - }, - { - "file": "member-technic.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1825.45, - "process_cpu_ms": 2218.75 - }, - { - "file": "member-total.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1720.2, - "process_cpu_ms": 2078.12 - }, - { - "file": "member-training.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1832.44, - "process_cpu_ms": 2250.0 - }, - { - "file": "member-visual.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1841.85, - "process_cpu_ms": 2265.62 - }, - { - "file": "snap-performance.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 710.33, - "process_cpu_ms": 1000.0 - }, - { - "file": "snap-technic.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 630.85, - "process_cpu_ms": 906.25 - }, - { - "file": "snap-visual.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 624.72, - "process_cpu_ms": 890.62 - } - ], - "scope": "Same process constrained to listed logical CPUs. Scan includes localization and field reading; excludes file decoding and HTTP. Other host workloads are not controlled.", - "inference_core_sha256": "bf95bcd9579ae61c7dceedcf883befaa1cff81dd929d592e069249683c86dbfe", - "package_version": "0.1.0", - "cpu_model_verified": "12th Gen Intel(R) Core(TM) i5-12400F", - "runtime_versions": { - "numpy": "2.4.3", - "Pillow": "12.1.1", - "opencv-python-headless": "5.0.0.93", - "onnxruntime": "1.20.1" - } -} \ No newline at end of file diff --git a/reports/performance/cpu-4-logical-cores.json b/reports/performance/cpu-4-logical-cores.json deleted file mode 100644 index be1457c..0000000 --- a/reports/performance/cpu-4-logical-cores.json +++ /dev/null @@ -1,605 +0,0 @@ -{ - "logical_cpu_affinity": [ - 0, - 1, - 2, - 3 - ], - "threads": 4, - "platform": "Windows-10-10.0.19044-SP0", - "processor": "Intel64 Family 6 Model 151 Stepping 2, GenuineIntel", - "host_logical_cpus": 12, - "model_sha256": { - "encoder.onnx": "a0a64115241a0eb22e2aab3b151ee3a3f57dc317d88524c027aaa443d051efa1", - "fields.onnx": "768889db7b475dce28ed949299d2e7a624faa3ee18f3c28bb22deac2bcdd1a8d" - }, - "startup_ms": 1133.48, - "startup_memory": { - "rss_bytes": 161935360, - "peak_rss_bytes": 203960320 - }, - "final_memory": { - "rss_bytes": 266350592, - "peak_rss_bytes": 631832576 - }, - "input_sha256": { - "member.jpg": "94762016747673434bb8c749b067029419c929d6f98961f606db97755ff17e90", - "snap.jpg": "b4a62b4dc1fe09fa691e8893d1ae62832c8a26f2a8a26fd58043df23c009b4fc", - "member-performance.jpg": "419aa52b9911efc6a8ce0538f43f5dc8ed964d4d5c97d9703fa3851160c244ca", - "member-technic.jpg": "0a4cbdbfbb332cb6a6566ac6dcd92f5a0d6c9ac6f4161753e09a2923c23105e8", - "member-total.jpg": "07296f65c564e92998d48f57b6580361dc30072239f7b77e63d797757d391f40", - "member-training.jpg": "bac4cb7896d318fa9004a628c694d38da5558e4cef1161d2efbf8fe46ec9a1fc", - "member-visual.jpg": "3e25579f69e3000e753f4e95d6428d580ae972563194968b47321906ccb8202c", - "snap-performance.jpg": "ac472e94e10e92aa21042ee706dd75e1d610c2b5caecce16171945a6569a77e8", - "snap-technic.jpg": "4b0d3acabd475328b0cd59338e51280de6f82ae7b59a6d1816114f6f88ac6d9e", - "snap-visual.jpg": "9b9c9824cc2dde88a067d3a430c8b6374c00769f80124b3558d65111873a161f" - }, - "warmup": "one pass of all images", - "repeats": 5, - "median_ms": 1661.27, - "p95_ms": 2900.105, - "min_ms": 447.75, - "max_ms": 6893.82, - "measurements": [ - { - "file": "member.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1458.93, - "process_cpu_ms": 2500.0 - }, - { - "file": "snap.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 463.05, - "process_cpu_ms": 1078.12 - }, - { - "file": "member-performance.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1727.43, - "process_cpu_ms": 2593.75 - }, - { - "file": "member-technic.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2879.81, - "process_cpu_ms": 3078.12 - }, - { - "file": "member-total.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2879.47, - "process_cpu_ms": 3078.12 - }, - { - "file": "member-training.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1984.73, - "process_cpu_ms": 2906.25 - }, - { - "file": "member-visual.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2306.83, - "process_cpu_ms": 2828.12 - }, - { - "file": "snap-performance.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 605.55, - "process_cpu_ms": 1218.75 - }, - { - "file": "snap-technic.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 785.48, - "process_cpu_ms": 1281.25 - }, - { - "file": "snap-visual.jpg", - "repeat": 0, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 592.59, - "process_cpu_ms": 1140.62 - }, - { - "file": "member.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2070.74, - "process_cpu_ms": 2968.75 - }, - { - "file": "snap.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 760.22, - "process_cpu_ms": 1265.62 - }, - { - "file": "member-performance.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2101.23, - "process_cpu_ms": 2796.88 - }, - { - "file": "member-technic.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2072.7, - "process_cpu_ms": 3000.0 - }, - { - "file": "member-total.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2523.86, - "process_cpu_ms": 3078.12 - }, - { - "file": "member-training.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2339.93, - "process_cpu_ms": 3156.25 - }, - { - "file": "member-visual.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1833.58, - "process_cpu_ms": 2656.25 - }, - { - "file": "snap-performance.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 608.63, - "process_cpu_ms": 1234.38 - }, - { - "file": "snap-technic.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 552.64, - "process_cpu_ms": 1187.5 - }, - { - "file": "snap-visual.jpg", - "repeat": 1, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 532.79, - "process_cpu_ms": 1078.12 - }, - { - "file": "member.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1515.57, - "process_cpu_ms": 2593.75 - }, - { - "file": "snap.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 447.75, - "process_cpu_ms": 1109.38 - }, - { - "file": "member-performance.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1595.11, - "process_cpu_ms": 2546.88 - }, - { - "file": "member-technic.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1766.09, - "process_cpu_ms": 2765.62 - }, - { - "file": "member-total.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1577.44, - "process_cpu_ms": 2500.0 - }, - { - "file": "member-training.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2764.49, - "process_cpu_ms": 3109.38 - }, - { - "file": "member-visual.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 3075.36, - "process_cpu_ms": 3468.75 - }, - { - "file": "snap-performance.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 1070.7, - "process_cpu_ms": 1406.25 - }, - { - "file": "snap-technic.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 1060.56, - "process_cpu_ms": 1390.62 - }, - { - "file": "snap-visual.jpg", - "repeat": 2, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 782.51, - "process_cpu_ms": 1234.38 - }, - { - "file": "member.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2072.22, - "process_cpu_ms": 2921.88 - }, - { - "file": "snap.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 769.12, - "process_cpu_ms": 1109.38 - }, - { - "file": "member-performance.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2592.36, - "process_cpu_ms": 3125.0 - }, - { - "file": "member-technic.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2916.71, - "process_cpu_ms": 3203.12 - }, - { - "file": "member-total.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 6893.82, - "process_cpu_ms": 3687.5 - }, - { - "file": "member-training.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2044.5, - "process_cpu_ms": 2843.75 - }, - { - "file": "member-visual.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1551.61, - "process_cpu_ms": 2500.0 - }, - { - "file": "snap-performance.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 615.34, - "process_cpu_ms": 1015.62 - }, - { - "file": "snap-technic.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 600.86, - "process_cpu_ms": 1250.0 - }, - { - "file": "snap-visual.jpg", - "repeat": 3, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 532.02, - "process_cpu_ms": 1078.12 - }, - { - "file": "member.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1911.38, - "process_cpu_ms": 2765.62 - }, - { - "file": "snap.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 699.63, - "process_cpu_ms": 1359.38 - }, - { - "file": "member-performance.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1890.87, - "process_cpu_ms": 3093.75 - }, - { - "file": "member-technic.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1761.52, - "process_cpu_ms": 2609.38 - }, - { - "file": "member-total.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1966.79, - "process_cpu_ms": 2859.38 - }, - { - "file": "member-training.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 1986.95, - "process_cpu_ms": 2937.5 - }, - { - "file": "member-visual.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 24, - "elapsed_ms": 2172.58, - "process_cpu_ms": 3000.0 - }, - { - "file": "snap-performance.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 495.71, - "process_cpu_ms": 1031.25 - }, - { - "file": "snap-technic.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 508.53, - "process_cpu_ms": 1171.88 - }, - { - "file": "snap-visual.jpg", - "repeat": 4, - "pixels": [ - 1441, - 1080 - ], - "cards": 3, - "elapsed_ms": 543.64, - "process_cpu_ms": 1125.0 - } - ], - "scope": "Same process constrained to listed logical CPUs. Scan includes localization and field reading; excludes file decoding and HTTP. Other host workloads are not controlled.", - "inference_core_sha256": "bf95bcd9579ae61c7dceedcf883befaa1cff81dd929d592e069249683c86dbfe", - "package_version": "0.1.0", - "cpu_model_verified": "12th Gen Intel(R) Core(TM) i5-12400F", - "runtime_versions": { - "numpy": "2.4.3", - "Pillow": "12.1.1", - "opencv-python-headless": "5.0.0.93", - "onnxruntime": "1.20.1" - } -} \ No newline at end of file diff --git a/reports/training/card-coverage.json b/reports/training/card-coverage.json deleted file mode 100644 index 67043c8..0000000 --- a/reports/training/card-coverage.json +++ /dev/null @@ -1,570 +0,0 @@ -{ - "schema": "ournotes-boxlens.coverage/1", - "catalog_sha256": "902b59ed37ea50e601187b4ccf387cacfbae61c714f52f524f42d6d320df7818", - "catalog": { - "count": 122, - "by_kind_rarity": { - "member/R": 25, - "member/SR": 25, - "member/SSR": 10, - "snap/EX": 2, - "snap/R": 25, - "snap/SR": 25, - "snap/SSR": 10 - }, - "by_kind_type": { - "member/Blue": 12, - "member/Green": 12, - "member/Purple": 12, - "member/Red": 12, - "member/Yellow": 12, - "snap/Blue": 12, - "snap/Green": 12, - "snap/Purple": 13, - "snap/Red": 14, - "snap/Yellow": 11 - } - }, - "rarity_enum": { - "2": "R", - "3": "SR", - "4": "SSR", - "10": "EX", - "20": "BD" - }, - "card_type_enum": { - "1": "Red", - "2": "Blue", - "3": "Green", - "4": "Yellow", - "5": "Purple" - }, - "enum_evidence": { - "rarity_type_index": 34458, - "card_type_type_index": 19592, - "client": "1.0.1-25", - "metadata_hash_verified": true - }, - "absent_enum_rarities": [ - "BD" - ], - "datasets": { - "work\\box-vision\\remote-audit-v13\\synthetic\\scale-v6\\train\\truth.json": { - "truth_sha256": "bbb1b2aea52b1bd04f6bd6b09f6130f82111cd825c7b94493f339716826d353e", - "screenshots": 2048, - "unique": { - "count": 105, - "by_kind_rarity": { - "member/R": 22, - "member/SR": 21, - "member/SSR": 9, - "snap/EX": 1, - "snap/R": 22, - "snap/SR": 21, - "snap/SSR": 9 - }, - "by_kind_type": { - "member/Blue": 10, - "member/Green": 11, - "member/Purple": 9, - "member/Red": 10, - "member/Yellow": 12, - "snap/Blue": 12, - "snap/Green": 11, - "snap/Purple": 10, - "snap/Red": 11, - "snap/Yellow": 9 - } - }, - "observations": { - "count": 35483, - "by_kind_rarity": { - "member/R": 7372, - "member/SR": 6930, - "member/SSR": 2972, - "snap/EX": 344, - "snap/R": 7480, - "snap/SR": 7291, - "snap/SSR": 3094 - }, - "by_kind_type": { - "member/Blue": 3338, - "member/Green": 3657, - "member/Purple": 2997, - "member/Red": 3321, - "member/Yellow": 3961, - "snap/Blue": 4086, - "snap/Green": 3771, - "snap/Purple": 3492, - "snap/Red": 3754, - "snap/Yellow": 3106 - } - }, - "missing_catalog_ids": [ - [ - "member", - 7 - ], - [ - "member", - 14 - ], - [ - "member", - 21 - ], - [ - "member", - 28 - ], - [ - "member", - 35 - ], - [ - "member", - 42 - ], - [ - "member", - 49 - ], - [ - "member", - 56 - ], - [ - "snap", - 7 - ], - [ - "snap", - 14 - ], - [ - "snap", - 21 - ], - [ - "snap", - 28 - ], - [ - "snap", - 35 - ], - [ - "snap", - 42 - ], - [ - "snap", - 49 - ], - [ - "snap", - 56 - ], - [ - "snap", - 70 - ] - ], - "modes": { - "unrecorded": 35483 - }, - "geometric_visibility": { - "whole_card": 31989, - "partial_card": 3494 - } - }, - "work\\box-vision\\remote-audit-v13\\synthetic\\modes-v7\\train\\truth.json": { - "truth_sha256": "ceb7bc7c1fe2e634de8633455bc80867fa14007bcb0d3e60201cf8a87004228d", - "screenshots": 350, - "unique": { - "count": 105, - "by_kind_rarity": { - "member/R": 22, - "member/SR": 21, - "member/SSR": 9, - "snap/EX": 1, - "snap/R": 22, - "snap/SR": 21, - "snap/SSR": 9 - }, - "by_kind_type": { - "member/Blue": 10, - "member/Green": 11, - "member/Purple": 9, - "member/Red": 10, - "member/Yellow": 12, - "snap/Blue": 12, - "snap/Green": 11, - "snap/Purple": 10, - "snap/Red": 11, - "snap/Yellow": 9 - } - }, - "observations": { - "count": 5935, - "by_kind_rarity": { - "member/R": 1216, - "member/SR": 1173, - "member/SSR": 572, - "snap/EX": 50, - "snap/R": 1264, - "snap/SR": 1159, - "snap/SSR": 501 - }, - "by_kind_type": { - "member/Blue": 612, - "member/Green": 635, - "member/Purple": 491, - "member/Red": 561, - "member/Yellow": 662, - "snap/Blue": 698, - "snap/Green": 604, - "snap/Purple": 551, - "snap/Red": 598, - "snap/Yellow": 523 - } - }, - "missing_catalog_ids": [ - [ - "member", - 7 - ], - [ - "member", - 14 - ], - [ - "member", - 21 - ], - [ - "member", - 28 - ], - [ - "member", - 35 - ], - [ - "member", - 42 - ], - [ - "member", - 49 - ], - [ - "member", - 56 - ], - [ - "snap", - 7 - ], - [ - "snap", - 14 - ], - [ - "snap", - 21 - ], - [ - "snap", - 28 - ], - [ - "snap", - 35 - ], - [ - "snap", - 42 - ], - [ - "snap", - 49 - ], - [ - "snap", - 56 - ], - [ - "snap", - 70 - ] - ], - "modes": { - "level": 1057, - "training": 456, - "performance": 939, - "total": 458, - "technic": 984, - "visual": 1003, - "hide": 1038 - }, - "geometric_visibility": { - "whole_card": 5321, - "partial_card": 614 - } - }, - "work\\box-vision\\audit-scenes-v8\\scale-v6\\validation\\truth.json": { - "truth_sha256": "ec7ca137870ebbd47c4bd4c988ae3a1809712ed9b8f8c08f0b160d158b58d8c4", - "screenshots": 512, - "unique": { - "count": 122, - "by_kind_rarity": { - "member/R": 25, - "member/SR": 25, - "member/SSR": 10, - "snap/EX": 2, - "snap/R": 25, - "snap/SR": 25, - "snap/SSR": 10 - }, - "by_kind_type": { - "member/Blue": 12, - "member/Green": 12, - "member/Purple": 12, - "member/Red": 12, - "member/Yellow": 12, - "snap/Blue": 12, - "snap/Green": 12, - "snap/Purple": 13, - "snap/Red": 14, - "snap/Yellow": 11 - } - }, - "observations": { - "count": 8887, - "by_kind_rarity": { - "member/R": 1777, - "member/SR": 1814, - "member/SSR": 742, - "snap/EX": 145, - "snap/R": 1830, - "snap/SR": 1846, - "snap/SSR": 733 - }, - "by_kind_type": { - "member/Blue": 864, - "member/Green": 847, - "member/Purple": 878, - "member/Red": 861, - "member/Yellow": 883, - "snap/Blue": 860, - "snap/Green": 863, - "snap/Purple": 1003, - "snap/Red": 991, - "snap/Yellow": 837 - } - }, - "missing_catalog_ids": [], - "modes": { - "unrecorded": 8887 - }, - "geometric_visibility": { - "whole_card": 8119, - "partial_card": 768 - } - }, - "work\\box-vision\\audit-scenes-v8\\scale-v6\\stress\\truth.json": { - "truth_sha256": "92349a86c6ad6420249ba9e538c4deae7264801b2f0d77ed5622ba88c6bf54c0", - "screenshots": 256, - "unique": { - "count": 122, - "by_kind_rarity": { - "member/R": 25, - "member/SR": 25, - "member/SSR": 10, - "snap/EX": 2, - "snap/R": 25, - "snap/SR": 25, - "snap/SSR": 10 - }, - "by_kind_type": { - "member/Blue": 12, - "member/Green": 12, - "member/Purple": 12, - "member/Red": 12, - "member/Yellow": 12, - "snap/Blue": 12, - "snap/Green": 12, - "snap/Purple": 13, - "snap/Red": 14, - "snap/Yellow": 11 - } - }, - "observations": { - "count": 4273, - "by_kind_rarity": { - "member/R": 876, - "member/SR": 875, - "member/SSR": 349, - "snap/EX": 73, - "snap/R": 888, - "snap/SR": 854, - "snap/SSR": 358 - }, - "by_kind_type": { - "member/Blue": 444, - "member/Green": 422, - "member/Purple": 388, - "member/Red": 425, - "member/Yellow": 421, - "snap/Blue": 448, - "snap/Green": 431, - "snap/Purple": 463, - "snap/Red": 468, - "snap/Yellow": 363 - } - }, - "missing_catalog_ids": [], - "modes": { - "unrecorded": 4273 - }, - "geometric_visibility": { - "whole_card": 3776, - "partial_card": 497 - } - }, - "work\\box-vision\\audit-scenes-v8\\modes-v7\\validation\\truth.json": { - "truth_sha256": "521b9bd19836ed2333cfc115018a35b3f2642e63ecec39a5e257f64406fe7138", - "screenshots": 140, - "unique": { - "count": 122, - "by_kind_rarity": { - "member/R": 25, - "member/SR": 25, - "member/SSR": 10, - "snap/EX": 2, - "snap/R": 25, - "snap/SR": 25, - "snap/SSR": 10 - }, - "by_kind_type": { - "member/Blue": 12, - "member/Green": 12, - "member/Purple": 12, - "member/Red": 12, - "member/Yellow": 12, - "snap/Blue": 12, - "snap/Green": 12, - "snap/Purple": 13, - "snap/Red": 14, - "snap/Yellow": 11 - } - }, - "observations": { - "count": 2344, - "by_kind_rarity": { - "member/R": 474, - "member/SR": 510, - "member/SSR": 190, - "snap/EX": 42, - "snap/R": 461, - "snap/SR": 472, - "snap/SSR": 195 - }, - "by_kind_type": { - "member/Blue": 221, - "member/Green": 233, - "member/Purple": 238, - "member/Red": 240, - "member/Yellow": 242, - "snap/Blue": 220, - "snap/Green": 213, - "snap/Purple": 246, - "snap/Red": 271, - "snap/Yellow": 220 - } - }, - "missing_catalog_ids": [], - "modes": { - "level": 435, - "training": 176, - "performance": 370, - "total": 173, - "technic": 421, - "visual": 395, - "hide": 374 - }, - "geometric_visibility": { - "whole_card": 2165, - "partial_card": 179 - } - }, - "work\\box-vision\\audit-scenes-v8\\modes-v7\\stress\\truth.json": { - "truth_sha256": "1540ced231e83c2c4b716b52312d9ad2e20367416182473be6c2e0e4fce8553e", - "screenshots": 70, - "unique": { - "count": 122, - "by_kind_rarity": { - "member/R": 25, - "member/SR": 25, - "member/SSR": 10, - "snap/EX": 2, - "snap/R": 25, - "snap/SR": 25, - "snap/SSR": 10 - }, - "by_kind_type": { - "member/Blue": 12, - "member/Green": 12, - "member/Purple": 12, - "member/Red": 12, - "member/Yellow": 12, - "snap/Blue": 12, - "snap/Green": 12, - "snap/Purple": 13, - "snap/Red": 14, - "snap/Yellow": 11 - } - }, - "observations": { - "count": 1126, - "by_kind_rarity": { - "member/R": 229, - "member/SR": 249, - "member/SSR": 96, - "snap/EX": 13, - "snap/R": 225, - "snap/SR": 242, - "snap/SSR": 72 - }, - "by_kind_type": { - "member/Blue": 115, - "member/Green": 116, - "member/Purple": 122, - "member/Red": 118, - "member/Yellow": 103, - "snap/Blue": 109, - "snap/Green": 115, - "snap/Purple": 120, - "snap/Red": 112, - "snap/Yellow": 96 - } - }, - "missing_catalog_ids": [], - "modes": { - "level": 162, - "training": 82, - "performance": 209, - "total": 62, - "technic": 203, - "visual": 217, - "hide": 191 - }, - "geometric_visibility": { - "whole_card": 990, - "partial_card": 136 - } - } - }, - "scope": "Observed identities in saved truth; presence is not accuracy. Catalog snapshot only; no assertion of all current game cards or all UI styles." -} \ No newline at end of file diff --git a/reports/training/checkpoint-selection.json b/reports/training/checkpoint-selection.json deleted file mode 100644 index 7e52dce..0000000 --- a/reports/training/checkpoint-selection.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "policy": "v10-field-confidence-0.995", - "encoder": "v11 additional2400steps", - "fields": "v12 screen-acquisition step5500", - "selection_data": "development scenes-validation and scenes-stress only", - "rule": "preserve ordinary numeric correct count and zero accepted errors; stress wrong+false-visible no greater than baseline; maximize correct stress fields among eligible checkpoints", - "baseline": { - "normal_correct": 8669, - "normal_wrong": 0, - "stress_correct": 2836, - "stress_wrong": 1 - }, - "chosen": { - "normal_correct": 8671, - "normal_wrong": 0, - "stress_correct": 3245, - "stress_wrong": 1 - }, - "final_test_metrics_inspected_before_selection": false, - "files": { - "encoder.onnx": "a0a64115241a0eb22e2aab3b151ee3a3f57dc317d88524c027aaa443d051efa1", - "encoder.pt": "8d67b87580188473d3408c4c1995f0e20b943d40ddcd2f8fc6a0f7a7fd26abda", - "fields.onnx": "768889db7b475dce28ed949299d2e7a624faa3ee18f3c28bb22deac2bcdd1a8d", - "fields.pt": "15351548a593c67cd939b72c3b03908e2665be5c39a0ca8148bd0cfc33feb0ce" - } -} \ No newline at end of file diff --git a/reports/training/field-learning-curve.json b/reports/training/field-learning-curve.json deleted file mode 100644 index 7d02710..0000000 --- a/reports/training/field-learning-curve.json +++ /dev/null @@ -1,499 +0,0 @@ -{ - "checkpoints": [ - { - "step": 0, - "train_loss": null, - "elapsed_s": 7.042200013529509e-05, - "validation": { - "samples": 10278, - "cross_entropy": 3.675770099897678e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8669, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8669, - "numeric_precision": 1.0, - "numeric_coverage": 0.9993083573487032 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 2.5390430733447578, - "raw_correct": 3829, - "visible_truth": 4058, - "numeric_correct": 2836, - "numeric_wrong": 1, - "false_visible": 0, - "accepted_numeric": 2837, - "numeric_precision": 0.9996475149806133, - "numeric_coverage": 0.6988664366683095 - } - }, - { - "step": 500, - "train_loss": 0.09132587164640427, - "elapsed_s": 15.111585792999904, - "validation": { - "samples": 10278, - "cross_entropy": 0.0006108868760325915, - "raw_correct": 10277, - "visible_truth": 8675, - "numeric_correct": 8647, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8647, - "numeric_precision": 1.0, - "numeric_coverage": 0.9967723342939482 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7669240415872289, - "raw_correct": 4074, - "visible_truth": 4058, - "numeric_correct": 2889, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 2889, - "numeric_precision": 1.0, - "numeric_coverage": 0.7119270576638739 - } - }, - { - "step": 1000, - "train_loss": 0.09128552675247192, - "elapsed_s": 27.311867651000284, - "validation": { - "samples": 10278, - "cross_entropy": 0.0001453194820200131, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8634, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8634, - "numeric_precision": 1.0, - "numeric_coverage": 0.9952737752161384 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7635507690041833, - "raw_correct": 4118, - "visible_truth": 4058, - "numeric_correct": 2923, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 2923, - "numeric_precision": 1.0, - "numeric_coverage": 0.7203055692459339 - } - }, - { - "step": 1500, - "train_loss": 0.10142043232917786, - "elapsed_s": 39.44284239700028, - "validation": { - "samples": 10278, - "cross_entropy": 0.0003346889207292204, - "raw_correct": 10276, - "visible_truth": 8675, - "numeric_correct": 8648, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8648, - "numeric_precision": 1.0, - "numeric_coverage": 0.9968876080691642 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.6338517496866194, - "raw_correct": 4202, - "visible_truth": 4058, - "numeric_correct": 2996, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 2996, - "numeric_precision": 1.0, - "numeric_coverage": 0.7382947264662395 - } - }, - { - "step": 2000, - "train_loss": 0.05872093513607979, - "elapsed_s": 51.6124459499988, - "validation": { - "samples": 10278, - "cross_entropy": 0.00013889489101641435, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8660, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8660, - "numeric_precision": 1.0, - "numeric_coverage": 0.9982708933717579 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.6158585762233619, - "raw_correct": 4251, - "visible_truth": 4058, - "numeric_correct": 3044, - "numeric_wrong": 1, - "false_visible": 0, - "accepted_numeric": 3045, - "numeric_precision": 0.999671592775041, - "numeric_coverage": 0.7501232134056185 - } - }, - { - "step": 2500, - "train_loss": 0.04945474863052368, - "elapsed_s": 63.76623347399982, - "validation": { - "samples": 10278, - "cross_entropy": 5.6292341478291474e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8663, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8663, - "numeric_precision": 1.0, - "numeric_coverage": 0.9986167146974063 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.6309616813533536, - "raw_correct": 4226, - "visible_truth": 4058, - "numeric_correct": 3062, - "numeric_wrong": 1, - "false_visible": 0, - "accepted_numeric": 3063, - "numeric_precision": 0.9996735226901731, - "numeric_coverage": 0.7545588960078856 - } - }, - { - "step": 3000, - "train_loss": 0.024036047980189323, - "elapsed_s": 75.92363203700006, - "validation": { - "samples": 10278, - "cross_entropy": 6.66658003976416e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8663, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8663, - "numeric_precision": 1.0, - "numeric_coverage": 0.9986167146974063 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7032289374931511, - "raw_correct": 4216, - "visible_truth": 4058, - "numeric_correct": 3108, - "numeric_wrong": 1, - "false_visible": 0, - "accepted_numeric": 3109, - "numeric_precision": 0.9996783531682213, - "numeric_coverage": 0.7658945293247905 - } - }, - { - "step": 3500, - "train_loss": 0.01852165162563324, - "elapsed_s": 88.16779316999964, - "validation": { - "samples": 10278, - "cross_entropy": 0.00010820655988561583, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8655, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8655, - "numeric_precision": 1.0, - "numeric_coverage": 0.9976945244956772 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.5914165698242818, - "raw_correct": 4266, - "visible_truth": 4058, - "numeric_correct": 3136, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 3136, - "numeric_precision": 1.0, - "numeric_coverage": 0.7727944800394283 - } - }, - { - "step": 4000, - "train_loss": 0.023388206958770752, - "elapsed_s": 100.32361364799908, - "validation": { - "samples": 10278, - "cross_entropy": 5.0441540648751985e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8667, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8667, - "numeric_precision": 1.0, - "numeric_coverage": 0.9990778097982709 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7854994949069637, - "raw_correct": 4243, - "visible_truth": 4058, - "numeric_correct": 3170, - "numeric_wrong": 2, - "false_visible": 0, - "accepted_numeric": 3172, - "numeric_precision": 0.9993694829760403, - "numeric_coverage": 0.7811729916214885 - } - }, - { - "step": 4500, - "train_loss": 0.03287220746278763, - "elapsed_s": 112.49909522999951, - "validation": { - "samples": 10278, - "cross_entropy": 0.00018233103606218366, - "raw_correct": 10277, - "visible_truth": 8675, - "numeric_correct": 8648, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8648, - "numeric_precision": 1.0, - "numeric_coverage": 0.9968876080691642 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.6995464876974681, - "raw_correct": 4234, - "visible_truth": 4058, - "numeric_correct": 3135, - "numeric_wrong": 2, - "false_visible": 0, - "accepted_numeric": 3137, - "numeric_precision": 0.9993624481989162, - "numeric_coverage": 0.7725480532281912 - } - }, - { - "step": 5000, - "train_loss": 0.006961233913898468, - "elapsed_s": 124.65819761700004, - "validation": { - "samples": 10278, - "cross_entropy": 3.676820720320683e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8664, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8664, - "numeric_precision": 1.0, - "numeric_coverage": 0.9987319884726225 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.703947842182712, - "raw_correct": 4275, - "visible_truth": 4058, - "numeric_correct": 3186, - "numeric_wrong": 1, - "false_visible": 0, - "accepted_numeric": 3187, - "numeric_precision": 0.9996862252902416, - "numeric_coverage": 0.7851158206012814 - } - }, - { - "step": 5500, - "train_loss": 0.010203785263001919, - "elapsed_s": 136.805964838999, - "validation": { - "samples": 10278, - "cross_entropy": 1.3232420637136333e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8671, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8671, - "numeric_precision": 1.0, - "numeric_coverage": 0.9995389048991354 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.6462320788730938, - "raw_correct": 4309, - "visible_truth": 4058, - "numeric_correct": 3245, - "numeric_wrong": 1, - "false_visible": 0, - "accepted_numeric": 3246, - "numeric_precision": 0.9996919285274184, - "numeric_coverage": 0.7996550024642681 - } - }, - { - "step": 6000, - "train_loss": 0.012832986190915108, - "elapsed_s": 148.961909353, - "validation": { - "samples": 10278, - "cross_entropy": 0.0001034889966256715, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8661, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8661, - "numeric_precision": 1.0, - "numeric_coverage": 0.998386167146974 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7798035010298217, - "raw_correct": 4258, - "visible_truth": 4058, - "numeric_correct": 3195, - "numeric_wrong": 1, - "false_visible": 0, - "accepted_numeric": 3196, - "numeric_precision": 0.9996871088861077, - "numeric_coverage": 0.7873336619024149 - } - }, - { - "step": 6500, - "train_loss": 0.007399114780128002, - "elapsed_s": 161.12536387399996, - "validation": { - "samples": 10278, - "cross_entropy": 3.7486831888542617e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8665, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8665, - "numeric_precision": 1.0, - "numeric_coverage": 0.9988472622478386 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7477295131174881, - "raw_correct": 4275, - "visible_truth": 4058, - "numeric_correct": 3250, - "numeric_wrong": 4, - "false_visible": 0, - "accepted_numeric": 3254, - "numeric_precision": 0.9987707437000615, - "numeric_coverage": 0.8008871365204534 - } - }, - { - "step": 7000, - "train_loss": 0.014253060333430767, - "elapsed_s": 173.32757122600015, - "validation": { - "samples": 10278, - "cross_entropy": 2.050999839067503e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8670, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8670, - "numeric_precision": 1.0, - "numeric_coverage": 0.9994236311239193 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7918146521277998, - "raw_correct": 4286, - "visible_truth": 4058, - "numeric_correct": 3246, - "numeric_wrong": 2, - "false_visible": 0, - "accepted_numeric": 3248, - "numeric_precision": 0.999384236453202, - "numeric_coverage": 0.7999014292755052 - } - }, - { - "step": 7500, - "train_loss": 0.0146775022149086, - "elapsed_s": 185.45766124500005, - "validation": { - "samples": 10278, - "cross_entropy": 4.3793693203201776e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8669, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8669, - "numeric_precision": 1.0, - "numeric_coverage": 0.9993083573487032 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7994022722159518, - "raw_correct": 4294, - "visible_truth": 4058, - "numeric_correct": 3253, - "numeric_wrong": 4, - "false_visible": 0, - "accepted_numeric": 3257, - "numeric_precision": 0.9987718759594719, - "numeric_coverage": 0.8016264169541646 - } - }, - { - "step": 8000, - "train_loss": 0.009320268407464027, - "elapsed_s": 197.62880408300043, - "validation": { - "samples": 10278, - "cross_entropy": 5.81605661810712e-05, - "raw_correct": 10278, - "visible_truth": 8675, - "numeric_correct": 8659, - "numeric_wrong": 0, - "false_visible": 0, - "accepted_numeric": 8659, - "numeric_precision": 1.0, - "numeric_coverage": 0.9981556195965418 - }, - "development_stress": { - "samples": 4839, - "cross_entropy": 0.7361146373476808, - "raw_correct": 4301, - "visible_truth": 4058, - "numeric_correct": 3281, - "numeric_wrong": 4, - "false_visible": 0, - "accepted_numeric": 3285, - "numeric_precision": 0.9987823439878234, - "numeric_coverage": 0.8085263676688024 - } - } - ], - "initial_sha256": "351db40ac89f611480d05131b7e626e92a928cfdd71278865dd9e87eb2d04b1c", - "scope": "Development corpus only; additional fine-tuning with fresh AdamW state, not an uninterrupted optimizer resume" -} \ No newline at end of file diff --git a/reports/training/hard-field-validation.json b/reports/training/hard-field-validation.json deleted file mode 100644 index 9864310..0000000 --- a/reports/training/hard-field-validation.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "scope": "Additional independent field validation; candidate already frozen, final test already inspected. No model selection.", - "corpus": { - "schema": "bdon-native-field-corpus/1", - "field_count": 21200, - "field_counts": { - "0": 4240, - "101": 848, - "82": 120, - "87": 110, - "55": 134, - "102": 848, - "63": 123, - "14": 143, - "42": 140, - "103": 848, - "94": 122, - "61": 110, - "36": 136, - "104": 848, - "84": 120, - "28": 124, - "83": 137, - "105": 848, - "32": 121, - "74": 126, - "27": 134, - "99": 130, - "5": 116, - "85": 116, - "57": 136, - "67": 122, - "10": 143, - "60": 116, - "8": 158, - "44": 119, - "34": 127, - "73": 115, - "79": 123, - "65": 119, - "38": 159, - "100": 133, - "48": 144, - "30": 144, - "39": 117, - "91": 130, - "53": 125, - "9": 137, - "69": 120, - "98": 142, - "11": 122, - "19": 113, - "54": 144, - "76": 133, - "20": 112, - "77": 132, - "96": 124, - "15": 125, - "21": 123, - "41": 138, - "47": 121, - "49": 148, - "72": 117, - "89": 122, - "58": 127, - "17": 128, - "75": 126, - "18": 136, - "37": 149, - "66": 141, - "3": 136, - "80": 123, - "78": 129, - "29": 124, - "16": 102, - "92": 120, - "81": 126, - "46": 125, - "71": 139, - "88": 124, - "97": 125, - "45": 146, - "86": 120, - "43": 133, - "13": 128, - "4": 107, - "93": 130, - "24": 99, - "31": 129, - "40": 113, - "35": 123, - "25": 130, - "7": 128, - "70": 136, - "6": 131, - "52": 133, - "68": 155, - "33": 109, - "62": 119, - "23": 112, - "22": 139, - "50": 121, - "56": 125, - "2": 132, - "95": 118, - "90": 118, - "59": 113, - "26": 137, - "1": 135, - "51": 134, - "64": 101, - "12": 121 - }, - "modes": { - "technic": 940, - "training": 4240, - "level": 12720, - "performance": 979, - "hide": 988, - "total": 389, - "visual": 944 - }, - "locales": { - "en": 4189, - "ja": 4260, - "zh-Hans": 4267, - "ko": 4206, - "zh-Hant": 4278 - }, - "locale_names": [ - "ja", - "en", - "zh-Hant", - "zh-Hans", - "ko" - ], - "split": "validation", - "seed": 120031, - "acquisition": "screen", - "complete": true, - "label_encoding": { - "0": "no level/cultivation value", - "1..100": "level", - "101..105": "awake_count1..5" - }, - "font_manifest_sha256": "0bab78b1afef78ef8372fbf3d736541ca7150bb7ea90afa3af8d1e598da68a91", - "provenance": "native prefab text geometry; actual localized glyphs, material and master text; raster AA approximate" - }, - "models": { - "baseline": { - "sha256": "351db40ac89f611480d05131b7e626e92a928cfdd71278865dd9e87eb2d04b1c", - "samples": 21200, - "cross_entropy": 1.10949891720178, - "raw_correct": 18247, - "visible_truth": 16960, - "numeric_correct": 12036, - "numeric_wrong": 3, - "false_visible": 0, - "accepted_numeric": 12039, - "numeric_precision": 0.9997508098679292, - "numeric_coverage": 0.7096698113207547 - }, - "candidate": { - "sha256": "15351548a593c67cd939b72c3b03908e2665be5c39a0ca8148bd0cfc33feb0ce", - "samples": 21200, - "cross_entropy": 0.07150177991615152, - "raw_correct": 20770, - "visible_truth": 16960, - "numeric_correct": 15148, - "numeric_wrong": 4, - "false_visible": 0, - "accepted_numeric": 15152, - "numeric_precision": 0.9997360084477297, - "numeric_coverage": 0.8931603773584905 - } - } -} \ No newline at end of file diff --git a/reports/training/level-distribution.json b/reports/training/level-distribution.json deleted file mode 100644 index 5781bd2..0000000 --- a/reports/training/level-distribution.json +++ /dev/null @@ -1,199 +0,0 @@ -{ - "scene_visible_levels": { - "1": 643, - "2": 681, - "3": 596, - "4": 629, - "5": 674, - "6": 615, - "7": 655, - "8": 614, - "9": 685, - "10": 634, - "11": 645, - "12": 602, - "13": 583, - "14": 629, - "15": 558, - "16": 598, - "17": 606, - "18": 592, - "19": 621, - "20": 616, - "21": 619, - "22": 623, - "23": 637, - "24": 613, - "25": 675, - "26": 597, - "27": 586, - "28": 638, - "29": 657, - "30": 649, - "31": 543, - "32": 502, - "33": 527, - "34": 518, - "35": 513, - "36": 476, - "37": 539, - "38": 508, - "39": 526, - "40": 470, - "41": 365, - "42": 363, - "43": 396, - "44": 427, - "45": 400, - "46": 393, - "47": 405, - "48": 391, - "49": 395, - "50": 386, - "51": 266, - "52": 273, - "53": 293, - "54": 255, - "55": 303, - "56": 240, - "57": 273, - "58": 282, - "59": 273, - "60": 280, - "61": 156, - "62": 140, - "63": 156, - "64": 178, - "65": 149, - "66": 147, - "67": 149, - "68": 164, - "69": 157, - "70": 170, - "71": 68, - "72": 59, - "73": 57, - "74": 63, - "75": 65, - "76": 67, - "77": 75, - "78": 75, - "79": 65, - "80": 69, - "81": 15, - "82": 19, - "83": 15, - "84": 14, - "85": 15, - "86": 22, - "87": 17, - "88": 16, - "89": 16, - "90": 9 - }, - "native_field_training_levels": { - "20": 664, - "29": 593, - "99": 612, - "91": 650, - "87": 656, - "65": 654, - "36": 650, - "27": 576, - "68": 622, - "24": 631, - "5": 658, - "40": 671, - "76": 655, - "90": 676, - "86": 709, - "89": 666, - "3": 650, - "70": 629, - "45": 633, - "96": 677, - "62": 630, - "61": 590, - "41": 632, - "12": 582, - "74": 617, - "72": 657, - "30": 590, - "94": 605, - "66": 634, - "88": 611, - "71": 663, - "44": 617, - "18": 591, - "69": 678, - "54": 645, - "4": 612, - "59": 670, - "34": 618, - "56": 636, - "14": 593, - "75": 616, - "32": 644, - "64": 704, - "95": 631, - "11": 653, - "31": 627, - "25": 666, - "8": 644, - "1": 643, - "78": 664, - "79": 677, - "43": 630, - "9": 661, - "92": 671, - "33": 616, - "23": 606, - "73": 600, - "67": 653, - "10": 639, - "51": 674, - "7": 686, - "60": 611, - "55": 635, - "81": 659, - "48": 638, - "22": 616, - "53": 619, - "28": 614, - "97": 642, - "63": 614, - "26": 615, - "13": 663, - "77": 641, - "83": 682, - "47": 629, - "93": 612, - "2": 604, - "82": 642, - "84": 614, - "57": 639, - "16": 642, - "49": 607, - "39": 591, - "6": 641, - "38": 680, - "15": 621, - "19": 605, - "85": 622, - "42": 636, - "35": 680, - "58": 664, - "17": 622, - "50": 630, - "52": 631, - "100": 645, - "21": 635, - "46": 583, - "80": 611, - "37": 610, - "98": 647 - }, - "scene_total": 32938, - "native_field_total": 63600, - "scope": "Actual saved training truth / field manifest, not the rank-isolation matrix. Scene values obey per-card limits; isolated field pretraining covers numeric glyphs1..100." -} \ No newline at end of file diff --git a/reports/training/model-selection.json b/reports/training/model-selection.json new file mode 100644 index 0000000..e086c6c --- /dev/null +++ b/reports/training/model-selection.json @@ -0,0 +1,1035 @@ +{ + "schema": "ournotes-boxlens.model-selection/1", + "release": "0.2.0", + "rule": "Every checkpoint and threshold was chosen on development sets (profiles validation and stress, plus held-out crop banks); the final-profile sets were scored only after freezing.", + "models": { + "locator": { + "onnx_sha256": "f160f2a5022a9ff1295426014bd03a76815d42a0bba8826dda93d8dc54120ec6", + "pt_sha256": "67b2be9f43f301790381379358198b9384afb06a6670ea3c3090c22cb28f36ca", + "architecture": "MobileNetV3-Large features, stride-8 feature pyramid (width 96), per-class centre heatmap, tile size and centre offset heads", + "stages": [ + { + "initial": "torchvision MobileNetV3-Large ImageNet weights (backbone only)", + "train_sets": [ + "v2-train", + "synthetic.py train screens (16 sets x 40, seeds 300000-315000)" + ], + "validation_set": "v2-validation (600 evenly sampled screens)", + "args": { + "validation_limit": 600, + "steps": 15000, + "batch": 32, + "crop": 512, + "width": 96, + "lr": 0.002, + "backbone_lr": 0.5, + "weight_decay": 0.0001, + "warmup": 500, + "l1_weight": 0.5, + "giou_weight": 2.0, + "eval_every": 1000, + "eval_edge": 640, + "seed": 1, + "bf16": true + }, + "train_scenes": 16640, + "best": { + "step": 3000, + "ap50": 0.9999963717864261, + "ap75": 0.9999963717864261, + "at_0.3": { + "truth": 8187, + "iou0.5": { + "recall": 1.0, + "precision": 0.9995116591380784, + "false_positives": 4, + "false_positives_on_ui": 0, + "kind_accuracy": 1.0, + "recalled": 8187, + "kind_correct": 8187 + }, + "iou0.75": { + "recall": 1.0, + "precision": 0.9995116591380784, + "false_positives": 4, + "false_positives_on_ui": 0, + "kind_accuracy": 1.0, + "recalled": 8187, + "kind_correct": 8187 + } + }, + "score": 3.9990160618490087 + } + }, + { + "initial": "stage 1 best checkpoint", + "train_sets": [ + "v2.1-train", + "v2-train", + "synthetic.py train screens (16 sets x 40, seeds 300000-315000)" + ], + "validation_set": "v2.1-validation (600 evenly sampled screens)", + "args": { + "validation_limit": 600, + "steps": 4000, + "batch": 32, + "crop": 512, + "width": 96, + "lr": 0.0006, + "backbone_lr": 0.5, + "weight_decay": 0.0001, + "warmup": 100, + "l1_weight": 0.5, + "giou_weight": 2.0, + "eval_every": 500, + "eval_edge": 640, + "seed": 3, + "bf16": true + }, + "train_scenes": 32640, + "best": { + "step": 500, + "ap50": 0.9900965646083006, + "ap75": 0.9900965646083006, + "at_0.3": { + "truth": 8106, + "iou0.5": { + "recall": 0.9998766345916605, + "precision": 0.9995067209273647, + "false_positives": 4, + "false_positives_on_ui": 0, + "kind_accuracy": 1.0, + "recalled": 8105, + "kind_correct": 8105 + }, + "iou0.75": { + "recall": 0.9998766345916605, + "precision": 0.9995067209273647, + "false_positives": 4, + "false_positives_on_ui": 0, + "kind_accuracy": 1.0, + "recalled": 8105, + "kind_correct": 8105 + } + }, + "score": 3.9789599619618 + } + } + ], + "deployment": { + "long_edge": 640, + "threshold": 0.4 + } + }, + "member-encoder": { + "onnx_sha256": "e2110efcbef161a7f0783841d07be6f7da742ccd6a5406361e59ae728a837b9a", + "pt_sha256": "1cca598adf035eedccf6c15921fb99c2d8aaec84cf5a038e90002d2acf6bd0cb", + "architecture": "MobileNetV3-Small features, fixed 2x2 average pooling, linear projection to 128-d, L2 normalisation", + "initial": "shared encoder of release 0.1.0 (encoder.pt, sha256 8d67b87580188473d3408c4c1995f0e20b943d40ddcd2f8fc6a0f7a7fd26abda)", + "train_bank": "tile crops of v2.1-train, identities with id % 7 == 0 excluded from queries and references", + "config": { + "kind": "member", + "steps": 3000, + "batch": 256, + "lr": 0.00012, + "head_lr": 0.00012, + "weight_decay": 0.0001, + "warmup": 100, + "temperature": 0.08, + "margin": 0.0, + "unknown_level": 0.55, + "open_weight": 0.0, + "cos_weight": 0.15, + "same_name_margin": 0.0, + "kd_weight": 1.0, + "ref_mode": "frozen", + "ref_every": 20, + "cross_kind": true, + "own_fraction": 0.6, + "shift": 0.05, + "scale_jitter": 0.07, + "min_resolution": 0.35, + "overlay_prob": 0.5, + "erase_prob": 0.3, + "edge_prob": 0.1, + "min_inside": 0.999, + "eval_every": 250, + "seed": 52817, + "augmentation": { + "shift": 0.05, + "scale": 0.07, + "aspect": 0.03, + "color": 1.0, + "min_resolution": 0.35, + "resolution_prob": 0.5, + "overlay_prob": 0.5, + "erase_prob": 0.3, + "edge_prob": 0.1, + "noise": 0.02 + }, + "train_queries": { + "member": 27437, + "snap": 28688 + }, + "dropped_queries": { + "member": { + "held_out_identity": 0, + "foreign": 4605, + "window_partly_outside": 3566 + }, + "snap": { + "held_out_identity": 0, + "foreign": 4431, + "window_partly_outside": 1974 + } + }, + "train_identities": [ + 1, + 2, + 3, + 4, + 5, + 6, + 8, + 9, + 10, + 11, + 12, + 13, + 15, + 16, + 17, + 18, + 19, + 20, + 22, + 23, + 24, + 25, + 26, + 27, + 29, + 30, + 31, + 32, + 33, + 34, + 36, + 37, + 38, + 39, + 40, + 41, + 43, + 44, + 45, + 46, + 47, + 48, + 50, + 51, + 52, + 53, + 54, + 55, + 57, + 58, + 59, + 60, + 61, + 62, + 64 + ], + "held_out_identities": [ + 7, + 14, + 21, + 28, + 35, + 42, + 49, + 56, + 63 + ], + "train_identities_without_queries": [], + "train_fingerprints": { + "member": [ + "d105ede90410c4c38e30d72afd4531ba3392c4e403d8114223099204d755d25c" + ], + "snap": [ + "0831db4c72b2ff8a035ad272f3b73435abe57f100b8058071264bb5d127a8b7e" + ] + }, + "initial_sha256": "8d67b87580188473d3408c4c1995f0e20b943d40ddcd2f8fc6a0f7a7fd26abda", + "bank_on_gpu": true + }, + "selection": { + "rule": "max validation score = mean(seen, unseen) accepted-correct rate at thresholds with unseen leave-true-out and foreign false accepts <= 1% and accepted precision >= 99.5%; ties broken by stress score at the validation thresholds, then earlier step", + "step": 1750, + "validation_score": 0.9910454164857482, + "stress_score": 0.8461126419604923, + "thresholds": { + "similarity": 0.88, + "margin": 0.15, + "score": 0.9910454164857482, + "feasible": true, + "max_false_accept": 0.01, + "min_precision": 0.995, + "tied_similarity_range": [ + 0.88, + 0.88 + ] + } + }, + "export_checks": { + "torch_vs_onnx_max_abs_diff": 7.227063179016113e-07, + "torch_vs_onnx_random_max_abs_diff": 5.513429641723633e-07, + "batch_vs_single_max_abs_diff": 0.0, + "vector_norm_range": [ + 0.9999997615814209, + 1.000000238418579 + ], + "alternative_resampling_cosine_min": 0.9972051382064819, + "alternative_resampling_cosine_mean": 0.9997297525405884, + "alternative_resampling_self_retrieval": 64, + "cards": 64 + } + }, + "snap-encoder": { + "onnx_sha256": "ef3db30237d3d840727c298c7100b86c56dfec77a83aff6faa7712bfcee33b93", + "pt_sha256": "75b41723536e86a0044be429a40de83f7529bd4ce291aab6936b9367ee46d182", + "architecture": "MobileNetV3-Small features, fixed 2x2 average pooling, linear projection to 128-d, L2 normalisation", + "initial": "shared encoder of release 0.1.0 (encoder.pt, sha256 8d67b87580188473d3408c4c1995f0e20b943d40ddcd2f8fc6a0f7a7fd26abda)", + "train_bank": "tile crops of v2.1-train, identities with id % 7 == 0 excluded from queries and references", + "config": { + "kind": "snap", + "steps": 3000, + "batch": 256, + "lr": 0.00012, + "head_lr": 0.00012, + "weight_decay": 0.0001, + "warmup": 100, + "temperature": 0.08, + "margin": 0.0, + "unknown_level": 0.55, + "open_weight": 0.0, + "cos_weight": 0.15, + "same_name_margin": 0.0, + "kd_weight": 1.0, + "ref_mode": "frozen", + "ref_every": 20, + "cross_kind": true, + "own_fraction": 0.6, + "shift": 0.05, + "scale_jitter": 0.07, + "min_resolution": 0.35, + "overlay_prob": 0.5, + "erase_prob": 0.3, + "edge_prob": 0.1, + "min_inside": 0.999, + "eval_every": 250, + "seed": 52817, + "augmentation": { + "shift": 0.05, + "scale": 0.07, + "aspect": 0.03, + "color": 1.0, + "min_resolution": 0.35, + "resolution_prob": 0.5, + "overlay_prob": 0.5, + "erase_prob": 0.3, + "edge_prob": 0.1, + "noise": 0.02 + }, + "train_queries": { + "snap": 28688, + "member": 27437 + }, + "dropped_queries": { + "snap": { + "held_out_identity": 0, + "foreign": 4431, + "window_partly_outside": 1974 + }, + "member": { + "held_out_identity": 0, + "foreign": 4605, + "window_partly_outside": 3566 + } + }, + "train_identities": [ + 1, + 2, + 3, + 4, + 5, + 6, + 8, + 9, + 10, + 11, + 12, + 13, + 15, + 16, + 17, + 18, + 19, + 20, + 22, + 23, + 24, + 25, + 26, + 27, + 29, + 30, + 31, + 32, + 33, + 34, + 36, + 37, + 38, + 39, + 40, + 41, + 43, + 44, + 45, + 46, + 47, + 48, + 50, + 51, + 52, + 53, + 54, + 55, + 57, + 58, + 59, + 60, + 61, + 62, + 64 + ], + "held_out_identities": [ + 7, + 14, + 21, + 28, + 35, + 42, + 49, + 56, + 63, + 70 + ], + "train_identities_without_queries": [], + "train_fingerprints": { + "snap": [ + "0831db4c72b2ff8a035ad272f3b73435abe57f100b8058071264bb5d127a8b7e" + ], + "member": [ + "d105ede90410c4c38e30d72afd4531ba3392c4e403d8114223099204d755d25c" + ] + }, + "initial_sha256": "8d67b87580188473d3408c4c1995f0e20b943d40ddcd2f8fc6a0f7a7fd26abda", + "bank_on_gpu": true + }, + "selection": { + "rule": "max validation score = mean(seen, unseen) accepted-correct rate at thresholds with unseen leave-true-out and foreign false accepts <= 1% and accepted precision >= 99.5%; ties broken by stress score at the validation thresholds, then earlier step", + "step": 1000, + "validation_score": 0.9905531422536268, + "stress_score": 0.8422593021915821, + "thresholds": { + "similarity": 0.81, + "margin": 0.16, + "score": 0.9905531422536268, + "feasible": true, + "max_false_accept": 0.01, + "min_precision": 0.995, + "tied_similarity_range": [ + 0.81, + 0.82 + ] + } + }, + "export_checks": { + "torch_vs_onnx_max_abs_diff": 7.413327693939209e-07, + "torch_vs_onnx_random_max_abs_diff": 4.991888999938965e-07, + "batch_vs_single_max_abs_diff": 0.0, + "vector_norm_range": [ + 0.9999998211860657, + 1.0000001192092896 + ], + "alternative_resampling_cosine_min": 0.9975796341896057, + "alternative_resampling_cosine_mean": 0.9997220635414124, + "alternative_resampling_self_retrieval": 65, + "cards": 65 + } + }, + "member-fields": { + "onnx_sha256": "bc6ff2c7221d0ea197661dd6670fb12d3bd8d0e6811c00b8caa904a793b59264", + "pt_sha256": "d86eb13d221864f87de0d93c4d7898c5761d3968c2f2c040f2203388fa7da603", + "architecture": "four-block CNN (24/48/64/96 channels) with a linear head", + "train_banks": [ + "parameter-field crops of v2.1-train", + "hard-g acquisition-degraded crops (member)" + ], + "dev_sets": [ + "v2.1-validation crops", + "v2.1-stress crops", + "hard-l acquisition-degraded validation crops (member)" + ], + "initial": "stage 1 selected checkpoint (see stage1)", + "config": { + "kind": "member", + "steps": 3000, + "batch": 512, + "eval_every": 250, + "lr": 0.0001, + "hard_fraction": 0.4, + "freeze_bn": false, + "jitter": { + "shift_px": [ + 4.0, + 7.0 + ], + "scale": 0.035, + "stored_margin_px": 8, + "reach_px": [ + 6.503, + 7.963 + ] + }, + "train_counts": { + "scenes": 98353, + "hard": 60000 + }, + "dev_counts": { + "validation": 12806, + "stress": 6033, + "hard": 16000 + }, + "acceptance": { + "confidence": 0.995, + "margin": 0.5 + }, + "schedule": "linear warmup 5% then cosine decay", + "selection_rule": "per checkpoint: lowest confidence threshold (not below the default, margin fixed) with no misread on any development crop; choose the highest recall of normal and stress scene targets at that threshold, then the highest recall on degraded crops; checkpoints where no threshold removes every misread rank after all others, by fewest misreads at the default threshold, then recall" + }, + "selected_step": 2500, + "selected": { + "step": 2500, + "train_loss": 0.018748672679066658, + "sets": { + "validation": { + "visible_total": 3548, + "hidden_total": 9258, + "correct": 3534, + "wrong": 0, + "unknown": 14, + "false_visible": 3, + "false_visible_latent_match": 3, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.996054114994363 + }, + "stress": { + "visible_total": 1824, + "hidden_total": 4209, + "correct": 1369, + "wrong": 1, + "unknown": 454, + "false_visible": 1, + "false_visible_latent_match": 1, + "false_visible_latent_mismatch": 0, + "errors": 1, + "coverage": 0.7505482456140351 + }, + "hard": { + "visible_total": 9091, + "hidden_total": 6909, + "correct": 8215, + "wrong": 0, + "unknown": 876, + "false_visible": 174, + "false_visible_latent_match": 172, + "false_visible_latent_mismatch": 2, + "errors": 2, + "coverage": 0.903640963590364 + } + }, + "zero_misread": { + "threshold": 0.9993960857391359, + "achievable": true, + "highest_misread_confidence": 0.9993960857391357, + "misreads_at_default": 3, + "recall": { + "scenes": 0.8790022338049144, + "hard": 0.8329116708832912 + }, + "recall_at_default": { + "scenes": 0.9126954579300074, + "hard": 0.903640963590364 + }, + "per_set": { + "validation": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.996054114994363, + "misreads_at_default": 0 + }, + "stress": { + "zero_misread_threshold": 0.9987905621528627, + "achievable": true, + "recall_at_zero_misread_threshold": 0.7138157894736842, + "misreads_at_default": 1 + }, + "hard": { + "zero_misread_threshold": 0.9993960857391359, + "achievable": true, + "recall_at_zero_misread_threshold": 0.8329116708832912, + "misreads_at_default": 2 + } + } + } + }, + "deployment_accept": { + "confidence": 0.9994, + "margin": 0.5 + }, + "stage1": { + "initial": "shared field model of release 0.1.0 (fields.pt, sha256 15351548a593c67cd939b72c3b03908e2665be5c39a0ca8148bd0cfc33feb0ce)", + "config": { + "kind": "member", + "steps": 8000, + "batch": 512, + "eval_every": 500, + "lr": 0.0003, + "hard_fraction": 0.4, + "freeze_bn": false, + "train_counts": { + "scenes": 98353, + "hard": 60000 + }, + "dev_counts": { + "validation": 12806, + "stress": 6033, + "hard": 16000 + }, + "acceptance": { + "confidence": 0.995, + "margin": 0.5 + }, + "schedule": "linear warmup 5% then cosine decay", + "selection_rule": "per checkpoint: lowest confidence threshold (not below the default, margin fixed) with no misread on any development crop; choose the highest recall of normal and stress scene targets at that threshold, then the highest recall on degraded crops; checkpoints where no threshold removes every misread rank after all others, by fewest misreads at the default threshold, then recall" + }, + "selected_step": 6500 + } + }, + "snap-fields": { + "onnx_sha256": "fa35b8a80688e3c1f92b3dbd6c567364b842a3a8f25387fac0879b7f740591b4", + "pt_sha256": "3917584b999cfe3ea9d359c6da3af38ee06c3631fc68b3a56b6d708ec3f6891b", + "architecture": "four-block CNN (24/48/64/96 channels) with a linear head", + "train_banks": [ + "parameter-field crops of v2.1-train", + "hard-g acquisition-degraded crops (snap)" + ], + "dev_sets": [ + "v2.1-validation crops", + "v2.1-stress crops", + "hard-l acquisition-degraded validation crops (snap)" + ], + "initial": "shared field model of release 0.1.0 (fields.pt, sha256 15351548a593c67cd939b72c3b03908e2665be5c39a0ca8148bd0cfc33feb0ce)", + "config": { + "kind": "snap", + "steps": 8000, + "batch": 512, + "eval_every": 500, + "lr": 0.0003, + "hard_fraction": 0.4, + "freeze_bn": false, + "train_counts": { + "scenes": 99860, + "hard": 60000 + }, + "dev_counts": { + "validation": 12991, + "stress": 6392, + "hard": 16000 + }, + "acceptance": { + "confidence": 0.995, + "margin": 0.5 + }, + "schedule": "linear warmup 5% then cosine decay", + "selection_rule": "per checkpoint: lowest confidence threshold (not below the default, margin fixed) with no misread on any development crop; choose the highest recall of normal and stress scene targets at that threshold, then the highest recall on degraded crops; checkpoints where no threshold removes every misread rank after all others, by fewest misreads at the default threshold, then recall" + }, + "selected_step": 4000, + "selected": { + "step": 4000, + "train_loss": 0.018172895535826683, + "sets": { + "validation": { + "visible_total": 2797, + "hidden_total": 10194, + "correct": 2701, + "wrong": 0, + "unknown": 96, + "false_visible": 3, + "false_visible_latent_match": 3, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.9656775116195924 + }, + "stress": { + "visible_total": 1226, + "hidden_total": 5166, + "correct": 871, + "wrong": 0, + "unknown": 355, + "false_visible": 2, + "false_visible_latent_match": 2, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.7104404567699837 + }, + "hard": { + "visible_total": 9310, + "hidden_total": 6690, + "correct": 8341, + "wrong": 0, + "unknown": 969, + "false_visible": 20, + "false_visible_latent_match": 20, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.8959183673469387 + } + }, + "zero_misread": { + "threshold": 0.995, + "achievable": true, + "highest_misread_confidence": 0.9916692972183228, + "misreads_at_default": 0, + "recall": { + "scenes": 0.8878946060154114, + "hard": 0.8959183673469387 + }, + "recall_at_default": { + "scenes": 0.8878946060154114, + "hard": 0.8959183673469387 + }, + "per_set": { + "validation": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.9656775116195924, + "misreads_at_default": 0 + }, + "stress": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.7104404567699837, + "misreads_at_default": 0 + }, + "hard": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.8959183673469387, + "misreads_at_default": 0 + } + } + } + }, + "deployment_accept": { + "confidence": 0.995, + "margin": 0.5 + } + }, + "member-rank": { + "onnx_sha256": "e8e8daa2a28def53b7d1baca281980db118fc9a3364cfde02f4bd8b3f8af68fc", + "pt_sha256": "d918e88fac08ced7c2687b2cae654261e96ec1e51ea7e5b5a74d76b636044e01", + "architecture": "five-block CNN (32/32/64/96/128 channels) with a linear head", + "train_banks": [ + "card-rank icon crops of v2.1-train", + "hard-n acquisition-degraded crops (member)" + ], + "dev_sets": [ + "v2.1-validation crops", + "v2.1-stress crops", + "hard-l acquisition-degraded validation crops (member)" + ], + "initial": "random initialisation", + "config": { + "kind": "member", + "steps": 6000, + "batch": 256, + "eval_every": 500, + "lr": 0.002, + "hard_fraction": 0.4, + "train_counts": { + "scenes": 109594, + "hard": 60000 + }, + "dev_counts": { + "validation": 13954, + "stress": 6673, + "hard": 16000 + }, + "input": { + "side": 48, + "logical_side": 96 + }, + "acceptance": { + "confidence": 0.995, + "margin": 0.5 + }, + "selection_rule": "per checkpoint: lowest confidence threshold (not below the default, margin fixed) with no misread on any development crop; choose the highest recall of normal and stress scene targets at that threshold, then the highest recall on degraded crops; checkpoints where no threshold removes every misread rank after all others, by fewest misreads at the default threshold, then recall; icon-inside crops only" + }, + "selected_step": 3500, + "selected": { + "step": 3500, + "train_loss": 1.9801658709184267e-05, + "sets": { + "validation": { + "visible_total": 10814, + "hidden_total": 1876, + "correct": 10794, + "wrong": 0, + "unknown": 20, + "false_visible": 3, + "false_visible_latent_match": 3, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.9981505455890513, + "outside_emitted": 0 + }, + "stress": { + "visible_total": 5197, + "hidden_total": 808, + "correct": 4988, + "wrong": 0, + "unknown": 209, + "false_visible": 15, + "false_visible_latent_match": 15, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.9597844910525303, + "outside_emitted": 5 + }, + "hard": { + "visible_total": 13184, + "hidden_total": 2816, + "correct": 13175, + "wrong": 0, + "unknown": 9, + "false_visible": 327, + "false_visible_latent_match": 317, + "false_visible_latent_mismatch": 10, + "errors": 10, + "coverage": 0.999317354368932, + "outside_emitted": 0 + } + }, + "zero_misread": { + "threshold": 0.9997752308845521, + "achievable": true, + "highest_misread_confidence": 0.999775230884552, + "misreads_at_default": 10, + "recall": { + "scenes": 0.9732059209293611, + "hard": 0.9941595873786407 + }, + "recall_at_default": { + "scenes": 0.985697333083505, + "hard": 0.999317354368932 + }, + "per_set": { + "validation": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.9981505455890513, + "misreads_at_default": 0 + }, + "stress": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.9597844910525303, + "misreads_at_default": 0 + }, + "hard": { + "zero_misread_threshold": 0.9997752308845521, + "achievable": true, + "recall_at_zero_misread_threshold": 0.9941595873786407, + "misreads_at_default": 10 + } + } + } + }, + "deployment_accept": { + "confidence": 0.9998, + "margin": 0.5 + } + }, + "snap-rank": { + "onnx_sha256": "38c5eb44cecc3380bcc2adf3c0f59049e904b5618593950b27c2d552412883f1", + "pt_sha256": "6fc054aae5f03f43f51ee4b64692f9a13a324a7f60f536c2bbcdb4c43245a363", + "architecture": "five-block CNN (32/32/64/96/128 channels) with a linear head", + "train_banks": [ + "card-rank icon crops of v2.1-train", + "hard-g acquisition-degraded crops (snap)" + ], + "dev_sets": [ + "v2.1-validation crops", + "v2.1-stress crops", + "hard-l acquisition-degraded validation crops (snap)" + ], + "initial": "random initialisation", + "config": { + "kind": "snap", + "steps": 6000, + "batch": 256, + "eval_every": 500, + "lr": 0.002, + "hard_fraction": 0.4, + "train_counts": { + "scenes": 105912, + "hard": 60000 + }, + "dev_counts": { + "validation": 13719, + "stress": 6814, + "hard": 16000 + }, + "input": { + "side": 48, + "logical_side": 96 + }, + "acceptance": { + "confidence": 0.995, + "margin": 0.5 + }, + "selection_rule": "per checkpoint: lowest confidence threshold (not below the default, margin fixed) with no misread on any development crop; choose the highest recall of normal and stress scene targets at that threshold, then the highest recall on degraded crops; checkpoints where no threshold removes every misread rank after all others, by fewest misreads at the default threshold, then recall; icon-inside crops only" + }, + "selected_step": 6000, + "selected": { + "step": 6000, + "train_loss": 0.004346613772213459, + "sets": { + "validation": { + "visible_total": 10585, + "hidden_total": 2384, + "correct": 10543, + "wrong": 0, + "unknown": 42, + "false_visible": 1, + "false_visible_latent_match": 1, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.9960321209258385, + "outside_emitted": 3 + }, + "stress": { + "visible_total": 5230, + "hidden_total": 1145, + "correct": 4755, + "wrong": 0, + "unknown": 475, + "false_visible": 0, + "false_visible_latent_match": 0, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.9091778202676865, + "outside_emitted": 1 + }, + "hard": { + "visible_total": 13259, + "hidden_total": 2741, + "correct": 13202, + "wrong": 0, + "unknown": 57, + "false_visible": 2, + "false_visible_latent_match": 2, + "false_visible_latent_mismatch": 0, + "errors": 0, + "coverage": 0.9957010332604269, + "outside_emitted": 0 + } + }, + "zero_misread": { + "threshold": 0.995, + "achievable": true, + "highest_misread_confidence": null, + "misreads_at_default": 0, + "recall": { + "scenes": 0.9673095162820108, + "hard": 0.9957010332604269 + }, + "recall_at_default": { + "scenes": 0.9673095162820108, + "hard": 0.9957010332604269 + }, + "per_set": { + "validation": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.9960321209258385, + "misreads_at_default": 0 + }, + "stress": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.9091778202676865, + "misreads_at_default": 0 + }, + "hard": { + "zero_misread_threshold": 0.995, + "achievable": true, + "recall_at_zero_misread_threshold": 0.9957010332604269, + "misreads_at_default": 0 + } + } + } + }, + "deployment_accept": { + "confidence": 0.995, + "margin": 0.5 + } + } + }, + "crop_banks": { + "hard-g": { + "module": "bdon_vision.kind_hardcrops", + "count": 60000, + "seeds": { + "member": 190001, + "snap": 190003 + }, + "split": "train", + "occlusion_rate": 0.08 + }, + "hard-n": { + "module": "bdon_vision.kind_hardcrops", + "count": 60000, + "seeds": { + "member": 180001, + "snap": 180003 + }, + "split": "train", + "occlusion_rate": 0 + }, + "hard-l": { + "module": "bdon_vision.kind_hardcrops", + "count": 16000, + "seeds": { + "member": 170001, + "snap": 170003 + }, + "split": "validation" + } + } +} diff --git a/reports/training/rank-distribution.json b/reports/training/rank-distribution.json deleted file mode 100644 index 0ec1a07..0000000 --- a/reports/training/rank-distribution.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "groups": { - "scale-v6/train/truth.json": { - "1": 6332, - "2": 6265, - "3": 6455, - "4": 6461, - "5": 6476 - }, - "modes-v7/train/truth.json": { - "1": 876, - "2": 824, - "3": 916, - "4": 896, - "5": 896 - } - }, - "overall": { - "1": 7208, - "2": 7089, - "3": 7371, - "4": 7357, - "5": 7372 - }, - "scope": "Card-rank labels present in generated training scenes. Rank inference itself uses original sprite matching, not a separately trained classifier. Historical clipped-icon composition caveat applies." -} \ No newline at end of file diff --git a/reports/training/scene-coverage.json b/reports/training/scene-coverage.json new file mode 100644 index 0000000..d0ac00f --- /dev/null +++ b/reports/training/scene-coverage.json @@ -0,0 +1,604 @@ +{ + "schema": "ournotes-boxlens.scene-coverage/1", + "scope": "Counts from saved truth files: presence of identities, views and field values, not recognition accuracy.", + "catalog": { + "cards": 129, + "by_kind_rarity": { + "member/BD": 1, + "member/R": 25, + "member/SR": 26, + "member/SSR": 12, + "snap/EX": 2, + "snap/R": 25, + "snap/SR": 26, + "snap/SSR": 12 + }, + "catalog_sha256": "c6995ccac52fa98438eeca3cd82e375e6e417e64b5b0f3e91ecacfb5fee8647e" + }, + "sets": { + "v2-train": { + "generator": "scenes-v2", + "profile": "train", + "seed": 20261101, + "truth_sha256": "6ac1a0a291daa836049f478c94abbf0790f3e95382d3577e551d2a0e76be50bd", + "screens": 16000, + "negative_screens": 802, + "cards": 187844, + "member": 96181, + "partly_outside": 25912, + "occluded": 8501, + "foreign_cards": 26722, + "snap": 91663, + "unique_identities": 110, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 22, + "member/SR": 21, + "member/SSR": 11, + "snap/EX": 1, + "snap/R": 22, + "snap/SR": 22, + "snap/SSR": 10 + }, + "heldout_identities_present": 0, + "locales": { + "en": 3178, + "zh-Hans": 3206, + "ko": 3194, + "ja": 3216, + "zh-Hant": 3206 + }, + "display_modes": { + "visual": 32252, + "technic": 32027, + "performance": 31976, + "level": 32253, + "hide": 32755, + "total": 13665, + "training": 12916 + }, + "source_resolutions": { + "1600x900": 3173, + "1280x720": 3197, + "2160x1080": 3206, + "1440x1080": 3127, + "1920x1080": 3297 + }, + "visible_level_bins": { + "1": 557, + "2-20": 10722, + "21-40": 10340, + "41-60": 6118, + "61-80": 1977, + "81-100": 150 + }, + "visible_levels_distinct": 90, + "visible_level_max": 90, + "visible_rank": { + "1": 28518, + "2": 28421, + "3": 28466, + "4": 28240, + "5": 28209 + }, + "visible_training_count": { + "1": 2310, + "2": 2381, + "3": 2298, + "4": 2278, + "5": 2264 + } + }, + "v2.1-train": { + "generator": "scenes-v2.1", + "profile": "train", + "seed": 20261111, + "truth_sha256": "73ad36ee216e8852ad8dbf89aedb24fe4fa2f13d4ea845098420ea6be450c079", + "screens": 16000, + "negative_screens": 808, + "cards": 188200, + "member": 95728, + "partly_outside": 26685, + "occluded": 8926, + "foreign_cards": 27306, + "snap": 92472, + "unique_identities": 110, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 22, + "member/SR": 21, + "member/SSR": 11, + "snap/EX": 1, + "snap/R": 22, + "snap/SR": 22, + "snap/SSR": 10 + }, + "heldout_identities_present": 0, + "locales": { + "en": 3239, + "zh-Hans": 3176, + "ko": 3130, + "ja": 3211, + "zh-Hant": 3244 + }, + "display_modes": { + "level": 31686, + "hide": 32149, + "performance": 32346, + "visual": 32194, + "technic": 31953, + "training": 13391, + "total": 14481 + }, + "source_resolutions": { + "1600x900": 3158, + "2160x1080": 3141, + "1280x720": 3261, + "1920x1080": 3245, + "1440x1080": 3195 + }, + "visible_level_bins": { + "1": 504, + "2-20": 10389, + "21-40": 10253, + "41-60": 5905, + "61-80": 1986, + "81-100": 139 + }, + "visible_levels_distinct": 90, + "visible_level_max": 90, + "visible_rank": { + "1": 28421, + "2": 28352, + "3": 28798, + "4": 28396, + "5": 28443 + }, + "visible_training_count": { + "1": 2368, + "2": 2395, + "3": 2328, + "4": 2405, + "5": 2403 + } + }, + "v2-validation": { + "generator": "scenes-v2", + "profile": "validation", + "seed": 20261102, + "truth_sha256": "2f8a72da4d3dbc001e4d9364cf9243035fb6f99befde2b7e8d8be4abbe7269ac", + "screens": 2000, + "negative_screens": 98, + "cards": 24326, + "member": 12281, + "partly_outside": 3129, + "occluded": 1119, + "foreign_cards": 3419, + "snap": 12045, + "unique_identities": 129, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 25, + "member/SR": 26, + "member/SSR": 12, + "snap/EX": 2, + "snap/R": 25, + "snap/SR": 26, + "snap/SSR": 12 + }, + "heldout_identities_present": 19, + "locales": { + "ja": 403, + "zh-Hans": 394, + "en": 396, + "ko": 384, + "zh-Hant": 423 + }, + "display_modes": { + "training": 1947, + "technic": 3880, + "level": 4214, + "hide": 4276, + "visual": 3978, + "total": 1656, + "performance": 4375 + }, + "source_resolutions": { + "1920x1200": 374, + "1366x768": 401, + "2340x1080": 409, + "2560x1440": 397, + "2048x1536": 419 + }, + "visible_level_bins": { + "1": 75, + "2-20": 1453, + "21-40": 1346, + "41-60": 786, + "61-80": 273, + "81-100": 21 + }, + "visible_levels_distinct": 88, + "visible_level_max": 90, + "visible_rank": { + "1": 3770, + "2": 3661, + "3": 3774, + "4": 3680, + "5": 3683 + }, + "visible_training_count": { + "1": 356, + "2": 360, + "3": 345, + "4": 363, + "5": 350 + } + }, + "v2.1-validation": { + "generator": "scenes-v2.1", + "profile": "validation", + "seed": 20261112, + "truth_sha256": "9ff9dd478f3b239874e48e9f8e9b0d13a54314eef121dd657009db24abe345fb", + "screens": 2000, + "negative_screens": 106, + "cards": 24031, + "member": 12065, + "partly_outside": 3169, + "occluded": 1196, + "foreign_cards": 3642, + "snap": 11966, + "unique_identities": 129, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 25, + "member/SR": 26, + "member/SSR": 12, + "snap/EX": 2, + "snap/R": 25, + "snap/SR": 26, + "snap/SSR": 12 + }, + "heldout_identities_present": 19, + "locales": { + "en": 419, + "ko": 385, + "zh-Hans": 418, + "ja": 404, + "zh-Hant": 374 + }, + "display_modes": { + "level": 4392, + "hide": 4006, + "technic": 3981, + "visual": 4571, + "performance": 3926, + "training": 1450, + "total": 1705 + }, + "source_resolutions": { + "2048x1536": 399, + "1366x768": 377, + "2340x1080": 420, + "2560x1440": 397, + "1920x1200": 407 + }, + "visible_level_bins": { + "1": 80, + "2-20": 1473, + "21-40": 1450, + "41-60": 786, + "61-80": 283, + "81-100": 26 + }, + "visible_levels_distinct": 88, + "visible_level_max": 90, + "visible_rank": { + "1": 3818, + "2": 3707, + "3": 3665, + "4": 3682, + "5": 3663 + }, + "visible_training_count": { + "1": 260, + "2": 263, + "3": 264, + "4": 257, + "5": 299 + } + }, + "v2-stress": { + "generator": "scenes-v2", + "profile": "stress", + "seed": 20261103, + "truth_sha256": "ed8516c4cd3d6b603d4e5bff5480f7f8e17f3e1e2eebd8205e64e08e770745e0", + "screens": 1000, + "negative_screens": 58, + "cards": 11239, + "member": 5536, + "partly_outside": 1591, + "occluded": 467, + "foreign_cards": 1663, + "snap": 5703, + "unique_identities": 129, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 25, + "member/SR": 26, + "member/SSR": 12, + "snap/EX": 2, + "snap/R": 25, + "snap/SR": 26, + "snap/SSR": 12 + }, + "heldout_identities_present": 19, + "locales": { + "en": 188, + "zh-Hant": 194, + "ja": 204, + "ko": 196, + "zh-Hans": 218 + }, + "display_modes": { + "total": 708, + "performance": 2172, + "training": 574, + "visual": 1747, + "technic": 1976, + "level": 2008, + "hide": 2054 + }, + "source_resolutions": { + "2800x1752": 190, + "2280x1080": 181, + "3840x2160": 215, + "2532x1170": 204, + "960x540": 210 + }, + "visible_level_bins": { + "1": 34, + "2-20": 670, + "21-40": 639, + "41-60": 378, + "61-80": 121, + "81-100": 4 + }, + "visible_levels_distinct": 83, + "visible_level_max": 90, + "visible_rank": { + "1": 1672, + "2": 1662, + "3": 1707, + "4": 1623, + "5": 1709 + }, + "visible_training_count": { + "1": 94, + "2": 111, + "3": 93, + "4": 111, + "5": 92 + } + }, + "v2.1-stress": { + "generator": "scenes-v2.1", + "profile": "stress", + "seed": 20261113, + "truth_sha256": "7335344a89024d1dbf0b6b0bfe197c64f1523303dd2812087810a47146a7da0c", + "screens": 1000, + "negative_screens": 49, + "cards": 11831, + "member": 5810, + "partly_outside": 1703, + "occluded": 522, + "foreign_cards": 1656, + "snap": 6021, + "unique_identities": 129, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 25, + "member/SR": 26, + "member/SSR": 12, + "snap/EX": 2, + "snap/R": 25, + "snap/SR": 26, + "snap/SSR": 12 + }, + "heldout_identities_present": 19, + "locales": { + "ja": 198, + "ko": 215, + "zh-Hant": 206, + "en": 190, + "zh-Hans": 191 + }, + "display_modes": { + "hide": 1784, + "performance": 1996, + "technic": 2077, + "level": 2156, + "visual": 2274, + "training": 769, + "total": 775 + }, + "source_resolutions": { + "2280x1080": 215, + "960x540": 205, + "2800x1752": 190, + "3840x2160": 203, + "2532x1170": 187 + }, + "visible_level_bins": { + "1": 40, + "2-20": 727, + "21-40": 690, + "41-60": 399, + "61-80": 128, + "81-100": 6 + }, + "visible_levels_distinct": 84, + "visible_level_max": 86, + "visible_rank": { + "1": 1907, + "2": 1809, + "3": 1839, + "4": 1806, + "5": 1835 + }, + "visible_training_count": { + "1": 145, + "2": 143, + "3": 141, + "4": 138, + "5": 139 + } + }, + "v2-final": { + "generator": "scenes-v2", + "profile": "final", + "seed": 20261104, + "truth_sha256": "6f78126d3708361bb4e79f458b89da8cff7b0f6cbfb07e51bdfd975353365ab3", + "screens": 2000, + "negative_screens": 106, + "cards": 24405, + "member": 12247, + "partly_outside": 3019, + "occluded": 946, + "foreign_cards": 3493, + "snap": 12158, + "unique_identities": 129, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 25, + "member/SR": 26, + "member/SSR": 12, + "snap/EX": 2, + "snap/R": 25, + "snap/SR": 26, + "snap/SSR": 12 + }, + "heldout_identities_present": 19, + "locales": { + "en": 413, + "ja": 387, + "zh-Hans": 420, + "ko": 388, + "zh-Hant": 392 + }, + "display_modes": { + "total": 1853, + "level": 4428, + "technic": 4271, + "hide": 3971, + "visual": 4003, + "performance": 3941, + "training": 1938 + }, + "source_resolutions": { + "1792x828": 403, + "2224x1668": 438, + "2388x1668": 374, + "2400x1080": 406, + "1334x750": 379 + }, + "visible_level_bins": { + "1": 96, + "2-20": 1443, + "21-40": 1436, + "41-60": 831, + "61-80": 275, + "81-100": 23 + }, + "visible_levels_distinct": 88, + "visible_level_max": 90, + "visible_rank": { + "1": 3827, + "2": 3745, + "3": 3788, + "4": 3764, + "5": 3776 + }, + "visible_training_count": { + "1": 378, + "2": 333, + "3": 366, + "4": 344, + "5": 368 + } + }, + "v2.1-final": { + "generator": "scenes-v2.1", + "profile": "final", + "seed": 20261114, + "truth_sha256": "83dda3e94d5ce63e6df93e957f5e405a444ee9b9ce74aa377a4465d9a48ddfbc", + "screens": 2000, + "negative_screens": 99, + "cards": 24604, + "member": 12594, + "partly_outside": 3165, + "occluded": 952, + "foreign_cards": 3725, + "snap": 12010, + "unique_identities": 129, + "unique_by_kind_rarity": { + "member/BD": 1, + "member/R": 25, + "member/SR": 26, + "member/SSR": 12, + "snap/EX": 2, + "snap/R": 25, + "snap/SR": 26, + "snap/SSR": 12 + }, + "heldout_identities_present": 19, + "locales": { + "ko": 375, + "zh-Hans": 417, + "ja": 396, + "en": 411, + "zh-Hant": 401 + }, + "display_modes": { + "visual": 4023, + "hide": 4103, + "total": 1620, + "level": 4578, + "performance": 3986, + "technic": 4608, + "training": 1686 + }, + "source_resolutions": { + "2224x1668": 377, + "2400x1080": 404, + "1792x828": 430, + "2388x1668": 409, + "1334x750": 380 + }, + "visible_level_bins": { + "1": 80, + "2-20": 1490, + "21-40": 1445, + "41-60": 843, + "61-80": 309, + "81-100": 23 + }, + "visible_levels_distinct": 88, + "visible_level_max": 89, + "visible_rank": { + "1": 3870, + "2": 3723, + "3": 3797, + "4": 3709, + "5": 3739 + }, + "visible_training_count": { + "1": 285, + "2": 312, + "3": 320, + "4": 344, + "5": 303 + } + } + } +} From 239bb4aa32336948536b7c68c016cda11772c222 Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:45:56 +0800 Subject: [PATCH 7/7] chore: release 0.2.0 Bump the package version and regenerate public-manifest.json for the release source tree. --- public-manifest.json | 408 ++++++++++++++++++++++++++----------------- pyproject.toml | 2 +- 2 files changed, 249 insertions(+), 161 deletions(-) diff --git a/public-manifest.json b/public-manifest.json index f5c3158..bf90e27 100644 --- a/public-manifest.json +++ b/public-manifest.json @@ -1,6 +1,6 @@ { "schema": "ournotes-boxlens.public-source/1", - "version": "v0.1.0", + "version": "v0.2.0", "game_assets_included": false, "files": { ".gitattributes": { @@ -8,20 +8,40 @@ "sha256": "aebc1e10d451ab7acbabbcfad3f2ed01ffc5bf09f4b4b129ad270bd48c8ddfcf" }, ".github/workflows/ci.yml": { - "bytes": 653, - "sha256": "5936cdce7a0d29f92c7b54c901b71770beab2394d7ff39ead2785472c1a8b8b2" + "bytes": 627, + "sha256": "54b552190d87d61eb4edf79f18d2fa7ec4746267aa2c12e416f722f2413e9644" }, ".gitignore": { "bytes": 251, "sha256": "fc0b56ba097d5a9f0ea1b374ddf561c5c64d3c1acfc143b3c0d037c86963e29c" }, + "LICENSE": { + "bytes": 1080, + "sha256": "13ee0a55b4b50909b685464b38013fa32814c5c1a2dcf1e3c6f402bad0cb2d8f" + }, + "MANIFEST.in": { + "bytes": 30, + "sha256": "9b7a072d77a84c7211e800362aedcf609fcc895a246442cc4583370bd52bd7b2" + }, + "README.md": { + "bytes": 7363, + "sha256": "aad4ae936899341009d99fb74e98446cbf8b63c25e2ec472f6893eee9ad098fc" + }, + "THIRD_PARTY_NOTICES.md": { + "bytes": 1304, + "sha256": "6ad4199eed247ab69b31ec70896c05875ef74222229063717268f600c1023ece" + }, "bdon_vision/__init__.py": { - "bytes": 56, - "sha256": "51810dd8dad6636cef2eda8790ed4c2ee62c4ea75f61abfc31b4f565967e0b91" + "bytes": 81, + "sha256": "c08f90e2f4b82704c785e94274d831c2733342ed60309902997bc9eeedf60eab" + }, + "bdon_vision/__main__.py": { + "bytes": 61, + "sha256": "3129adff95e0f38b87ab34cddfc27082c7892bcaec267ff5d40f160fdf55b44a" }, "bdon_vision/assets.py": { - "bytes": 10606, - "sha256": "dc2e63e50134cb5b3f4b0a83d5ceaae0d4ce2f6666329f9d349458afce7d803f" + "bytes": 9721, + "sha256": "b01925ec5f1877e98124e675cdf66598367c488f08cdaca7aaff41b868572360" }, "bdon_vision/audit_datasets.py": { "bytes": 4620, @@ -32,56 +52,148 @@ "sha256": "caa94798f3848f74956f6fdb51d39e3dff60e2e76df19818a5436152f85f7096" }, "bdon_vision/benchmark.py": { - "bytes": 5344, - "sha256": "05ee2b8e8cb3597fbdc5b50264db9a44a7f47dd19779892dcd76a04dea3ddd35" + "bytes": 5396, + "sha256": "197f175c8c38ae17fafa725f7b759c33acfcea1bf62067b51dc9567dca5ce053" }, "bdon_vision/cli.py": { - "bytes": 2620, - "sha256": "418346ef78258f5d35451e71fd2d9e3e2d5da21ca9513546d26bc469ea8e8407" + "bytes": 2760, + "sha256": "6c90da780ec26eb43e419a36a62188889397818dda803740af7cc2cfc5dcf3ab" }, "bdon_vision/corpus.py": { "bytes": 4393, "sha256": "9fcccc00bccb7e98f4929f4fedaac54baa4d54c9116ef47db7a5b1efb58b6ab9" }, - "bdon_vision/diagnose_fields.py": { - "bytes": 4054, - "sha256": "42e9f43ceaf5647e4b0df8665bdeb55f7ce61e06b407cae1fc37135afa67ac57" + "bdon_vision/deck/__init__.py": { + "bytes": 1, + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + }, + "bdon_vision/deck/adapter.py": { + "bytes": 14866, + "sha256": "61896115de11849531c8f66acafd536b9807e0126371c961c54cdafce492c3e8" + }, + "bdon_vision/deck/cli.py": { + "bytes": 3914, + "sha256": "09c03f6ed6ba33f7db00127e111cd54987d154b056da0f3f20a7ba58af2cb9c3" + }, + "bdon_vision/deck/common.py": { + "bytes": 2515, + "sha256": "6daf9d685f5c1043c86c00fde1de4fc8a7b8eac368bf1345172e0ddbb77917c5" + }, + "bdon_vision/deck/mock.py": { + "bytes": 9053, + "sha256": "dbf6cabdca328151961aa8678b151fb514a6e069d2143eea001ad086edf42d35" + }, + "bdon_vision/deck/recommend.py": { + "bytes": 14072, + "sha256": "16c69512d1315d8c380f0ce3bf23495100ea13a9b7db08ad3e5c4d910391c8f0" + }, + "bdon_vision/deck/review_state.py": { + "bytes": 2079, + "sha256": "4bd900513192bcb5f1f2335f04648fce8f4c8e7878d6e6279defc11ee62733a8" + }, + "bdon_vision/deck/server.py": { + "bytes": 10993, + "sha256": "e7a626350cef4a3c6998055255868feddcfa4f086cd86e6cf508815729395aae" + }, + "bdon_vision/deck/visual_data.py": { + "bytes": 3778, + "sha256": "5370e828640f40faf02679ffa303dd7bc36aa031037df998a9c8612b70058ea2" + }, + "bdon_vision/deck/web.html": { + "bytes": 26571, + "sha256": "8bb9a4d8b374ea0c6e934be415b5a5dce2b61c950dc7712f78fb1c03b0091c0f" }, "bdon_vision/engine.py": { - "bytes": 16414, - "sha256": "5141cddb5da7e923e765078a32f5fccc33d0f0c589078a44d2d7fd152dc58b81" + "bytes": 14601, + "sha256": "378cd7c4d13eae3db60a2330d129a33171bfeafe629a6ed186feae2777101800" }, - "bdon_vision/evaluate_open_set.py": { - "bytes": 2906, - "sha256": "6cb15018d0de0b7a02be94bfa8d10eccdcec3b3d5e456e72cdd0aad00395be94" + "bdon_vision/evaluate_engine.py": { + "bytes": 12213, + "sha256": "50beea6fc39644cd67179216b65755fb8fbfff0e68248c3ec9e1ea5135c3bf23" }, - "bdon_vision/fielddata.py": { - "bytes": 7327, - "sha256": "04116e8776699313e91aafc89ce80376a92a73e06c26c6756e2af736aaa602fe" + "bdon_vision/evaluate_kind_encoder.py": { + "bytes": 7833, + "sha256": "a428d5ce730c9e6fa42e92258327ff67da467f43af49e50e458dcf6aa704bb74" + }, + "bdon_vision/evaluate_kind_fields.py": { + "bytes": 18190, + "sha256": "b80da41a8d1d47cd064093addd7e0d56ac9d3fdbfada6c15d3de8f2611261f15" + }, + "bdon_vision/evaluate_locator.py": { + "bytes": 8731, + "sha256": "732a6fc8445c0c7ed0cb3a150d5d892fa34f1f3d97806516ba4ebbae96792a20" + }, + "bdon_vision/export_kind_encoder.py": { + "bytes": 6243, + "sha256": "0809dafc6bf9762b08c7de77129d8139f8f465d4f1fe1866a040902cf508776e" + }, + "bdon_vision/export_locator.py": { + "bytes": 3947, + "sha256": "e132d5d3200039af8857b79c2a66c263de801f13c17507089cd28dd4fbbb07d6" }, "bdon_vision/font.py": { "bytes": 7513, "sha256": "7ffe0a233823423caa8fd3787b6111a4d290211bb91fcfb019db71608973efd2" }, - "bdon_vision/inference.py": { - "bytes": 1985, - "sha256": "b79e22b66bcc358371c076154352b8bec2b94450a70b97f3371721eee65e4186" + "bdon_vision/kind_corpus.py": { + "bytes": 12194, + "sha256": "a28c9ee8e47548070826003722f2713aa4ccc93b39fdec729b2737f59b37e961" + }, + "bdon_vision/kind_encoder.py": { + "bytes": 4635, + "sha256": "108412dd274a891ad782ba65340194d6c928ecc7c5689cb4e6f511db1e2c64a3" + }, + "bdon_vision/kind_fields.py": { + "bytes": 15906, + "sha256": "d1cbdacd63cdcbe43459cb382f5f8714519acd29b3d03f4796a479df3ca48e28" + }, + "bdon_vision/kind_geometry.py": { + "bytes": 7167, + "sha256": "b8855fb419c63612093cdd8f10459df76c6c6ac81b5f88bd8968e79d91db24d0" + }, + "bdon_vision/kind_hardcrops.py": { + "bytes": 13595, + "sha256": "e9f214fa35fc08b17c3b21dccf3b304494fe597dbca5a5069b3e00682452dc84" + }, + "bdon_vision/kind_metrics.py": { + "bytes": 6278, + "sha256": "f1bec2dee4f292ab233e6dc1180b03ad7ef1443006c5fdfd88a8b807f83535f6" + }, + "bdon_vision/kind_pose.py": { + "bytes": 12729, + "sha256": "7231d03035e573056df1b575511f017ff3668161fc7705bc1cfe34bd73a9ac6f" + }, + "bdon_vision/locator.py": { + "bytes": 9360, + "sha256": "307c0eefbe2fc62c73ae8927b174759d0e5789fee79461d94ed1d554a0d69481" + }, + "bdon_vision/locator_eval.py": { + "bytes": 10596, + "sha256": "c6adbc5e80ac3c22272faaba7b53cfa032b88bf269bab08b5711dadae365e06a" + }, + "bdon_vision/locator_model.py": { + "bytes": 3597, + "sha256": "0bb24d5db21865b31f4a4c7b20cedd69d35ea3a38461ce2f240191e557e1b5d3" }, "bdon_vision/native_renderer.py": { "bytes": 28748, "sha256": "8a125c8376fd521a1488cbb4dc6e52fe08747cbeb36023b39f4566a222e1399e" }, - "bdon_vision/ocr.py": { - "bytes": 5019, - "sha256": "668ed8204da7805d7e80f71573d91b91d52b774a25b5a5750704a92a6fdb8b23" - }, "bdon_vision/scenarios.py": { - "bytes": 1873, - "sha256": "a7469ec68f48ab5325e785de337e8fb1f6b29f1b2b4842633bf7b12ecf2cb80a" + "bytes": 2211, + "sha256": "44decca81bed0c668ec212b00739332e8b659cfddd28621e9bc666c8568fffcc" + }, + "bdon_vision/scenes.py": { + "bytes": 55297, + "sha256": "4d43fe2bca0c319a6fc41dead8bdd53c6424cce7e8eb185f811c84d3b25509fd" + }, + "bdon_vision/select_kind_models.py": { + "bytes": 5912, + "sha256": "8b57f86e34f0ba310e01145beba36f5fab648a11ffa31cc51a596d48125c47d0" }, "bdon_vision/server.py": { "bytes": 5159, - "sha256": "66390c4e593830e8d7c3b6dbb7e634e040998330305d5ee08180002e9e504eca" + "sha256": "8bb5179f01edf0d76ae02aa07dc21aa1987c5416eb679962e57375735ba3e08f" }, "bdon_vision/synthetic.py": { "bytes": 14667, @@ -91,149 +203,113 @@ "bytes": 11566, "sha256": "1468ac17ce7f7d45a87707a7028050f2c0df021cc349d4c00c670ad1c0b1223b" }, - "bdon_vision/train_fields.py": { - "bytes": 7249, - "sha256": "26321182c814b4af8b2ca35e8521a039b418d94f60bae028aa68eeb400a96f72" + "bdon_vision/train_kind_encoder.py": { + "bytes": 25059, + "sha256": "1324908386b76243f50fead7f6f05d8db12fb6d228602282a8736a640007a6d8" }, - "bdon_vision/train_scene_encoder.py": { - "bytes": 7392, - "sha256": "52a450f0e4c462e5ace0325463b0db206a8b6577fcb4ff6f845121f513dfbf5f" + "bdon_vision/train_kind_fields.py": { + "bytes": 20847, + "sha256": "d5c98b5f1a2e172b5f98ff6e76de8be755c2eb293b089bd0f9eda31b59685404" + }, + "bdon_vision/train_kind_rank.py": { + "bytes": 9949, + "sha256": "c2c1d313b9a4aac0f4c8727a0d66c0a37de65f18a8193e8c49e19c76b9e178a5" + }, + "bdon_vision/train_locator.py": { + "bytes": 19196, + "sha256": "868d3ea6565fe0da08ca2c974fdd8c60be653312ce81aef3561712bb538fef86" }, "bdon_vision/web.html": { "bytes": 5165, "sha256": "cf267b47e977ac279f0c8ff88931326711022a1ae26e9b7a8271982ef5ed2d0c" }, "bdon_vision/web.js": { - "bytes": 8250, - "sha256": "c4da611db039459644ed362f81b09f3fb6e8dfe35ce6f12e36e2bdd25f3c5b95" + "bytes": 8473, + "sha256": "0286266abb25c52c0269eea04330fba5467a2c7059d45f40b6b0061395c5c732" }, "boxvision.py": { "bytes": 73, "sha256": "9c8e39c2a8f16f237d37e5093ffbb515d22f059dd7810750f46264f300458cc9" }, "docs/ASSET_BOUNDARY.md": { - "bytes": 1695, - "sha256": "b1b1140b3181636846d871be1084ac6919336648d2960524c7041c5c3e5adb7f" + "bytes": 1623, + "sha256": "5a08d40a0e40cfc53eee891df8c43b7797d10e114ba1c5ee16291808ca966310" }, "docs/CONTRACT.md": { - "bytes": 3497, - "sha256": "011a64c34b54f61b07f99e536cc2a34b2be0a6c0d17c5d5bb9a603e422915e80" + "bytes": 4688, + "sha256": "cc3d60a099b3a9743c2f9a8972d50182e04046a44bcc8a97133bbbeaba342d19" }, "docs/COVERAGE.md": { - "bytes": 3336, - "sha256": "6fe031cbdbaeec6e649a7970ab24645a3784d6c8b11287d6de9e9d90b45518e0" + "bytes": 2704, + "sha256": "a9abbee43af39fa8e9b4871ccf7d99c5c938f18b689cddcf4ae2908e92a47853" }, "docs/DATA_CARD.md": { - "bytes": 5063, - "sha256": "0a464541be60b5c0c6b3ed8db81fd24681eea3697468748091b939cdb97b2780" - }, - "docs/EVALUATION.md": { - "bytes": 4579, - "sha256": "9bc35824c95741dacd31f4ca35705a055d16cb52371c3daa4c4ab18097650b69" - }, - "docs/evidence/language-fonts.json": { - "bytes": 2299, - "sha256": "097bb7998febc671dd355df5899880a96f113498442b2a454a89478c5d19c761" + "bytes": 5147, + "sha256": "85a93e9ca4e51e002b03759d9de1e687f7a6dcbf5014c54225ad6cd46f759e2e" }, - "docs/evidence/snap-display-reference.json": { - "bytes": 303, - "sha256": "5172665102f53c6711fa99cd90b3ddda0d90f40564d587ce8faaf07ec91faf67" + "docs/DECK_PIPELINE.md": { + "bytes": 8331, + "sha256": "baf42a19760d47b935e1f34f936e4e49845d5829e1477f88d91241b94c1d078b" }, - "docs/EXPERIMENTS.md": { - "bytes": 2265, - "sha256": "0b0e053e7fc37538ee58750b8c65d49e3e8697ce54b5b4c06b757df257393759" + "docs/EVALUATION.md": { + "bytes": 4462, + "sha256": "1cef4599b74aa4facfdcd208acfdc17d19c2453d85337fecf470a203c0bc90b6" }, "docs/MODEL_CARD.md": { - "bytes": 5929, - "sha256": "a4aae4e4b7382eeb4a9f86a9ca99630281723befb0779eb304f7a9eefa769aad" + "bytes": 8166, + "sha256": "50b90d3be7f5b541efdba639c5724fc9edc6b034c5039d1b0811022b786b43f5" }, "docs/NEW-CARDS.md": { - "bytes": 2508, - "sha256": "b8725f365479e0f8ad0e86e81c64a2cf330adc401a2d3a5c7d6ba0f4e2a98233" + "bytes": 2657, + "sha256": "bea1ad16b93fd8f047619157d2774e575bc94742411e9ad11de1310ead911679" }, "docs/REPRODUCTION.md": { - "bytes": 2998, - "sha256": "adb183e0914b21b5c6a09e8636b026509e1dff4df69b22acea3d09be11e310ae" + "bytes": 2859, + "sha256": "b932ecc5037a1e493b23367b27c3e63d89b67c1b5f0a5c6715607b240075256f" }, "docs/RESOURCES.md": { - "bytes": 3264, - "sha256": "325375a2055362f06cf2db7abc107400c7a120bd0a6e3f63eab810cdf1af5e51" + "bytes": 3318, + "sha256": "51aeb2db5d3643230f68a111aeab68b1179752acfe5c97ee9f57740c054d4d4e" }, "docs/TRAINING.md": { - "bytes": 5103, - "sha256": "3ea925da36da39c0d721f0a39be90341dfa71289ff19918a104a3b5b44d27c18" + "bytes": 10116, + "sha256": "88f4dd470b6df49f02ad2469a43f28908daf3a0a365cf32e2a6e9bc2a0a8587d" }, - "LICENSE": { - "bytes": 1080, - "sha256": "13ee0a55b4b50909b685464b38013fa32814c5c1a2dcf1e3c6f402bad0cb2d8f" + "docs/evidence/language-fonts.json": { + "bytes": 2299, + "sha256": "097bb7998febc671dd355df5899880a96f113498442b2a454a89478c5d19c761" + }, + "docs/evidence/snap-display-reference.json": { + "bytes": 303, + "sha256": "5172665102f53c6711fa99cd90b3ddda0d90f40564d587ce8faaf07ec91faf67" }, "licenses/torchvision-BSD-3-Clause.txt": { "bytes": 1516, "sha256": "2b63e4f6569e0e58260881a1f7ea8b1d9802859ab3cb4b32a45cba246245c8da" }, "pyproject.toml": { - "bytes": 846, - "sha256": "4d59f0aa7782a82fb31b11457890fecae5f2577f4fd68538c54d63defce01544" - }, - "README.md": { - "bytes": 5280, - "sha256": "5ae854d86e8d621f8f2b43cd893e50c80100bf38cb6861daff0004c6506e1c80" - }, - "reports/acceptance/clear-rank-matrix.json": { - "bytes": 9281, - "sha256": "da49d3a0f4590c71dd8dcc9793706ecc80342b4ec38b2ce87069e898950d0bd6" - }, - "reports/acceptance/real-sample-regression.json": { - "bytes": 6186, - "sha256": "24fc6dbce8537fe4e7acfc9418e2eda0388fd1633433101cc5f6cf71dc3c7729" - }, - "reports/acceptance/synthetic-data-audit.json": { - "bytes": 6151, - "sha256": "6be3c4920d66d8d8ffbc35aeedd43cb6afc32cc208ad5c2dfb6975c6b34d669b" - }, - "reports/acceptance/synthetic-full-screen.json": { - "bytes": 134841, - "sha256": "8e53b9db61e8d654468bb5e926625f09330338d393494a23a9bb4ba01f0a52b0" - }, - "reports/acceptance/unknown-card-results.json": { - "bytes": 24210, - "sha256": "659db8b7caa44a00485d40da8b73d4e4458b5ee9e93d20903e86ee75202cc5b2" - }, - "reports/performance/cpu-2-logical-cores.json": { - "bytes": 12762, - "sha256": "769c25844cc4259edf8f709abd9ab03c2bd7aa54e60e6bce50fa9e34b5b27093" - }, - "reports/performance/cpu-4-logical-cores.json": { - "bytes": 12798, - "sha256": "5aff6c8d667aee2dcea13a7014f5b29b70185b5017c39a7a90013f7a32879186" + "bytes": 917, + "sha256": "687005a8615542ba68349c081fd65b7e40c8d2fff0a3b39f1c04c5e9fcba4d51" }, "reports/README.md": { - "bytes": 1050, - "sha256": "f55b4e82c5ed198331d4653dfc7e08b61cd015c837e99db05c8495e9b3409db4" + "bytes": 1109, + "sha256": "3ffe7a099f8ff63fc904700937e53c05a51346f4db6d34aa10a13d8e91ecb496" }, - "reports/training/card-coverage.json": { - "bytes": 13545, - "sha256": "fdc8352f6fb5893a847eecd6177d0b3cb9670fedc6c6bfa2d4d679efa14cc9d4" + "reports/acceptance/holdout-full-screen.json": { + "bytes": 65828, + "sha256": "bfb23860ebb175b2dee5e4b17e429acf6e25fcbfd1dc4d589e471233123fc932" }, - "reports/training/checkpoint-selection.json": { - "bytes": 1060, - "sha256": "f9c8794fb7f743a613e04d12561646bd98ceba61ddd62c99e30a077bfc53a737" - }, - "reports/training/field-learning-curve.json": { - "bytes": 14947, - "sha256": "268f23d30b956cfb8bcfa736d88636267aaae040cc1f41cd7d7e4cedd3ad5140" - }, - "reports/training/hard-field-validation.json": { - "bytes": 3797, - "sha256": "ac4b522b7fea5a70d80c33743bae87ebca4490cdd71e2ae02add612f5cca8a65" + "reports/acceptance/real-sample-regression.json": { + "bytes": 5833, + "sha256": "d6b3a55d75882b6887e3cde9fc803ca7d218ebc59d3aba8dbf23143b2f1c1cae" }, - "reports/training/level-distribution.json": { - "bytes": 3321, - "sha256": "77da1ff83c2915e72ab74effcd3b8b8f97d4ac853b759e9fbc95929e9cc2bc26" + "reports/training/model-selection.json": { + "bytes": 32539, + "sha256": "6565e60ca5879522a4d4afe1e3ea04a0d2ae49ba9e0d4ef019473918a7338b0e" }, - "reports/training/rank-distribution.json": { - "bytes": 597, - "sha256": "5131224dbafeed410d7657b49aeb994bbe9756288c01fd9ad7b7f6e41731033e" + "reports/training/scene-coverage.json": { + "bytes": 14191, + "sha256": "7ccc50b11da1bdca25530ec6dd2c448797a3c2d004f2fdf88ce604d4c483b25b" }, "reproduction/metadata/checkpoint-hashes.json": { "bytes": 582, @@ -644,8 +720,8 @@ "sha256": "5d7d6fdfe6157c5488de9c9c0a5663d574365aa89ea51596239cabac8f0978e1" }, "schemas/box.schema.json": { - "bytes": 13179, - "sha256": "f787e1a3918efa78c23c0e0dbd9f11bde64baa303501932fd77e0e1317e3b0dc" + "bytes": 13517, + "sha256": "d68f167aa18ffa523a3599750297000a9cfbdbacd288419387e0ea88cc77e702" }, "schemas/error.schema.json": { "bytes": 321, @@ -656,29 +732,25 @@ "sha256": "5d68ed0164e71a4b3423edbe934695604e0571b33277f7c795f92ef6f39beea7" }, "schemas/examples/scan-response.json": { - "bytes": 531, - "sha256": "67e45152e88eb84b3702002e645ca33971e36e547d3bc48543e0530ee37496f9" + "bytes": 558, + "sha256": "aa9827e389301aa7eb85136500b89857c05a487b4c4576256d7d7e9ad6702c70" }, "schemas/observations.schema.json": { - "bytes": 7189, - "sha256": "4f99840a874a8c46a645499e8092e2b0bbaf16849092b380338e20825fd48fbc" + "bytes": 13187, + "sha256": "2938f1ab0dc49382ad959ea242400f6ca6bdd510572504d0b5ec6c6e9aa32d50" }, "schemas/scan-request.schema.json": { "bytes": 932, "sha256": "ae26ed6b0b2fac5be0b56109f6342ea815bb9b576e9520f4863492e03fece279" }, "schemas/scan-response.schema.json": { - "bytes": 23438, - "sha256": "834275785d9cfa259a3ebba02b5e19c39b6a041a72dda0909a8ff6b5f6a65475" + "bytes": 30670, + "sha256": "a6667a51d27ad6cf3954aea0c4e44e2d900c3cdbed30a08be0c6742a8dbff982" }, "scripts/audit_coverage.py": { "bytes": 3451, "sha256": "368c40def815ef813aa79fdd9c1e80911f4a34c9e3eb3c4b14ae76914e731cf7" }, - "scripts/diagnose_occlusion.py": { - "bytes": 7441, - "sha256": "77d19dfb15ba1636d510b2edb79241271aa4ff8ddeac8941b0f62da4409af9b4" - }, "scripts/export_fonts.py": { "bytes": 5196, "sha256": "4a6302c95160a12d1f88f857db9122d2c63abbc35de30ee112e7632a03370974" @@ -691,10 +763,6 @@ "bytes": 3197, "sha256": "aeab6de929f221ed6521468b87de16673f2df290eefae95b4cec0cf6505a6402" }, - "scripts/publish_metrics.py": { - "bytes": 4763, - "sha256": "df9fdfb804c80f10a30a8aecdf42a43c3d1b50ce0c49437ca53d24788ceb58ad" - }, "scripts/reproduce_training.py": { "bytes": 8950, "sha256": "879325187cd4b1d1adfb697a54c1a568ba89bae9cadc8c1c76327542f02773c3" @@ -704,32 +772,52 @@ "sha256": "6c4dfb6eecc3a4b4154e85fe51dcd2679e3025280aa7e5ee611d11b16a4441c2" }, "scripts/verify_gallery_update.py": { - "bytes": 3668, - "sha256": "0451fd36fc86250caccfb1e8124f9caa5751f89eefdd58293f29f3c92797848b" + "bytes": 3939, + "sha256": "f1dbbe2d47744baf36e3f7f51482c0186c31b7ecc2229796111ce3a47c8c59f2" }, "scripts/verify_rank_matrix.py": { "bytes": 7499, "sha256": "ecb5be51679496dfc3efba4a787f8296c9e35fbd96c2dbe93c815efd6a0428e4" }, - "scripts/verify_weights.py": { - "bytes": 2178, - "sha256": "cb7b3344e787987d62c74d40c9640806853cefac0d6e0b66b24be65a3ce8b80b" + "tests/test_deck_inputs.py": { + "bytes": 16536, + "sha256": "bef54ab3e89cad08748aaecdb03a9e8a373778ac22c316df945b6e199e6d9475" + }, + "tests/test_deck_json.py": { + "bytes": 2575, + "sha256": "a283b98c319ffa71b65f9a65c33d726d6288403a015c61627462799d051f5a73" }, "tests/test_integrity.py": { - "bytes": 1940, - "sha256": "e8e8ed5df768b58ec7047e8ae0e2d506cbace375e14eda8c6e19b7d27af3679a" + "bytes": 5449, + "sha256": "fe4eaa34694a939e1c2f6492a0313068f56392073ac12567335882e29fcfbd7e" + }, + "tests/test_kind_encoder.py": { + "bytes": 11347, + "sha256": "b6fb9569e25681deb55432842a424b75f07e1196118318bd211fe9c4987a5520" + }, + "tests/test_kind_fields.py": { + "bytes": 10737, + "sha256": "56def48382d46c460ab4d18a8b99edca57e9da41de6d7a2761155342624b845e" + }, + "tests/test_kind_pose.py": { + "bytes": 4521, + "sha256": "bb5e967bf75cdfc8fdbae974faf6f96ce571113533e1597944bf99306ad8c9b0" + }, + "tests/test_locator.py": { + "bytes": 6573, + "sha256": "66b6ea05b632e422dacd9112db63aab08a010d6193e53ac6f85ef8d00130fef4" }, "tests/test_merge.py": { "bytes": 1952, "sha256": "a65f2d4cd7fccd8771bd564dc95b27530410a81672e8d037c925737a2a84850f" }, - "tests/test_ui_geometry.py": { - "bytes": 955, - "sha256": "e59cf5933879299f2f9fb9192abc1993430619609ba90bed1d3fe44c4c4b2079" + "tests/test_scenes.py": { + "bytes": 2783, + "sha256": "160b8e13ee7d65efafc9d84e692bc5027746b14a79833eac0fbe78770cbce1fa" }, - "THIRD_PARTY_NOTICES.md": { - "bytes": 1209, - "sha256": "2749a42dfb22b9ee7d856e1c910730ff73e7609fe388d0742eaff0e14602cce2" + "tests/test_ui_geometry.py": { + "bytes": 1204, + "sha256": "268b28af2acab7079d7c124f12e6ee25d7330b2adbff7f31ac2f4a6e1af75668" } } } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3121e38..c5e3b97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ournotes-boxlens" -version = "0.1.0" +version = "0.2.0" description = "CPU screenshot importer for BanG Dream! Our Notes player collections" license = {file = "LICENSE"} readme = "README.md"