From 79ed05874eba9100aa3b3c7d3629b9757b6d06f9 Mon Sep 17 00:00:00 2001 From: Konstantin Nikolaou <87869540+KonstiNik@users.noreply.github.com> Date: Tue, 16 Jun 2026 19:13:19 +0200 Subject: [PATCH 1/8] fix(analyzer): cache per-batch gradient only when a cross pair needs it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-pair loop cached each eval batch's flat parameter-space gradient unconditionally, but only cross pairs consume it. For large models that kept a P-sized fp32 vector (~29 GB for a 7B model) resident while subsequent batches ran their own backward — enough to OOM a 94 GB H100 even when no cross pair was requested (the second batch's ~84 GB compute peak coincided with the first batch's retained gradient). Cache a batch's gradient only if some cross pair references it, and drop the local reference otherwise so it is freed before the next batch's backward. Self-pair-only runs now peak at a single batch's footprint. Unit suite green; ruff + mypy --strict clean. --- vatis/analyzer.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/vatis/analyzer.py b/vatis/analyzer.py index 5355cf1..5b0b1c3 100644 --- a/vatis/analyzer.py +++ b/vatis/analyzer.py @@ -250,18 +250,27 @@ def _run_one_checkpoint( for p in bundle.params: p.requires_grad_(True) - # Self pairs first; we cache the per-batch grads (parameter-vector flat) - # so the cross pairs can re-use them via dot product. + # Self pairs first. We cache a batch's full parameter-space gradient + # ONLY if some cross pair will consume it via the dot product. Caching it + # unconditionally keeps a P-sized vector resident (e.g. ~29 GB fp32 for a + # 7B model) while subsequent batches run their own backward — for large + # models that retained gradient is enough to OOM even when no cross pair + # was requested. So we cache selectively and drop the local reference + # otherwise, freeing it before the next batch's backward. + cross_names = {n for pair in self.cross_pairs for n in pair if pair[0] != pair[1]} per_batch_grad_cache: dict[str, torch.Tensor] = {} per_batch_n_valid: dict[str, int] = {} for name, spec in self.eval_batches.items(): self_result, g_full = self._compute_self_pair(bundle, ckpt_id, revision, name, spec) result.rows.extend(self_result) - # The cached gradient lives on the model device; sized P (params). - if g_full is not None: - per_batch_grad_cache[name] = g_full per_batch_n_valid[name] = self._last_n_valid + if g_full is not None and name in cross_names: + # Sized P (params); lives on the model device for the dot product. + per_batch_grad_cache[name] = g_full + # Drop the local reference; if it wasn't cached above it is freed now, + # so it can't pile up across batches (the large-model OOM footgun). + g_full = None # Cross pairs (if any) — only delta_loss + chi_pos. for a, b in self.cross_pairs: From c7ef413db8fb31ef52bdd5dea96eeda234bfa849 Mon Sep 17 00:00:00 2001 From: Konstantin Nikolaou <87869540+KonstiNik@users.noreply.github.com> Date: Tue, 16 Jun 2026 19:13:37 +0200 Subject: [PATCH 2/8] examples: OLMo-3-7B-Think-SFT spectral-position SFT analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A realistic, production-scale vatis example: load Olmo-3-7B-Think-SFT and compute the LNP decomposition (chi_loss/chi_net/delta_loss and the headline chi_pos spectral position) on real Dolci-Think-SFT data — 10 code + 10 English samples, chat-templated with completion-only loss masking (the actual SFT objective). Two-stage offline workflow for the internet-less compute nodes: prefetch.py (login node, online) selects samples -> samples.json and warms the model into the HF cache (needs 'datasets', a documented manual venv install, deliberately not a repo dep); olmo_sft_analyze.py (GPU node, offline) runs analyze() with resource tracking; analyze_results.py turns parquet+resources into plots/tables with zero vatis imports. resource_tracker.py records wall time, torch peak memory, and GPU engine activity via DCGM dmon (nvidia-smi fallback). mem_profile.py is the GPU memory profiler used to localize the 7B memory cost. Adds tokenize_chat_sft_sample / stack_lm_batch to examples/_helpers.py. First run (1x H100): chi_pos ~1e-9 — the SFT loss gradient sits deep in the spectral tail, as the dpo_spectral_filter hypothesis predicts for a late-SFT checkpoint; peak 84 GB, 195 s, autograd/memory-bound (tensor-core ~0%). Cross-pair alignment is left off (needs two gradients resident); CPU-offload is the planned follow-up. --- examples/_helpers.py | 137 + examples/olmo_sft_analysis/README.md | 211 + examples/olmo_sft_analysis/analyze_results.py | 264 + examples/olmo_sft_analysis/gpu_activity.png | Bin 0 -> 72533 bytes examples/olmo_sft_analysis/lnp_components.png | Bin 0 -> 68702 bytes examples/olmo_sft_analysis/mem_profile.md | 14 + examples/olmo_sft_analysis/mem_profile.py | 191 + examples/olmo_sft_analysis/mem_profile.sbatch | 25 + .../olmo_sft_analysis/olmo_sft_analyze.py | 194 + examples/olmo_sft_analysis/prefetch.py | 211 + .../olmo_sft_analysis/resource_tracker.py | 399 + examples/olmo_sft_analysis/resources.json | 8695 +++++++++++++++++ examples/olmo_sft_analysis/results.parquet | Bin 0 -> 3576 bytes examples/olmo_sft_analysis/run.sbatch | 54 + examples/olmo_sft_analysis/samples.json | 451 + .../olmo_sft_analysis/spectral_position.png | Bin 0 -> 33537 bytes 16 files changed, 10846 insertions(+) create mode 100644 examples/olmo_sft_analysis/README.md create mode 100644 examples/olmo_sft_analysis/analyze_results.py create mode 100644 examples/olmo_sft_analysis/gpu_activity.png create mode 100644 examples/olmo_sft_analysis/lnp_components.png create mode 100644 examples/olmo_sft_analysis/mem_profile.md create mode 100644 examples/olmo_sft_analysis/mem_profile.py create mode 100644 examples/olmo_sft_analysis/mem_profile.sbatch create mode 100644 examples/olmo_sft_analysis/olmo_sft_analyze.py create mode 100644 examples/olmo_sft_analysis/prefetch.py create mode 100644 examples/olmo_sft_analysis/resource_tracker.py create mode 100644 examples/olmo_sft_analysis/resources.json create mode 100644 examples/olmo_sft_analysis/results.parquet create mode 100644 examples/olmo_sft_analysis/run.sbatch create mode 100644 examples/olmo_sft_analysis/samples.json create mode 100644 examples/olmo_sft_analysis/spectral_position.png diff --git a/examples/_helpers.py b/examples/_helpers.py index 1d50209..f5d8349 100644 --- a/examples/_helpers.py +++ b/examples/_helpers.py @@ -5,6 +5,143 @@ import torch from transformers import AutoTokenizer +# A "chat sample" is a list of OpenAI-style message dicts, e.g. +# ``[{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]``. +ChatSample = list[dict[str, str]] + + +def _common_prefix_len(a: list[int], b: list[int]) -> int: + """Length of the longest shared prefix of two token-id lists.""" + n = 0 + for x, y in zip(a, b, strict=False): + if x != y: + break + n += 1 + return n + + +def _render_full_and_prompt_len( + messages: ChatSample, tokenizer: AutoTokenizer +) -> tuple[list[int], int]: + """Render the conversation and locate the prompt/completion boundary. + + transformers 5.x returns a ``BatchEncoding`` from + ``apply_chat_template(tokenize=True)``, not a flat ``list[int]``. To stay + robust across versions we render to *text* first (``tokenize=False``) and + tokenize the string with ``add_special_tokens=False`` (the template already + emits the special / role tokens). This reproduces the tokenized template + exactly while giving plain id lists to diff. + + Returns ``(full_ids, prompt_len)`` where ``prompt_len`` is the longest + common prefix of the prompt-only (``messages[:-1]``, ``add_generation_prompt + =True``) and full renderings — the token index at which the assistant's + completion content begins. The common-prefix rule is robust to templates + that re-emit the generation header slightly differently in the two passes. + """ + full_text = tokenizer.apply_chat_template(messages, add_generation_prompt=False, tokenize=False) + prompt_text = tokenizer.apply_chat_template( + messages[:-1], add_generation_prompt=True, tokenize=False + ) + full_ids = tokenizer(full_text, add_special_tokens=False)["input_ids"] + prompt_ids = tokenizer(prompt_text, add_special_tokens=False)["input_ids"] + return full_ids, _common_prefix_len(prompt_ids, full_ids) + + +def sft_prompt_completion_lengths( + messages: ChatSample, + tokenizer: AutoTokenizer, +) -> tuple[int, int]: + """Return ``(prompt_len, total_len)`` in tokens for an SFT chat sample. + + ``prompt_len`` is the number of leading tokens that belong to the prompt + (everything up to and including the final assistant header, i.e. the point + where the assistant's *content* begins); ``total_len`` is the length of the + full rendered conversation. Used by ``prefetch.py`` to keep only samples + that will still have real completion tokens after truncation to ``seq_len``. + + The boundary is computed as the longest common prefix between the + prompt-only and full renderings (see :func:`_render_full_and_prompt_len`). + """ + full_ids, prompt_len = _render_full_and_prompt_len(messages, tokenizer) + return prompt_len, len(full_ids) + + +def tokenize_chat_sft_sample( + messages: ChatSample, + tokenizer: AutoTokenizer, + *, + seq_len: int, +) -> dict[str, torch.Tensor]: + """Render one chat sample into a fixed-length, completion-only LM example. + + This is the *realistic* SFT objective: cross-entropy is computed only on + the assistant's response tokens; the prompt (system + user turns, plus the + assistant header) is masked out with ``-100``. Contrast with + :func:`tokenize_into_lm_batch`, which trains on every token of a flat text. + + The model's own chat template defines the prompt/completion boundary. We + render the conversation twice — full and prompt-only — and take the common + prefix as the boundary (see :func:`sft_prompt_completion_lengths`). + + Labels follow the **pre-shifted** HF-causal-LM convention that vatis expects + (``causal_lm_loss`` compares ``logits[i]`` against ``labels[i]`` with no + further shift): ``labels[i] = ids[i + 1]`` when ``ids[i + 1]`` is a + completion token, else ``-100``; ``labels[-1] = -100``. + + The example is then right-truncated (keeping the prompt prefix and as much + of the completion as fits) or right-padded to exactly ``seq_len``. Padding + uses ``pad_token_id`` (falling back to ``eos_token_id``) with + ``attention_mask = 0`` and ``labels = -100`` so pads count toward neither + the loss nor the valid-token denominator. + + Returns a dict of ``(seq_len,)`` tensors: ``input_ids``, ``attention_mask``, + ``labels``. Stack several with :func:`stack_lm_batch` to form a batch. + """ + if seq_len < 2: + raise ValueError(f"seq_len must be >= 2, got {seq_len}") + + full_ids, prompt_len = _render_full_and_prompt_len(messages, tokenizer) + + ids = torch.tensor(full_ids, dtype=torch.long) + total = ids.shape[0] + + # Pre-shifted next-token targets: labels[i] = ids[i + 1]. + labels = torch.full((total,), fill_value=-100, dtype=torch.long) + labels[:-1] = ids[1:] + # Mask every target that lands inside the prompt. Position i predicts + # ids[i + 1]; that target is a completion token iff i + 1 >= prompt_len, + # i.e. i >= prompt_len - 1. So positions [0, prompt_len - 2] are masked. + if prompt_len >= 1: + labels[: prompt_len - 1] = -100 + + attention_mask = torch.ones(total, dtype=torch.long) + + # Truncate (keep the prefix) or pad on the right to exactly seq_len. + pad_id = tokenizer.pad_token_id + if pad_id is None: + pad_id = tokenizer.eos_token_id + if pad_id is None: + pad_id = 0 + + if total >= seq_len: + ids = ids[:seq_len].contiguous() + labels = labels[:seq_len].contiguous() + attention_mask = attention_mask[:seq_len].contiguous() + else: + pad = seq_len - total + ids = torch.cat([ids, torch.full((pad,), pad_id, dtype=torch.long)]) + labels = torch.cat([labels, torch.full((pad,), -100, dtype=torch.long)]) + attention_mask = torch.cat([attention_mask, torch.zeros(pad, dtype=torch.long)]) + + return {"input_ids": ids, "attention_mask": attention_mask, "labels": labels} + + +def stack_lm_batch(samples: list[dict[str, torch.Tensor]]) -> dict[str, torch.Tensor]: + """Stack a list of equal-length single-example dicts into a ``(B, S)`` batch.""" + if not samples: + raise ValueError("cannot stack an empty list of samples") + return {key: torch.stack([s[key] for s in samples], dim=0) for key in samples[0]} + def tokenize_into_lm_batch( text: str, diff --git a/examples/olmo_sft_analysis/README.md b/examples/olmo_sft_analysis/README.md new file mode 100644 index 0000000..f8691c5 --- /dev/null +++ b/examples/olmo_sft_analysis/README.md @@ -0,0 +1,211 @@ +# Realistic SFT analysis: spectral position of `Olmo-3-7B-Think-SFT` + +A production-scale, real-data vatis example. It loads Ai2's **`Olmo-3-7B-Think-SFT`** +checkpoint and computes the LNP decomposition — `chi_loss`, `chi_net`, +`delta_loss`, and the headline **`chi_pos` (spectral position)** — on real +supervised-fine-tuning data: 10 *code* and 10 *English* samples drawn from +**`Dolci-Think-SFT`**, the exact mixture this checkpoint was trained on. Wall +time, peak GPU memory, and GPU engine activity are tracked throughout. + +## What gets computed + +Each sample is tokenized with the model's **chat template** and **completion-only +loss masking** — cross-entropy on the assistant's response tokens only, the +prompt masked with `-100`. That is the actual SFT objective, so the gradient we +probe is the real SFT gradient. + +The two batches (`code`, `english`) are evaluated as self-pairs, plus the +`(english, code)` cross-pair (free — the gradients are already cached): + +| observable | meaning here | +|---|---| +| `chi_pos` (self) | **spectral position** of that batch's SFT loss gradient, in `[0, 1]`. →1 = gradient rides the bulk (dominant, well-resolved eigenmodes); →0 = it lives in the tail (weak, fine-grained directions). | +| `chi_pos` (cross) | parameter-space cosine between the code and English SFT gradients, in `[-1, 1]`. | +| `delta_loss` (cross) | `⟨∇θ L_english, ∇θ L_code⟩` — does an SFT step on English help (`>0`) or fight (`<0`) code? | +| `chi_loss_*`, `chi_net_*` | the loss-curvature and Jacobian-norm factors of the decomposition. | + +Real, on-distribution text matters: `chi_net` is the squared Frobenius norm of +the parameter Jacobian *evaluated at the input*, so off-distribution tokens make +the observable meaningless. Using the model's own training mixture +(`Dolci-Think-SFT`) puts the evaluation on the manifold the weights actually saw. + +## The two-stage offline workflow + +The compute nodes on this cluster have **no internet** (`HF_HUB_OFFLINE=1` in the +repo `.env`). So everything is split into an online prefetch and an offline +compute step. + +### Prerequisites + +- `HF_HOME` set (shell or repo-root `.env`) — see [`../../run_config.py`](../../run_config.py). +- vatis with the examples extra (for matplotlib): `uv sync --extra examples`. +- **`datasets`** — needed *only* by `prefetch.py`. It is deliberately **not** a + vatis dependency (vatis is dataset-agnostic); install it into the venv just + for the prefetch step: + + ```bash + uv pip install --python .venv/bin/python datasets + ``` + +- **DCGM** for GPU-activity tracking — uses the system `dcgmi` CLI, **no Python + package**. If `dcgmi` (or its profiling fields, e.g. on a MIG slice) is + unavailable, the tracker automatically falls back to sampling `nvidia-smi`. + +### Stage 1 — prefetch (login node, **online**) + +Downloads the model into the HF cache and selects the 20 samples into +`samples.json` (so the offline step needs no dataset dependency at all): + +```bash +.venv/bin/python examples/olmo_sft_analysis/prefetch.py +``` + +`prefetch.py` forces HF online (overriding `.env`'s `offline=1`, which is meant +for compute nodes) while keeping `HF_HOME` from `.env`. Use +`--skip-model-download` to fetch only the tokenizer + data (the ~14 GB weights +can be pulled later). Knobs: `--n-per-batch`, `--seq-len`, `--max-prompt-frac`, +`--scan-limit`. + +### Stage 2 — analyze (GPU node, **offline**) + +```bash +examples/benchmark/submit.sh examples/olmo_sft_analysis/run.sbatch +# or interactively on a GPU: +.venv/bin/python examples/olmo_sft_analysis/olmo_sft_analyze.py +``` + +Loads the cached model in bf16, builds the two completion-only batches, runs +`analyze()` wrapped in resource tracking. Writes `results.parquet` and +`resources.json`. `run.sbatch` pins `--chi-net-method hutchinson`, +`--seq-len 1024`, `--no-cross-pairs` for memory reasons — see Hardware below. + +### Stage 3 — plots & tables (CPU) + +```bash +.venv/bin/python examples/olmo_sft_analysis/analyze_results.py +``` + +Zero vatis imports — only `pyarrow` + `matplotlib`. The parquet schema and +`resources.json` are the entire contract with the compute step (the same +compute/analysis split as [`../analyze_results.py`](../analyze_results.py)). + +## Resource tracking + +[`resource_tracker.py`](resource_tracker.py) records three things, sliced per +phase (`load_model`, `analyze`): + +- **wall time** — `perf_counter`. +- **peak GPU memory** — `torch.cuda.max_memory_allocated` / `max_memory_reserved`. +- **GPU activity** — `dcgmi dmon` in a background thread: `SM active`, + `tensor-pipe active`, `DRAM (memory-bandwidth) active`, `GR engine active` + (fractions), plus power / framebuffer / temperature. + +Why DCGM and not just `nvidia-smi`? The coarse `utilization.gpu` is the percent +of wall time *at least one* kernel was running — on a single-stream, +backward-heavy job it pins near 100% and says nothing about how hard the GPU +works. DCGM's SM-active / tensor-active / DRAM-active counters do (they show, for +example, whether the bf16 matmuls actually light up the tensor cores and whether +the job is compute- or memory-bound). + +## Hardware & runtime + +Target: a single ~93 GiB GPU (measured on a capella H100). It took five runs and +a dedicated profiler ([`mem_profile.py`](mem_profile.py)) to nail the 7B memory +model. The **measured** per-batch breakdown (`mem_profile.md`): + +- **delta_loss peak ≈ 75 GB, flat in `seq_len` and batch size.** It is + *param-bound*: bf16 weights (14.6) + the fp32 flat gradient accumulator for + `delta_loss` (29.2) + a bf16 grad tuple (14.6) + ~16 GB backward workspace. + Activations are negligible at these sizes, so **`seq_len` is essentially free** + (delta_loss peak: 74.7 GB at S=64, 74.8 GB at S=512). +- `chi_net` step: `hutchinson` ~44 GB, `per_sequence_cv` ~64 GB (both in + isolation); with the live 29 GB flat-grad that becomes ~73 GB / ~94 GB. + +The four earlier OOMs were **a vatis bug, not a sizing limit** (fixed in +`analyzer.py`): the analyzer cached *every* batch's full gradient +unconditionally — only cross pairs ever consume it — so the second batch ran its +own ~75 GB `delta_loss` while the first batch's ~29 GB gradient was still +resident (~104 GB → OOM). The fix caches a gradient only if a cross pair needs +it. With cross pairs off, nothing is retained and each batch peaks at ~75 GB. + +So `run.sbatch` uses **`--chi-net-method hutchinson`** (keeps the chi_net step +~73 GB, under the delta_loss peak; `per_sequence_cv` + the live flat-grad would +be ~94 GB, too tight) at **`--seq-len 1024`** (free, since seq_len barely moves +peak), plus `PYTORCH_ALLOC_CONF=expandable_segments:True` and `--no-cross-pairs`. +Trade-off: hutchinson re-runs the forward per probe (slower) and is +higher-variance than the control-variate method. + +Knobs: raise `--seq-len` to capture more of the long `` traces — it costs +almost no memory, only wallclock (raise `--time` accordingly). `--n-hutchinson` +does **not** change peak. + +**Still out of reach on one GPU:** the **cross pair** needs *two* full gradients +resident simultaneously (~104 GB) — the natural fix is to CPU-offload the cached +gradient (future work). `per_sequence_cv` is borderline (~94 GB) for the same +flat-grad-coexistence reason. + +**GPU activity sampler:** `dcgmi` requires a running host engine and is not +accessible on capella, so the tracker falls back to `nvidia-smi` (coarse +util% / memory / power). Peak memory still comes from `torch.cuda`, which is the +number that matters most here. + +## Outputs (next to the scripts) + +- `samples.json` — the 20 selected chat samples + selection metadata (from prefetch). +- `results.parquet` — canonical long-format LNP table. +- `resources.json` — time / memory / GPU-activity summary + per-sample timeline. +- `spectral_position.png`, `lnp_components.png`, `gpu_activity.png`. + +## Findings + +First successful run: `Olmo-3-7B-Think-SFT` on 10 code + 10 English +`Dolci-Think-SFT` samples, `S=1024`, `hutchinson` `n=32`, self-pairs, 1× H100. + +| observable | code | english | +|---|---|---| +| **`chi_pos` (spectral position)** | 1.11e-9 | 1.65e-9 | +| `delta_loss` (‖∇θ L‖²) | 2.95 | 4.38 | +| `chi_loss_normalized` | 0.322 | 0.323 | +| `chi_net_normalized` | 8.25e9 | 8.22e9 | +| `chi_loss` | 4.03e-5 | 4.19e-5 | +| `chi_net` (Tr eNTK) | 6.58e13 | 6.34e13 | + +- **The SFT loss gradient sits deep in the spectral tail.** `chi_pos ≈ 1e-9` for + both batches — essentially the bottom of `[0, 1]`. For a *fully* SFT-trained + checkpoint this is exactly what the [`dpo_spectral_filter`](../dpo_spectral_filter/README.md) + hypothesis predicts: late SFT has exhausted the bulk, so the CE gradient lives + in its own tail (`chi_pos(CE, CE)` small). The smallness is dominated by the + enormous `chi_net` (Tr eNTK ≈ 6.6e13); with `hutchinson n=32` carrying variance + on `chi_net`, treat the absolute `chi_pos` as order-of-magnitude, not 3-sig-fig. +- **Code and English are near-identical in `chi_loss`/`chi_net`** and differ + mainly in `delta_loss` (English's gradient is larger). The more interpretable + *cross*-distribution alignment (`chi_pos(english, code)`, the parameter-space + gradient cosine) is the natural next measurement — it needs the cross pair, + hence the memory work below. +- **Resources (1× H100):** wall 195 s (load 18 s + analyze 177 s); **torch peak + 84 GB allocated / 94 GB reserved** (fits the 93 GiB card). DCGM shows the run is + **autograd/memory-bound, not tensor-core-bound**: GR-engine ~86%, SM-active + ~54%, tensor-core ~0%, DRAM ~29%, ~276 W. (DCGM's `FB_USED` under-reports here + (~1.7 GB) — trust the `torch.cuda` peak for memory.) The real-run peak (84 GB) + is a bit above the per-step profiler figure (~75 GB) because `chi_loss`, + `delta_loss`, and `chi_net` overlap within one `_compute_self_pair` at S=1024. + +**Next steps:** (1) the cross pair + `per_sequence_cv` need two gradients +resident — CPU-offload the cached gradient to fit; (2) longer sequences are cheap +on memory but cost wallclock; (3) higher `n_hutchinson` to tighten `chi_net`. + +## Notes & knobs + +- **Sample selection** is deterministic: the first `--n-per-batch` code samples + (sources matching `Code|Python|Algorithms|Nemotron`) and English samples + (curated natural-language chat sources, e.g. `Persona Precise IF`) whose prompt + fits in `--max-prompt-frac × seq_len` so real completion tokens survive + truncation. The selected `id`s and lengths are recorded in `samples.json`. +- **The model is the SFT-stage Think checkpoint** (`Olmo-3-7B-Think-SFT`), + evaluated on its own training mixture. Point `--model` / `prefetch.py --dataset` + elsewhere (e.g. `Olmo-3-7B-Instruct-SFT` + `Dolci-Instruct-SFT`) to compare + branches; keep model and data on-distribution for the observables to mean + anything. +- A custom **DPO** loss (the next step toward the `dpo_spectral_filter` + diagnostics) is not wired in here — vatis v1 only supports the closed-form CE + path for `chi_loss`. This example deliberately stays within that supported path. diff --git a/examples/olmo_sft_analysis/analyze_results.py b/examples/olmo_sft_analysis/analyze_results.py new file mode 100644 index 0000000..c5f4c80 --- /dev/null +++ b/examples/olmo_sft_analysis/analyze_results.py @@ -0,0 +1,264 @@ +"""Stage 3 (analysis, CPU): plot + tabulate the OLMo SFT LNP results. + +Reads ``results.parquet`` (the canonical vatis output) and ``resources.json`` +(time / memory / GPU activity) and produces: + +- ``spectral_position.png`` — ``chi_pos`` for the code and English self-pairs + (the spectral position of each batch's SFT loss gradient, in [0, 1]) plus the + cross-pair ``chi_pos(english, code)`` (the parameter-space gradient cosine, + in [-1, 1]). +- ``lnp_components.png`` — the full decomposition: ``chi_loss_normalized``, + ``chi_net_normalized``, ``delta_loss``, ``chi_pos`` as small multiples. +- ``gpu_activity.png`` — the DCGM activity timeline (SM / tensor / DRAM active, + framebuffer memory) with the load vs analyze phases shaded. + +It also prints a markdown observable table and a resource summary. + +Like ``examples/analyze_results.py``, this script has **zero vatis imports** — +only ``pyarrow`` and ``matplotlib`` (and stdlib ``json``). The parquet schema +plus the resources JSON are the entire contract with the compute step. + +Run (CPU is fine):: + + .venv/bin/python examples/olmo_sft_analysis/analyze_results.py +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") # headless + +import matplotlib.pyplot as plt # noqa: E402 +import pyarrow.parquet as pq # noqa: E402 + +_HERE = Path(__file__).resolve() +PARQUET_PATH = _HERE.parent / "results.parquet" +RESOURCES_PATH = _HERE.parent / "resources.json" + +CODE = "code" +ENGLISH = "english" +CROSS = (ENGLISH, CODE) + +# (key, human label) for the fraction-valued DCGM activity channels. +_ACTIVITY_CHANNELS = [ + ("sm_active", "SM active"), + ("tensor_active", "tensor core"), + ("dram_active", "DRAM (mem BW)"), + ("gr_engine_active", "GR engine"), +] + + +def pair_label(a: str, b: str) -> str: + return a if a == b else f"{a}×{b}" + + +def load_observables(path: Path) -> dict[str, dict[str, float]]: + """Return ``{observable: {pair_label: value}}`` from the parquet.""" + rows = pq.read_table(path).to_pylist() + out: dict[str, dict[str, float]] = {} + for r in rows: + out.setdefault(r["observable"], {})[pair_label(r["batch_a"], r["batch_b"])] = r["value"] + return out + + +def plot_spectral_position(obs: dict[str, dict[str, float]], path: Path) -> None: + chi_pos = obs.get("chi_pos", {}) + labels = [CODE, ENGLISH, pair_label(*CROSS)] + values = [chi_pos.get(lbl) for lbl in labels] + present = [(lbl, v) for lbl, v in zip(labels, values, strict=True) if v is not None] + if not present: + return + fig, ax = plt.subplots(figsize=(6, 4.5)) + xs = range(len(present)) + colors = ["#1f77b4", "#1f77b4", "#d62728"] + ax.bar(list(xs), [v for _, v in present], color=colors[: len(present)]) + for x, (_, v) in zip(xs, present, strict=True): + ax.text(x, v, f"{v:.2e}", ha="center", va="bottom" if v >= 0 else "top", fontsize=9) + ax.set_xticks(list(xs)) + ax.set_xticklabels([lbl for lbl, _ in present]) + ax.axhline(0.0, color="black", linewidth=0.7, alpha=0.5) + ax.set_ylabel("chi_pos (spectral position)") + ax.set_title( + "Spectral position of the SFT loss gradient\n" + "self pairs ∈ [0,1]: bulk→1, tail→0 | cross = gradient cosine ∈ [-1,1]" + ) + ax.grid(True, axis="y", alpha=0.3) + fig.tight_layout() + fig.savefig(path, dpi=120) + plt.close(fig) + print(f"wrote {path}") + + +def plot_lnp_components(obs: dict[str, dict[str, float]], path: Path) -> None: + panels = ["chi_loss_normalized", "chi_net_normalized", "delta_loss", "chi_pos"] + logy = {"chi_net_normalized"} + labels = [CODE, ENGLISH, pair_label(*CROSS)] + fig, axes = plt.subplots(1, 4, figsize=(15, 4)) + for ax, name in zip(axes, panels, strict=True): + series = obs.get(name, {}) + present = [(lbl, series[lbl]) for lbl in labels if lbl in series] + if not present: + ax.set_visible(False) + continue + xs = range(len(present)) + ax.bar(list(xs), [v for _, v in present], color="#4c72b0") + ax.set_xticks(list(xs)) + ax.set_xticklabels([lbl for lbl, _ in present], rotation=20, ha="right") + ax.set_title(name) + ax.axhline(0.0, color="black", linewidth=0.7, alpha=0.5) + if name in logy and all(v > 0 for _, v in present): + ax.set_yscale("log") + ax.grid(True, axis="y", alpha=0.3) + fig.suptitle("LNP decomposition — Olmo-3-7B-Think-SFT on Dolci-Think-SFT samples") + fig.tight_layout() + fig.savefig(path, dpi=120) + plt.close(fig) + print(f"wrote {path}") + + +def plot_gpu_activity(resources: dict, path: Path) -> None: + timeline = resources.get("timeline", []) + summary = resources.get("summary", {}) + if not timeline: + print("no GPU timeline to plot (sampler collected no samples).") + return + ts = [s["t"] for s in timeline] + fig, ax = plt.subplots(figsize=(9, 4.5)) + plotted = False + for key, label in _ACTIVITY_CHANNELS: + ys = [s.get(key) for s in timeline] + if any(y is not None for y in ys): + ax.plot( + ts, + [(y * 100.0 if y is not None else None) for y in ys], + marker=".", + ms=3, + label=label, + ) + plotted = True + # Coarse util% fallback if no profiling channels (nvidia-smi sampler). + if not plotted: + ys = [s.get("gpu_util_pct") for s in timeline] + if any(y is not None for y in ys): + ax.plot(ts, ys, marker=".", ms=3, label="GPU util (coarse)") + plotted = True + ax.set_xlabel("time since tracker start (s)") + ax.set_ylabel("engine active (%)") + ax.set_ylim(0, 105) + + # Framebuffer memory on a secondary axis. + fb = [s.get("fb_used_mb") for s in timeline] + if any(v is not None for v in fb): + ax2 = ax.twinx() + ax2.plot( + ts, + [(v / 1024.0 if v is not None else None) for v in fb], + color="gray", + linestyle="--", + label="FB mem (GB)", + ) + ax2.set_ylabel("framebuffer memory (GB)") + ax2.legend(loc="upper right") + + # Shade the named phases. + phase_colors = {"load_model": "#ffeda0", "analyze": "#c7e9c0"} + for p in summary.get("phases", []): + t0, t1 = p.get("t_start"), p.get("t_end") + if t0 is None or t1 is None: + continue + ax.axvspan(t0, t1, color=phase_colors.get(p["name"], "#eeeeee"), alpha=0.5, zorder=0) + ax.text((t0 + t1) / 2, 102, p["name"], ha="center", va="top", fontsize=9) + + sampler = summary.get("sampler", "?") + ax.set_title(f"GPU activity during analysis (sampler: {sampler})") + ax.legend(loc="upper left") + ax.grid(True, alpha=0.3) + fig.tight_layout() + fig.savefig(path, dpi=120) + plt.close(fig) + print(f"wrote {path}") + + +def print_observable_table(obs: dict[str, dict[str, float]]) -> None: + cols = [CODE, ENGLISH, pair_label(*CROSS)] + order = [ + "chi_pos", + "delta_loss", + "chi_loss_normalized", + "chi_net_normalized", + "chi_loss", + "chi_net", + ] + names = [o for o in order if o in obs] + [o for o in obs if o not in order] + print("\n### LNP observables\n") + print("| observable | " + " | ".join(cols) + " |") + print("|" + "---|" * (len(cols) + 1)) + for name in names: + cells = [] + for c in cols: + v = obs[name].get(c) + cells.append(f"{v:+.4e}" if v is not None else "—") + print(f"| {name} | " + " | ".join(cells) + " |") + + +def print_resource_summary(resources: dict) -> None: + s = resources.get("summary", {}) + if not s: + return + print("\n### Resources\n") + print( + f"- device: `{s.get('device')}`" + (f" ({s.get('gpu_name')})" if s.get("gpu_name") else "") + ) + print( + f"- GPU sampler: `{s.get('sampler')}` " + f"({s.get('n_gpu_samples')} samples @ {s.get('sampler_interval_s')}s)" + ) + print(f"- wall total: {s.get('wall_s_total', 0):.1f} s") + print( + f"- torch peak: {s.get('torch_peak_alloc_mb', 0):.0f} MB allocated, " + f"{s.get('torch_peak_reserved_mb', 0):.0f} MB reserved" + ) + gpu = s.get("gpu", {}) + + def stat(key: str, scale: float = 1.0, unit: str = "") -> str: + st = gpu.get(key) + return ( + f"mean {st['mean'] * scale:.1f}{unit}, max {st['max'] * scale:.1f}{unit}" if st else "—" + ) + + print(f"- SM active: {stat('sm_active', 100, '%')}") + print(f"- tensor-core active: {stat('tensor_active', 100, '%')}") + print(f"- DRAM (mem-BW) active: {stat('dram_active', 100, '%')}") + print(f"- power: {stat('power_w', 1, ' W')}") + print("\n| phase | wall_s | torch peak MB | SM active mean |") + print("|---|---|---|---|") + for p in s.get("phases", []): + sm = p.get("gpu", {}).get("sm_active") + smtxt = f"{sm['mean'] * 100:.0f}%" if sm else "—" + wall = f"{p['wall_s']:.1f}" if p.get("wall_s") is not None else "—" + print(f"| {p['name']} | {wall} | {p.get('torch_peak_alloc_mb', 0):.0f} | {smtxt} |") + + +def main() -> None: + if not PARQUET_PATH.exists(): + raise SystemExit(f"{PARQUET_PATH} not found — run olmo_sft_analyze.py first.") + obs = load_observables(PARQUET_PATH) + + plot_spectral_position(obs, _HERE.parent / "spectral_position.png") + plot_lnp_components(obs, _HERE.parent / "lnp_components.png") + print_observable_table(obs) + + if RESOURCES_PATH.exists(): + resources = json.loads(RESOURCES_PATH.read_text()) + plot_gpu_activity(resources, _HERE.parent / "gpu_activity.png") + print_resource_summary(resources) + else: + print(f"\n(no {RESOURCES_PATH.name} — skipping resource plots/table)") + + +if __name__ == "__main__": + main() diff --git a/examples/olmo_sft_analysis/gpu_activity.png b/examples/olmo_sft_analysis/gpu_activity.png new file mode 100644 index 0000000000000000000000000000000000000000..4c31d55c30bfd6d156c727ba6e477d43085360ad GIT binary patch literal 72533 zcmdSBWmJ}37c~khq9~w(2uLU>ozg8xOLv1H-QA%mC@Ip7ARsLu-O}CN-QE4I`+48@ z8RHw{ob&(u5Q&@nx~{#~UTe;|=Hjc2ln6Q+J{l4d61teEpd1p?Ep{ZN8w7W5!*{SQ ziNfJOPCFqLJ9$e3JI8l6`bd)R?5xZz?aYjIpE>B;*cw|}FwrqG(lO9HGqSU@vgM?w zH~;q)be1-T^x@5^mhdijtwdFAk&qtQA^ssx84x=mAtND)3GypA#cfPF+9?PH`QI<#I|&iD*#G@1WLeDX z&A%^oPWb=zk66>6KKt}wboOX-oPf(A;M=zc8X6iq`}-*>`MTJ&Y6AvWwCcrHr+fPQ zhT-C${r&HyzKu*%%2E3@7_mEZy1$G?NZ48X;ik2XO@Zo?5Bk$m>-pQ8V?`-evwu}f ztVea=^NDKr*ynG2bSy0s=NI4mUY;M@UYuBSI&6s6eMBiN%AKerwuv>Y$}6a-j*7xv z>WmeEU&?@ML_{WzFAUncUtOHy1`6N4kNd4(vNf#2aeJ?u@4Bbl-qi0gbwq#XLxC?7 zZ{Wjn3;b^QSdl5USLZdA!mlKWD26(dsstX_v~sE4xyEqw@nY2l%sY2vkQS$^Tr-MO zH7gv@lFUYaKhI)gs)q}vO2o2C_O2hYMzWa>Ng#*q?Chjxk{8+ZWnQb|G8qKiP$6vJ z_@;QiFSL_=tmrVz=5gU9lr>1DoU73qv1c1w>5_|j>lTHFhX==GfP{Z=u(BUEW%3`( zY2Nf$l1~rH{0?Rw2ynA`oXny^MDkz@)ipMLuC0BI_LxdgK;Xvl*5o`qLL8541Rj&Y zP8}-o2u+G&meP`T{nlhfGi*_r!zTXbc**iWhCDXC)&-u0?9k~}#r8;{A*T0jbb*v! z7255dA2(CQ!~fRR&1S5L<;4A2^PGA=c!jM>&lTJWYah*Q{OBes#!tbok5sj)AAH@| z+&n#*55{BEy)|B9nQlGb#4-8e`KOl0FKVBEa-uVvDD5SfaJs%cK0WA{wzjn`Y;(N2 zaCX?9iZ|gP6%I(UUG7@ye*XOVK#qFk+ekV_y{7xsr%SOLMjk;=SxgjismSv}FTc6! zFXu;d+EexR_687eyc%D7Kq7m6b?JOO=?FLU9dU%_81GuaNxMhveh|V@%4j{;ps*iK ztsG>t*p^zMVQFbOcowLiVgX_q7D?GUSjNnwnl2b`fk%*I?FmL^Ji8NS0g)d`9c7c6Y5kay{_8 zIwhwH_k$oE$v*B&eS2{{?WvHZ6v%16HbO(rbJ!ija_?EEOQv$36oh7f5hlHMb;pOB zs69r^h{q-o^7|R%*cHcNRHSxxv}syowmnr9$L;*RE1pX}={2(LS%H50ox6A6HGFx9 zd;hk>=2&Bq=`e+2CN?@gv#7ns$FTLyiL%i3;k^0fWs&)&Kx+pFsiOEuI?XvaTJWa( ztojZbAqNS1(*08b3C*qgM{!>Q0v39HyqPGoMa9I#YzwCrS?x=u9dhrBdbp~}$Kq0z zpjv1k`$GT)ZrXxLTDYydgRyVb&TP6m9uAOPvfy2OW+R=hbP6>1atVI_1y{+pFYX_l6h`2l>OD$*SB-;cw@AFFiV-Xt%RjXL{RXS z{}cK&*d1oesko%1XON4<+}yZ7eE1MXquLjDJX`OJxZ9l_E5y2oQz>SzIXF0^L5$I; z{PudvX7Q7=DeQqjrHsuHIv#|o%W_w|JW^w0qmc8-&bzcWQ37wh#kO#z_N$Y5OcWWl z5({&w_8>;~2s+KoFk1B?ll$LpprBE}s+W{?!8$(XGb>R#t{33;B=HSPuM>WgS;4GEqd4fl$ssJgvs?&TPH; zmT%a)W~miFrFM0t5Z65%nqQETJ2KHZIb$_gG~W4|xVo^KAO7V^W+Z_$PriETa?)3fS`Ws}}LJS-ZE zh@e%cI^LcxQFSyK9vI3|4>#^lqa>G3kXVsnw6UwKWsj ztzWV!W)}0`U%thD4LL%t{Ma9oXZKH3-v=b!TvjtkNc1%quTJ6Z`U(PtNnyE=7>@l~ z=uC%mVcV$(ay4V&catg5(xu|(;LzKx4+YAnOQ~M`WwTw9&~iUiMDlw=FI8kRD3di9 z$8P-*{VD4Me0;z3bn<#%OvA&q!HF8r#CP>R(ap8L52;OTAj9f*#<0L!`NA1crSbpz z_1$QJKH{K2n53%t6c*aX90n6`e6P8_iiHd=Q{&<8e^~$bFVs?$!A!*zi^+21t$GNE zsAWNkQ4diL*U4*NB1ZO==4pXN`kD;yEgM+U_~ z3$I+HHl}M59jBa8y&$E@6q~6gr=)P$F5Mq5HlK%rdF$4#pK8IJf|-#Y9UGA=k-DO7KXfGZQ8ecamOn5IZMlfpI?>+)bmhwtFMg zWiv%3IEZirpA&N0)pm6SO;$MC9nj#4{Bq8u$>FJyI3 zEN5mh8F)rP@dbd<&VDyvf1#nYmdB~Sf59Wy?#+>W@8c;~`p!$J_uW--fq8eHH}uD{ zTPNS3c;j>BjjNp%G1u9 zsC4e9a&HTxL`!l5EU@&)7n3-~6a8u_)-v0GT(!g^{t-UD$(2T)77vPyuAZLxkomgu z2;}O<_V%~C2L}ps^YgTO-Eel1!-9FwC^~KWR`LNx(We)xac!y#t(;moVa*vq;xd%ODQL2kiOUNy8~trAeEQw>?3hR4-)hlaI1dnQPG@o2D##LXJLzaSEVeyv?Cq6BU=LL#IPbJB z;h#Q#j?SBv<~?oDB~(9W;^b6AJn)ShH^K@HyVVwg0eO`>Y-S?0#ju!WuSspj#PGNl zzj8aVT7;unsAlk1x4QzsBbDRU1Y^3dj}H}89W5tXh}}a&Xngzn}sSkgoNh3 z_1h)W9`Z=35Dqw-=$M$wh{9QT(LhAtwKu4&tx9vgQ;*r%KIObj*r>x06?q9?q9s+H zt!1&63{FmJ?(Xf04|^c+X*>e{EjSCOJbX4d${`#~)Cq}(wvS3N^F3plKr-ZnNvh7N zsRT;-G?hc$78CQaKm4-TLBxDr0JOgb1T+{iLpno(IAC3#M4Ull-k;9%9dPz`T^%>P6;fK{q^n3SG z0PsUW_T_OsN}}*{ev0;l?tU#G>CSnim*Sf-`0Q5C013P4R<-7+mtZh-&V;$S;Y&+P zr%A<&L!!?aQY|*a(;2&hxcAqmqoc#QDPuYEyUxIMa&j{Il|CAz^tVOkaHQtugZYX@ z*Qaavu*f88$H!yuVv(uDQ6maQM--z*g+r%-Gu*O9nGH(3lp#y5AHWtP1io(Be^e7k z&a3m2`F+a+U0q!yl^PG85{pUkqU;7^T}7(LU0q#abeb`cUuUU&ycgrt-M!pp?-3;-z0!s*D~M& zB&10E?fuf130YpJADV_tF(Eo2qw7=DhJ@JCFm1ZWaLTo{aP^XJd1srpAT;;UlE-fM$d^YEJRgpl*K(A>^U#jG(JK3S*?kn3@b zzc#11o$e{|qromRO zHY{A+mH<2^nG6NQiCm>3WgH0`lfaA~Rc<{lqb)X26b&XXdbz7?(K2S<_D>b(l_aFp zUx)*~k*!$~0b!kQG0BNR#Ep2HFM)xnBEdu#r^|_@rKLaBhWm39N3^_Pk^B>|@hGp2 zkETNn*q!?d7Mi!kP9dfsU9WO_+`W4jiP@;f3%KXc82ZH%s&A*JRH+T7%}T*O4d!B$ zq{oBgnz0$-i4@eqk6$Y&VC!&|l6`tWkB?w!dzG+PvZYq<0MRridS11Dc}QAN)dkcE zk_{dP0lSWEZAAm}Ev)@C{r#oRPrkm8Vu+z48}_cxLqfQKV~%r*m41`kZbhbiEOu}B zZvZYF$L*?PB7Yp(xtx+I-K~i-c6Rn2Na%KvAqi(t7+RrjM)A6H!fxvU+xY`)(1}cm z_0l0jnqx&jc_8i7-pWe!PqZm|Ni|8VYy0stxAw`zz(@1f;a{rZpK)>fvzr-o5(H&dsk?!iXphsrKj>xD(=U%#6=_ zx`2g*s4WpRs^2&5$%3qRCmp9p-PnovJh=1o^8w$?t*%BJIDf5iilCHln)@Lvi`^7N zXc}H4x!Cr+e(Ojm8N1w;k@LtFn1?52p30Q{R z2}SJFbqDS4b8k%4&R30xQJq#(BRllcK zyNyp)4qa@^lPTQJR@0?xkXNw@2qZ*B??CGFG_Xf|E@(l<19C+7XCR=3ro(hNG%ByV z6L?OK#x0GfDmgx*Jr>9t-k+(vS!_Ao_2Z4Nm4ic5bMtb>8Zb#>sNp#;^DOd=xB$ihZN;@x)`}il?Qy`mjYGw)tMQD*10?1tXgF97_QJTIQZBLWQ79feS_QR_<{&>U}yIe;#leC z#i$&p4hX<~cDUXt|H9z?@0x%CKAQzmm*cHYQ0ORRlW#%ElX4_%^nbD(uBIKsPbsc^yKu^dToFbUhpM0_UXfi z5Anu^vK0#6i$T!0L5f7|9~|Z+=a><|EwLq(JUs7TxdiD8I}C&**y;iRz5r7C$9y~j z?jJyEeU zA>A|^FAlu-giZ#Ki=8AEm)QtI_4So2Pzph)`KuFUOlmn1_Ul8|8zYRd^Jn*oxT7-U z(-Fr5p!gfWRtN^b&(9BUgB~v4cy)PB+ooM&K?uQ}1{(!x4XfEmEyNB;qAq03)YdTv z08R8=HQ_*$bI+@)gStEp6h(io;VWPlU+40wuxltP;v9}!af9Vw`y0p-Z!^#;=V>WI zaWg0+Vq;^|2W<(@zg#+mz|vu%HCJ2wspy#uV6U;e8`!~i^*Nn87hN<| zxKxNorSlZvJWG_?@=_W2goHVuSJa!tLn)@T1Z58^{i~#;T7SdXclPiVsBtuXZv+Iw z5QMlW?~N};+wXVv*tNQ7^W^{KpMQe>bX6V1VHFy5%1um6m}&X5m=67k>KavY#{;1X z)H^?TB01QgbQ5hQqLc=6Ne_C_ff7q?l;5SqWD&45Q~>{blhLK5q}m{2P1Kg%rVgKhJ_j89vfNIHFfx;;giM*P3@7(zv z)1)Yiz1S4^lr@j-H6EwkFQF+25o*X|g)J$ACgFV77xDl=mBqs--i7d4$BY0kR2zIs zKoGa)E-Q;2+pGn@4&_M>X%SB7>J+`eon7uwA_9Q-KH{l!m&enF3oRjt7)uce27=}n z8j1yn46zXzLB6m(_kguhYE?RwuTyJPvO`sS2PAxBsw#G7Mkg*lK11fm4Is{<$Mm6J z5&41BZlxAz(j9d4RHxlJJCI#JqTE9;nXEbq7JPKXL9wjdy^Rn(;Gs$@D&$keAMfXV z{hFze@fV7bYPA~&)R-5~pMQYk5F6=txw-746zB`M z9&hObdjiGkJ{-#!R&#MEcuV*++7q#UKUfSHYkbK_4RM=Z#@@QMj{=Xd;JOR@iHzm>Wihs*860AT)oy$^aE zmt#;*j|>F#YaN}WLHhCHwf2^?3T72c_lZV{U`l zQ1;_EY<-}YVRO8t3A}M-Fe}p83|-y+C$|ScTgbCax(#=Ray3m8e*6>-A^GPF1O)%d z%>eK)1K7$|IBI~|WZ(;Em<3j|0zg;%N=n$^+@wOz*QjzCT%u8dnm7vrE<#O42oztw z+<|t00O-3p22O}l0%`m`L5V!PRABQy{97Ko!?g##+E5$}$ujR7(QYwx;WX+85ZyzXwP$fP!W>E;I zgg}*h<$7esdv&4*F}ZG*`f$%3FAT3Z-n^y1Ke-?V1EEa;ce(jSc`+olIK# zzCaODPiXokCnlo!JUxnUv*l{n zAX)-=+)lr5u|a0XTdhO14@DZAo0&`oCh#`b-^|!vBQswtE zp!cmkPPd%D#$j=gS|3FqutR$6cSDV8w`kCAtvAP5u!VjAX|!AId)a+Bg^&fvzaT{pGLy;I z6++@YTOma-&H9}DpTp~4{rvuRGUDevLR-7wnEa_ht?@+Yk8W^Ed^1y4zY)Wur-Z(t zS)?X`mHZLoExWCg3nvR6HUTzl9K2<*1c?)pLDEBh{mpYG%Ras-$`Rql9vC*ZZ?fFW zn`eNy?`)P#$0VK$Zas&Ke>+hVkLR>s0lfgDr0Kg7=0hu5cUvzPa2G9X-2Qh1*^hJl|JK)q z-snPBRZ2VMXlcL4#Z^H=Deptt;{KwN;{R=Z|G$?0TQ32KIKT~{%aGi^DucZkJz&a= zFD5GbjPn(8(cBHHd8Q0DdOs%4kR&4pqvQnDct2VGJf;VaukVDb9d@jB-cz7@jPJ_Q zMI=>V33u>?TY!!weEvKoj z4w;XyMq&TUgyanZj@Nm^(7z)@C=Wd*MWFI3WNy^pWhziXS5P)%1Bv{#MQujDa)plX zS~NkVpdUVV;76!(cuKMBlxWb(I_Tw0whOk}w-2GmgwPYDOUHej{E_HK1GQ0+gAsGg z^3i#qn{T_CpBjH^+ZfWDKHG1bH-A!~K}~>`+#_OQ=@1g37cC*EVqT=Ez!I^ru--rp zgl2>LnsXG>Un6E+2Hc)shrfECr@vma^CRLhVb08s7N$bi$C4?sO;jEJFKTyhl-e$14CiXPlG!&l zHQFvsJbq+XZYZa1pkEw+yv;K+6gWOQKovr?M<{kDL@O)u>P@7_-S@pZ}m%%KhLW|9_I18Wu zD=cE2Sd%qVD-D1Y=x0fpzz`ftJI;3S+KDl~0{cl3j*e<$>H|X8gd$ zv-M%ig_DfJ(z3GN_3$n3fB-W7^!m}(5U-Eb4dgEv?APpIl_=EH?rUcO{)s-dB2G+b@Yc~lI` z;iWn5u;1vboh+M`@A#WM$qFf6YgP1|ZnMJ~r~5Q^uziT(EJd~>o-sIf*i!Dd4F8nN zwWq=_jk-9?m6z&IDcjl;S-;2pE&Ey#7bPQ8@4IZ7)k~E`=^VnE8{Y|J)3lK zOm*H=VjU(voQ<_x86)!ID%oPVjEpAWvm=1kQVLF>)8*zYJ0}P4A@3uB1wv2yt5{!5 zQb}TLl$7KorHt39$(rZI(E3qXzPYjS2b%`>bJ$|k-*LAG}WN1XM4N~bbQ%#iG2d}&x=CGc7LF|5U7m26bFw0?{c2!AvuVbFGQf@nnIqGs?BY zB$nxP@m5w=zp21HB)F!PDH|@vC*ANek;%n7A+%X)cUGMkiwKAR`iDF69rU|;*SVJg z_kw3OF{l5wH^-{>UnMJ7RBv!sqaQaTZ${;jHHqu(CMpCZd-xpbrY zYWJ<}S@|o92jh2!kcv2$Sv&*0^nCB#3!rxbpc{eO zSABV;ZhO_udmaFIxZH6YBg(}X(R~F{?kFuCk{3w0dreO*8K6aI+e1R?ocP3Xd#Z7~-5bf%^0j}KKNCRL zisO&FyGO6HdUetVbtBEt)Rhel=jI;WLBn7}W#5`W|2y1VPatY#Z%xK|_Hes$A0r`w zNTb57$*lv_!TIWwnDDG274WwKYeOEXcBc-+zjb3~eZF;*3-iRX-2gO!&PM-8=*yDu zr$e{9Yr<#Tk!WUfQ_8C*uMC~Cq)~~>o|A{a?j6o%sq;B+E}axMy@_8jRD1%0&AT2C z0wPP&8^fx%tJ#(^g1r%afr-5`*>W?=32u#EBNl}WrDze^-i|BxAR(E*It&bYy~MY$ zSto`F5X?w={B~#$u7iPvU>O`i(+JAX`_9evaXmy#{bSeO!#gsdXY&ix5~WcT1m@SnmflHR|&o>#L6dW{NC;bZ8YPkWZ! zr>g;F$qk76AEVxn0F-0;*9S7*fVy|G7^w|ak)R#1{hYuakf1Y|FY0JxlPnSaYH1O# z^PIb9{cy#Ob8peMNVGe#JDkhW8$|K?&OsFKUQsJ6+>lEThm(7r9%`}c~n zc=&VD3lCK9sCTK*$scqqMJQfHcf&uq_ zwnl+p{J7(zgJ)4ErB2BtDR=JOEoxbwUpL3F@S=>Gjpz- zx2YSFJG_p^ZFC3a;@hAKRN;#nJK!>^sS7{1K^T4#-Nnk`u_1Q zB|s(xszIiBbyZpH0AbSG-m3;bl(HD9yxN$9zKWZ>EY|BV=nBrZ4XUpkxS*Xl&+AzF zs^u(s$WY|x@8Zh#JPimQmVfucm$Rbxd;81uI?cJ0 z%w{UYC?mz@Pl0hAY!1f;^ASdxsn>p?6vs%C%jneCD)WdBO_#zVB5KWv9WJvwMFYJ^ z|KdsF{;aQ972RJ_5?me^W{`f9Jub_gTwH8+W)LcA9Z0+*#d`F;y}>=}=+GW~7+FDg z^5h`>Y96WiIJCOCaWK4ia}3F9W#+lZl`&dm9?IOdC7y)yeqafrDRnl+fGzZ#n>+5g zw@-oIIR-w9#ix{%o7626jwgN#BL#e3?T!xD`_&r=5c z01mklQ3QLl?0Yg@Xf|szb^4xpkt(clurr+YY4Ogba%g9>BqKg#H>GA-+1g)hQT^mW zu?>Bn+wt~WjoQMf^$KZH>b@lFTy4zEygr`NjAqm0b-#V2yA|LZMyVVVV{f7|;CQ}7 zc-x*>U>+y%L}%h0r+rRWSGC54^xTo~x6w2lfHL=ar3BvV1^l+jy}4c405rcCxJu+Q zj|Yp8?4#>TcE)hG)_8W08weFWtt@!zY?|WgcpB3 zOlB=vw@^@K*Vg`)7p<@ICGquNopdG#0I2M#k>-PDhBI3l2U!(5TE@+$^(Xevm*D^!?Gs=|Gk`rZ@jPz=P1Sxj}5% zzfRuN%qHO7J8-*5IMhsCA}_m=(~jaHPcj56#$sDU?NnD#zP4NavR-q4uBJLw7#ljx zaNhL;NG0uEgx9j0shPNsNz0&Wi>;q07+1cLiN{f^_(Tx;#q=6W2mb-)Z_=BXkN@f!S4^{@_ zj|j)2gJng(t!}@cV<9At_3HhlqCM{?pdnGo;eMHken8aYe?&MQTR1{>udyJTZn9!$ zc-vN`#)cG#Tm8zhh$>BRK)`#M;5uw7jW6`ocp`QUr2YnB6pA1B!l`;Je;=AMoKEg9 zwp3AOwXlR$EH=)k4CBc#i%mx+aG`zj@!4zM?x$^NC;;bXzKMM#O{dx3T;9OXj;&tPvgWS z`U!28jEJm%YM`+EUPMf#wsBQ#_Zb8v1| z9uBDkkDFaw)Gv~!P%JHo7t1}!z+<3D5$8% zWRRVGj1S{>2ErfIJUo6Nm8$kxJXHWCIQYlvJz?l;{8G=O+h6LMzr>tkvIN!H)*AYR zj)_`E%qBs^TwM{J>QU^*H}u=WSeL0a)3CjL@6UBd7oM!G=gF83j5xW=>x^xyanV{3 z(REjDHE4HgOltNR<>YCH6e85RY7gEx4tE?^L$0{Ec=q4R&)(uh2^TuGoC6aJKgs_#~dj*tzll@8m|d%Tt?Z)8S79k z6W7mF$cZy>X3cm;N7q$}{uLP`p03U&R(_#>{cxhx`X)KK$OO+`dby_@l}71qvBWNiEKONa=6}C ztohVHDzSCvVV=I2W-&lMFc<6`9Q+xZ`f{hgBT!3AtUUltP)N97V|x+Dnd?8J5uJ&8 zdU_nreFtc6eIE888rLrsD$VU{0fp>b{0buA(a{kc8zg9|b+=I1OxfbiP^b?vzaD{3 zcFq)Kh07roIF#BIwA9_)-E(e`M$&8Fmf(3UCuMjmZTQXiYrZnu<79R#+72q7!bjeO zWY&}sWFso1&Iv|R@ztM_DMpNo7P^I zb^*l4c%jvo-(hY9;*}hLos@2sGF_HsdX2xG zoq<7wV=?PY^6#XmSGN7NEN3Q8vqnaZquDHZYcBTg>9@h?ur72RWcdp=clmRatY`b|VF|3QcN>}-l6#GMxVLDN#i#E4#uYSa>8ZQ_bdSpQ zcq3Ko(ghCv{_)&PyVWr~XsRFu5pdk7y*$S}F&)xg+Okahb1o<(WDO#yUnGTrtryih zG@p!tOoDTp^r2q>QJk0B7yX4H8+zr5`)AHlLygnWPVl*;|~F>80*4XeWqmZNQ6 zuf!T+o^e%Pf|pWQZe6SH>z#6Znq*K`~>yjF$|8 znbXE3#Jmebo$AtbH`@=vY1FTQn!PSz9uMwv?m@>~F9GBWPTF|y0I5QP_jaRw`Ujhn zm|R@0Q*P^?#zTtvHZT1P+}+$j?^O^F*;-xw3Tlnx_5@QpF^3zr7xc4K-BAO-eq96$ z7LGo*q-ODpq10ZJ?7f$_?}1KC6Y;WvJj(kC_%`%Gq=X*M2NtCyBoE1d~tmLTl*hOqJuw()!O~A7}y9X%{ZfF0Ttk%L!4=nBrgjkyuT}(a(^P z{q2Pb`jD7t&+Lp&aA6ixwTIYz;mPTF>RHhHlC4RnaohvA$el%chZ>u+wRc~b#wrcqKL3er^J-Ez{iUfK51X2{SWZ1oawNMBL z5xwi`ch!+rb2Kt8+G?Yv-Vt-)%azM{RUx%Cv$a`)#M-_#k%D!IKUhdbZGAiZs{Ty% zXC#gM_fAySF|!Ko0&VV!!!7ZWDa|s+Wg=(^fBcNbyX#-@;BWE8)+u<@cd+(PaCQn$ zHda=mmN<)l7E5^?lOGRtNlm)iEqjtqzu$^0TSc=fEmJ$f7u|TcWs<`7K`HAcf8>Fl z!CVc!X5|;M#(L-7M~#$P$)$~NuKNe{{UPuyEG$0xs$PFnFZ0Vl?F52s-1GYb$7FYR zH}{)1Fr{>Xt2;qOBw1O@`7=1Bcfeh&3Tn(>qG3zZ3!z;%C%T&i;KG^>$*YUca z$-rO*#3_Opf%aqoKZtHSrwQ1U+0AGlaXUgj&+~bt^4A~dG0io_3k>E=*>%|zQqpv< zcqRWF+%#ix)&83LdQybZzDDZ1Mj(FL6a$;3sz?tZk%$N?`i4jeQJQwA79nQ~Nidp~ zIO{zz8~maeFnH-QF*YJj6`{6NX!e zG6qf19^YlSz7bFF$-B2Gjke|L;ySl#&O__AYLhBQNxd3b8Kz(6TCRT3gVgWtuy$M{ zS0lzlrIewQ-_hG!ryDc!=LGd6Jsz3{nN*5dGN~rvf0)STm1~2mZ<ZcS7mr<->;OX?%Pc{lI-`*iVeDx1^3$5{svIcnfGYyIea50`hYh-S6X4CXd$ zF9INI+!$gq@>>>3I4hflKJE+PFJPPf=rz*77|v$2&bU zbvwDj&3qZ_N5zB2x0UO#b>VEQW@biHBN1;9k_T3^Flyz9uhWub+}snoql@igY?|8I z#DD8)BGdJEX85^X4!#gqpMNk{FJUz5dF>x-H%}Hwsa#Dk?ZNZ`oRq3Lo@WEA_Xo+L zk$XuN5HGk{KyQ21fzsq@=;~z!o1uh3%-I2+Ij1s1D?&|g~U59V`SizdS)$h2;maLT(; zKYhFPzQDG}nLFa$$`rUV-_+SG^TAR=rIg)r5Z%vIbP3J01hW~_%hleJr=R^FH%(2! z%p~b2rw^Y#`@Fgg#`7{}=rC2zV>CP>pi;=LWy7lOIy&Rw=2po!;i%GDl{}0z=_#`% z=dP+mEYY?3X>bCM_78zZG~*`+*e=Jboeg(GM@rw3#&fD4m2UNyV44)gFs4lF11I@o zIvpW^{P|1Z*LgtM*=j|82w&Q^<)|$kGxNX&zQ0!9GI$C)Vp#ma-S*13??o0X7!CPg zZjFGi#_;;hZztA01+SF%t>2f7S^C=nsBzcoDJ?xN4yRH5_;y*k!1J2_h?u=4JZo4c zOYuH#kFOGKtbDSdwJkW4-c%IlWT-8+k}78nn)uYPhMgU1dFb+jwXB?lTIG+^-R9s# z!MtH6!y!93hOG%z=NYn>FKwq_EYjv^;Gt5sMrXbr`JH?B(wC_xS0*yNIl8|9$C=mm zdUGCZ1wK9Q(D+5viCp24P93IM-q)*={S=2mt|!9?-l4JwAsd=ol6 zVN+tEfn2_v&_Aw;g0GcwB%fNut6UBx4$pPk-1zJ2fJ)zBa7R6&m+?|9H0&+bjI|4# zW!flucdJ^XnaaK8Hw=TCUJ^R*^wizDt+s^e#__H2N!L#InkT#U#S6e1sbFfTQk@CUFsyhuP4d>MS-$2jZ0v=@h)Pf6Hu_!La4) zoH_nS0tQHds*w#YCb}O*J5PiF;ehGi#<@m+#84D45m?;aF0SpK-|yb|9}@Angsov- zAZ%9JZXu2?Jegqw!V=ig#V&^?oQ_)^pxvwtw4Tku5>a^#X47{6?GLEiP41<1Ir!E4 zOODu)G%JF?R>0h<6=68HGL$*XMSl{G6A)HVFphi%0*-KFqn^;O7Gw zSM}#4jM+g)M;;uTwpKSK3VSvtD6gix|e=@E?e{MQomY6Z|Em}i>iYLb$Y43(~7+@#EE_J;e} zLEYdWOXmf`PK_A%gUKNnRc+J-Pg-Xbt2sV2$aKNej+nyDQD+0E)%=EuNAWl1e{b!} zeV6r|lI%Y4cDI9MmLPVCm|FH4T&FBX7|fv8=mYtI@!j8B{`$GOxiHPscu@w%D#RQl zjMbA#MAatodsD!u%n&PYtmyy#fUVpf; znp+Pp6{L8nV+C{fi1+s~{u5k+5&a`sz3cX$ZuusQDr1D8QuqmgZhzam?!qcj9rF0S zCr_S0n8%tR3}HQ)iqt@RM~ls4J3j~|z6aMG5}2shGsezn#4!JTOWPwW67l0NJHHvA zQvz;y?mR*c?3b%WKFjt`6JPdME2JptPj2t}CBNVj9zvzeUj|n}ZENdY@X*j?aWRpQ zl2U?~8WdiA7+1h)?@HwBhOQ(oD4T$6S%>*NuefQ|{?xX&`$L3dJ$UfVL=#vNDTKZq z{OJjpKYjoh2%O@l;7ePD^z^U2i=H)H?Q?A2J;qsU+(!p1fzIi%uELbUC{c~N`~F-7;tUF>#NJ9 zZeUwX$6--09R|tjMy(MuI8?>ozyGO-w~D7>2aaTzT|szoV8?SJ@?e|_b{nau%GD7p zwIU$<3aO~z!Z1J>m7-r>%~cEx`@i?lv-SG-u15t@m1gI{?J+aT!vB}yo-nI{MU1@L ze@JL{c18+$dBBzMeb|2uTJ-cVa9)r-5uo;4hd)zK?ZtK=Lj42WXkJaVkLU*@MlTl^ z!%cV)_;DKif0^yz>FvCx(O}F>zvJUv{rcbng%RVJAH~Rk!wsF{Cj@WfJyy--pU1IV?(KuFC3A3nW@h7qsfRYe`qqL)YTCU4G&9lZt2gWp8|*CW z1I?Y2!EQt-xcghaN|?vj*{ULkYWCJPhkqxto-I-<8jfu<(cHKTU2#itH<6GzeQVp} z`npkXpFs5j!*1T#Ln3yoFSVS3JBK^VlZ@Lu*0E3MR7B&|K9q8n&CajBovd{+Q>p1q z-V(zilg)IZtemK_w(SxrZ0a1TFw7I;u)n#^XjJ*oZl4db>7d&Bw{-DUf%l} zh_Sd#OwM?kt{uf?`GKSRbpOf9PzvVe*dH=@w_Z%|6T`(vQi&YxU`_2jy*eJ^3V3>r zt{rgsX~ma+qRja&Y-J3yW&PD9D(za8=QXmnC)ZaY|79z-6C6MpZ|z-cPS>9LU=RtX zOA(g2Z{z%U`26B*TtO*MO}~q2EvMwv=0e$x^@!;&k&n&hl47O1o6%_HW7H$!-xEvc zm{H8ukM7~n&+xan!?cHph6Vv`Y>DpVm_3aObE0qWLybxs z`LY}$E|xW2)AAeSz5X&=LqIG-Fxs9SL4LKS1+)6=;cRYqj-~ah_o8I(@J1etrgO8C zOA`M~N}@oc3jc_P;`En$(&O=u!ALxdzi(M?fNQ0>=`f47Z+p5%y7$o{9``T4nrl`l zNyo}(HfslXba6Zwugm-fOP%*|hUpUPfRxAO32(ZTmX{Xjjn^R2sFX%|-1MlKZV9}a7G_vQM)G)O3pBjuNTYlC@)bl~|6>wDR@v*oIHa?&4cK6K4p?tDpR?ZdwJ z#O^JZEzaAR8(oTNUhSX%AB?82B(c_`O<}rW-C25pAexM|q|6h|Vxy~k?>j0N!)K<1 z2YoF{>Lr`cz)BxM8}pEmFff^7$(qGvIJ0i=an>CS3>X7K^`Z)=R=vfB)D}tI-lgf< zk(j0IdKY}{i5}i~mq*t|V?Q5OS6Ab#Z%h3Z+ude z)N2UCA?FhD1~8+tgJPbu2l??49-jUcIFUK7G;3n+hI;7qe47O&RX#%w#ixcdj7=Cr zBvtO=!`%l+K&`X(pw#xP+99YX^mdSCdZzg_$6=20Hk5OZj3u$maTj$rH}{;iBYGoE zM{r!g*hd(JTAu;oDRJJvS^Cvos|8kbGH!Rg<>C6nNttZku)+qMIf>RCe6$ z^3ND`@@+;*fCYRvxN1;l?(6$wtOvse67c79pb!3TjF`#Eb^sTA*LbXU4|sW}kS3~J zg8`wdWCP-LkBf`5vbSeRH!fNoC;$2(5C%Wx4^uN)_g~~yi2IdqPgWVGSgg|p!WNyt z6yW7hkzLW-|K%ycS1l|1zckd1E&~;<=2fk2?MdmnT+W7ZI%8o9;_t<0fAieP=vkfb zd59_ODi?aD7&cSgLJ={w7*;0p?glu$|p1Vxb$>6EhQE+wQ(TDn0&r36H}kra{c z2I-P+2|+-*yJp?SJMYZ#9rJVeW5RRC-fOQsFO-Ic#u4YX$ZUsZy?2m><@{?g|8VE40K zOvwX4Bz{ zH&4B)S_>xJIu#G!{5iCz9I3KDc{4zB8Rf?0gG;exJGY=L_vRvIM4Iolu4*RzwvZob zbPDs`8SzH9gLN8^t<~#(+VD(RfQ#$dvzW|QJ3A{X_mgB68sVxFQg+1_#)CXM-h@v_l>JD~Fpm>n{{e)inq_6SpK z+-)`r&zsrZbO$uj1ku<2wmg68<;Q686HfIZN{6oX!xolSq;39h*Sd-X{Cy;K_NRZV z1XwA-|A~}fNQ3V`UOHsA{=ffIWP#x>MVak9&^{!kxu|6>Ubv8`dnk=$%zhKHkfW}- zu7GC@wSKg}mCwZO@{<+?!322r6_u5r!skIx;@vyt7i*-yfpT;{h@x@#cECl?;pWDBs{(gM`v#;pNkt;4k*uj=L@5IBhR| zgCR+yo(dc&Dc|cabBo%z14oh8YmaU$Obb59nVVD%ck^_!ebW$VX4_3mH&@ z&9F?5@>@cQj*B=<(^t0UJq!eKI;=XjAVwG*k8>I~0HMGsdPu>tF6wt!`|st);c zrHTEHMZ(|hTMD!GL3d#aAKWac^oQBZe-A1iCvO3tL#5&cuKTyBx`Z2b@g>sPas0|f zdT6$qxIY&yXfBhJn}$NLa`;8P_!U#5n+J?lv7N*m*K=|heSiPPFAVzH8G6DQ)L%)b zC25R>_g1>POCEX(`;$dFv+;qc&z8;@xNFp3MtzLAWjNMo3)`9HN3=zUBQzpDe({3Y zP&z(W+5WI-*=DIP=r`JG%&@Q>up3aQjqEk<&k}-@GvcIaYiW%3&)HdT4U;ufkqhIQ z7dTZRg3*&!1~=0@%&vIxiMIS$ATFqtVaB_W>ESbMw}% z5ZgsA)#OtSXNqgQ1$Di>H`PbiV15{yd&EZV%re6|;<@5?%y_lfNmWX9sL>B_wAN~B zmqtq4VhrJ}VJ`{3uc|y;?-VT3Yyl(9pjzUs{pf9Xp*5Fqjj@z(b)IKdQXC3J2uyBH9nJUo7D?7BerBc?a_P8v4ZXbel|jKdz;y% zTMNV2psXs4P1y(;8kHg!{m=SDkY0R?v%u#3&Bb~IQjEj-_LQEOI}-MX1nB2sp0hyJ zZhndy$?yGKrOCHl2hW>@dRK!zWrtb{J zx^)Xl$3qiy^N!hQN1JpUw-`jSQc;|_^wrW@W^jCS=bYY89sN7e<6zlxJ*l)E?w^O! zPM~*Dq+aO7{ttcBY<#p#`ccXYdz<3sB=D*U@1rTEhbeu_XQ%9Qq3JaflZs?bK`XJ^ zULe?1uW;~baA!)AjO>99sbL++MDUil>|gMk?ZilcnfwCtevhkchP~MqtBP&p>^V%? z{tm$~0}82^J(C^2qvMN^!nl~)d2~m8xf&X-Q4PLVBsebM;%+gV^qO$5cl5agbA-;2m$vWcY?8aZ-^n1G zw}nUO5Mftk*iu_d@v?-w?dfHB@XyDJ2vCA zwC{_`SPZ9Cy-C4ppFfnDB0Xv8Y*z=~oK|wYJ$LbRHj_())&At+!S1q|(n;{^*Kf~E z{s=m_bJz7?yKs+u&+2-wTWq7po6WmDnpuae`6n^@C#K9K$^|bjXQ~#nV{+kdFHZzw zO-So)rgm(t1QXV`n5rAi_od(L{~Wzz;ov|LeKJ;WQGFsD`Z;udI3tZm6ZRIsLojIq z;uBQ2jZG(tfTl)<)|H@*j*Wp{ZomA|Hu}lZP!+nNp`r0$x|vdZ)AZBV4<9}38592d zU_@`B1w2cQF)U3}6?DYIolJLrq36}r#f++`;Je$iytL$Sn8}4IIh{nSK04qv?2Mi3 zP7)#Bhn^AT+zrYzXD&pcQ)xw&5h3P3lD&JU{SLefo7I8HVe2`Cs%I*YifT*1u82D` z`D^lg#2Hq|HgRqOBe8EPe{Nb!v^!)zjskJjAuR|-0}B`THjo=&AhWkr_sZtdE#Jfl zOp!|KL+P--(6IUk!UWq3V}214${J*xF&H>xvXH4kMX{TGg)xB~vr#+o5QbWq;dGc& z?_FXYA0Jozr2J}mw5T^x-`Z|))MpP5c9YuZ>sUFrX(Q)H;%A3(#!TyjF^(?B{oLS7n+^y*zb}YH^O%(G!dkH- zAx!bP3G!&CXHY;VOdavNsXs;^O7}k5C!m&O`+R6L89mn(2l4`DzISH~4L3dA`^7^^ zj>Pd4aRVlQK4+X;?zT`UlT5ei4^?&2xlYT5n3PtcJHyo$g6idwjq>SG05dnwTH3Q` zOSwqY_?%s}LZ=5z55kJ*z9|AtS_S(~7w$Jt$B~`8u%}<Eei=87Q(Tp%+ zW@c`CC;4fsouM6X4?T@`3n_NmH{HRC2cNf-pA;LPYU49L`ll$xr%oc+<# ze!8=IwkuCeH%ciZV59-dKjUIq*ZFtdEFnx!V!%C_jptbW)mL!!tv4-pZWvoeA22tk zgf3W5)zhUOAFyHVKDwVR(!D4(O0<1`I!XB*Bktsa2R@Pi1}$-@9~-=>&B;WX(@|+L zO3l8NR5V$lq|4OzFi_K(<+S4(FFo)H{61)1(4)okkRvCTr4atOv2ov#M8DO2SHL&p z4wO+L_2BEwW!{r$%z^5AbHT%ppw;sdGpXE+^&>gL5D_2Su?Oor4}JW+hPXqZC&TGE zw{Rs|sEC{5{`dA#iweqj-TRm6xt=P#KPO9-B&VY*N4+@w@wG_8WB!LTvmPwBuyu5N z&R;llpNPzBz0pi@WxxAVf1hhQsWUEl!l!H|qSlbc(eB|28B)dcR?6NJt(hrm6#HjY zBDjH1-#4QdF}dB2&Y}wGmA*O%o)7i&Yp`n2y6+K^OrDn)u!P4lTU>qXD$-?V;icYtkXk^r*qwcdXZ!KYdnU^I25oLjVjHJEp^Hfy z+mEYtJWIz$1k}5jo*Ct87W$tNi>M|)&O#ru>xl1`>CzZR}7quri`!PDV zt0L}R&b`KCkzj$r&%W#L=btf~4*8a;@V`4<7E>Lky|jh#&KXywKQlU?u%m#71XVZH z|3p8{H2<^C)eyET9r4Q;yAO8r9;s+^Pht4M zjUijpP#XaXme@hxMCJJ2kJ`TmEgG^UOqNDCWatkkmPw>zx#F9!+6ID{wLdrP8}iv- zp5DJRYyG&`mGUimQx`^*hMXVS=9_H+AGgOxc(R__vrFvb#!t}y3gzC z5n^dd77Yp4mHLs$#EFb1@!LsB4m!0iUS?MJ6zo8SUND<3dv2`6F(#3im}}BMRDf5s zRR3a$ePpCx!OZ?^PvI}FRc3XryJRX9)xB4`1kB6L+oZ8+zEK~VR?66@!G+n7n`z0r z9$Y)@rsOsjz@^*u+aJGta!IK!{G*hvLY&^5`jiuv-&5V8TuQUKCqsqCF|E0Id}+37 zb~@Z{0}Dg*MJu&0e%^u)Y)5PSx!ecl)ontX{S`<1F4I`*wN&c^EIiGYwnE3s$9#{g z4cJB)GjnC!Doww?{n0KIt$)Y&;a3Z57LAgvJq`6in@n;(gIss{G0bRt9q!1bA+vF# zdv<}v>czHrZ8jn<9FyA0gx=hnnW`cGA8TUK zW|DN;%fwq`MTvREO60zr(2U-^ezruC5cR| z%z*jgCfCVeHM2&AZ;^yr+Xm%CuxTs}=k6~2yL4RPP<-lf?v=VA02G@I*YV3s> z{Od8jX2aWiWzzJT)vinn{9YZq#UZ=ZA#)L_strxMiOQSDRP#r!9 z-m1F@mVZr_>t^yM_b^8+vq`QIrHzTnfpm@z|ABjPlG(&pmrkxz|9-NLiCUEM8eVt! zDt>=1nYmQ-$Oz#LQ+YZc*GZu;l}aa$SYCSrC^y+G^tBl8_cwW(fO(oy;lfGk_7)JJ zpnAw6{5X;1gBv7|`e4rFLo7ahCNW_Hc>zkcocrUitywQSg+x+{UF=`nI%f|9D5&=) zYm=?|j2Shm8-FoX>kR&=V~F63+>!d=!CW+e)_cYGpwh|mc*pD;Pa67PP)C zl_(O;8F=xjNi-U&-m@@c)@1x~h!*=O45li^F`H#Zi-|u$X{xwm`Mlj({xwI@hJo>S|k!_`kGLl^v zBd96hWB=_2yy}BxuBu2*{~Q1X*}Py*lL1I^qfD%L*DG|$zM|-#0O&BjJ8A!U{dob0 zZKWl0^80e$0G#~>;0>eU4`&1f-W~4GZ4IV$Gi#5qH=b_zrDxUOfBWPjhG)acp~;N+ z_}-EA6RBw7wu9~3aa??x?~ZFPQ;!b`2*}#zw-;MhXAQsG6q8d`i#VtLv2 zrvp3Y9*a6~3z2br9qzsi?Ly|OBX3lX9nR1=J1ZRDuV&HZwwuK^9LSUynNXGLVHt=# zBU@~Hk79YutM1+Wdl&L6f))xYH-3m2F@8(kWOaPsrg0I=X;VbWCy|zG^`!tEPUgT= zK3Lt>*hAXlmD_hlvB=49FsQg@eOu#h);A8uJL?sdjT608Fl&Z>y*k=x!l*FSmVk%% zfuw1V;qK9;tNJQ-d5c}QxvumINNB_dCmB_Wu%u^)Q@D+%vVO2|_dWLZ+w2|G^~)!K z!=%^Jg_M-zRyoawP9mwy&m=qKMSO!Ul{fglS_zJf)=f_)8mV*ju8aNriOLnz|3n`u zG&AL~-tO4uIy;Xm*Bz$MJM_rq;X=H^(o^IPg5)k;H5lH_up-TfBSEIDdO8k| ziYmBpw0q-V-R;O&Ta<)&V~%Z%sV*49RX&u~*J@k1XD*Lgj`5jTA*1bB)EX@@_0$dV z`bP@oG30Wkb0Sz5J#&)t`d{^T3?>~%dp8W}Zwzvy<>+)gPpkHxXdU#6gHDe~ccU-Fv zO?&K+7sTUwgUqb6(!b(;tlnYPO~u z8j@>=l-^uSOnB&c=^g!yZS2x_#i!Zt*2mGkW$w=`=s0THSaPcO-s7haJfdaEO?CE4 zLZ=rIY2);qw;1xAR9yTzI9d?+Azq?t{L(cX7O`C%r5o-c&ClDMb$mA;DzT$x+;V0L z=jO}~4&4RQaXz=s$VyY=G-jtiy{Iee-B5V{fndpDjSjD3*0)s3&<_*k@+a!H)~`M# zZ%U}AlZ)$x%m{34bB-x|rTWlwVJ~7Oqh(jBwc36>F4fG>S6W0ic7+2+ zLlxaStLY~_P89o@yN>mb+BUzB5`?Ce&wFJzKkqV@4M*l^z>AP-4d1W`Px73ZD6ikB zqheT18K+pI{WcXPx@n>0TTS!%>p8h_`T*@gw6?sl&FJw2nIiu;ZigHa9^_DF0;Uo` zapafhV>OKUSAAk`@^fq47k_-VvFrSzjpw#!(1)Y9n-S}BUzAi^y*`(`K6@{K1noW$ zABtBe=G;WjXOgjN?M^i)GOBT^n<=F$zH{A_x>tR2FZlGp5A((iK#W9zL-i#njHN7W zdttpdhUa^Qb2wFClMhN#Q;V`VUp|smkzSX_J58#Qa9uDKmp9Z`;2Lk<(S-Z{&X~O% z%`XN|{z!dxf4`n|6}DMG=1?B5q4xqwUUGfvv9Qqkb@mVnvysW!CDXzVBk{}i%=wls z1iRf=s*k?*?WevHze#nI$9f*0DU_*mm(neQ@H}UH-n~lT%O!6c~#OWvDrurPqLpuS73)Y*8u*Ckrucz!SCigUtsm{Qt(v5Jt`QBY2yfn%D!c&Df z&Ed=6QQkhT2a_1e0$XPu3+syRXgw#^(^U(0e{i;qX+sPCT~=DNW>zWYNWwaC?=_9P zbgrgZT(4I77+rSm&sv-u0YP~TPJI4%1>)kvxPFsw{mb4tn_=f1(_%CT$F^j*?BP{& zoE~D?V+S8{W{ExWEtFrG_=(r%^YE)A|L7Oeuqkq1Y0HkWQj3$c4QAiH-D>u8${*yu zX|SJq7zer4ZIBl`B%oVo-}WoWzw%>AsI4HF)4*85a)QM?Utsm_nF^nMT#Dko7-rid z+hMvDb1|Y;Yw;xe{e6d}KdRLSw)w|TpWIAKDai`PYzPV24HjXUJ$&=wD9V*k*O@Z) zh%9V_LT52f=Vr%ju7ssVGGDdS_9*RQ$jrV;j^(%-RA4F2Ch_HbGuXE0XqXV4HX2RG5!A3)V11&=Y_13rMC^l))z&6iY+%=A5M?E1fOoR;e1b*Pk&+cqGq%=SYF6i zd5c}NRQhr9_7%OCVcU!Ob_X^~lp?B!woBfICr9Z6r^2TvfN~vu_SDa5TbaX*lt?^O z-1gebK!9hjdCf;#&29_0My&5nn0_zxEAszPMzFis=Tp|7q1MsbQ|{_){4`x>{Ox1@ z(V`Tm248OK#eTG|6n%+%#m{Tsm<*OslE2Fize4RtSbJl5GDbz|skqPKol=*qEW({# znwrj?JrbqXF#YY-YDi*ke|}o5dNTJK>pLE9>f;|@IR`cK-1woV{egP9K&}}~m46qz zs&)C{dF|Ta#wF*6$-)|PEf}T8L`3Qi+>OqhiStyC#$%aZxi$sl&aGBkm(i8JZS5eU zcHC{Vfs(M|eAlZRW96IUIod|ET-AviZA_77sSan9O66(if(>-UhoznYBHg;Zcg#+P z^@f%Ap!bixs$;vb5GZ0*kR)Tmla6JgB=z@TnY!^w@e}!y+!TSVO{!*)q8(|p5ve`b zyWI6(kEq)9jW$Xn8h>?N{W2hNGE8EzPwLg!Fvd8-qdz*OY+)p!UjjYUN ziRJtoj1)o^?Ea6YCQ<9_Vq>L+q>ZkdM2C13RJfv1bM*%|G-qawo%kPHFjua|ReT|g zCqns;urt$sPTuXJ9E`M54=qop@%u0oyNabx6Q-Rjy^*Ix-)6_llX%PhDeKG};b