diff --git a/.gitignore b/.gitignore index fda3e64..c615a79 100644 --- a/.gitignore +++ b/.gitignore @@ -215,4 +215,8 @@ __marimo__/ .vscode/settings.json # Ignore logs -logs/ \ No newline at end of file +logs/ +*/logs/ +*ckpt +.nfs** +examples/cifar-10-batches-py/* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..75f0876 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,74 @@ +# perspic + +A tool to study neural network training dynamics. Package `perspic`, v0.0.1, authors Konstantin Nikolaou and Jonas Scheunemann, requires Python >=3.11. Built on PyTorch and PyTorch Lightning. + +## Overview + +perspic wraps a user's `pytorch_lightning.LightningModule` with an `analyzer()` factory, transparently instrumenting training to compute per-sample gradient-norm and linear-response metrics — quantities used to study training dynamics (gradient coupling, sensitivity, collective variables) — without requiring the user to modify their model code. The wrapped module trains normally under a standard `pytorch_lightning.Trainer`; the analysis runs and logs metrics (`chi_net`, `chi_loss`, `chi_coup`, `grad_norm_squared`, `loss`, `batch_size`, ...) alongside it. + +## Architecture + +`perspic/analyzer.py` — `analyzer(lightning_module, sample_wise_engine="opacus", disable_analyzer=False, log_metrics=True, opacus_strict=False, opacus_approximate_with_n=None, analyze_every=None, analysis_schedule=None, cross_response=False, micro_batch_size=None, effective_batch_size=None, measure_dataloader=None, measure_batch_size=None, measure_subset_seed=None, **model_kwargs)`. A factory function that dynamically subclasses the given `LightningModule` into an `Analyzer` class. `training_step` is overridden to: run pre-step analysis (`_before_training_step`) → delegate to the wrapped module's own `training_step` → manual backward and optimizer step → step any `interval='step'` LR schedulers → log results. `on_train_epoch_end` steps `interval='epoch'` LR schedulers. The wrapped module must define a `criterion` attribute. Supports `cross_response=True`, which expects a dict batch `{"train": ..., "measure": ...}` (via `CombinedLoader`) to additionally measure the model's linear response against a held-out batch, and sparse analysis scheduling via `analyze_every` or `analysis_schedule`. + +`micro_batch_size`/`effective_batch_size` enable train-side gradient accumulation: the DataLoader yields `micro_batch_size` micro-batches, and the optimizer steps only every `effective_batch_size // micro_batch_size` of them, with per-sample gradient-norm metrics accumulated across the cycle and combined (via the correct extensive/intensive scaling — see `examples/batch_accumulation_notes.md`) before logging. + +`measure_dataloader`/`measure_batch_size`/`measure_subset_seed` give the cross-response measurement side an **independent** batch size, decoupled from the train/`CombinedLoader` path above: the analyzer holds a persistent iterator over `measure_dataloader` and, on each analyzed step, gathers a pool of `max(measure_batch_size)` samples. `measure_batch_size` accepts an int or a `list[int]` to sweep multiple sizes per step (largest processed first; smaller sizes are seed-fixable random subsets of the same pool via `measure_subset_seed`). `measure_dataloader.batch_size` is the maximum single-pass batch size; each swept size `S <= measure_dataloader.batch_size` is measured with a single direct pass (no accumulation), while each `S >` it must be an exact multiple and is measured via gradient accumulation (`S // measure_dataloader.batch_size` passes combined into one measurement) — logging `cross_*` metrics with a `@bs{S}` suffix per swept size. See `examples/measurement_batch_sweep.md` for the full design and usage. + +`perspic/calculator/` — the analysis engines used by `analyzer()`: +- `coupling.py` — `CouplingCalculator`: computes the coupling value `chi_coup = ||grad_L||^2 / (chi_loss * chi_net)`. +- `linearizer.py` — `Linearizer`: computes the exact first-order linear response of the loss via gradient dot products. `compute(model, criterion, x1, y1, x2=None, y2=None)` returns `(loss, perturbed_loss, delta_loss)`, with an optional cross term against a second batch. +- `samplewise.py` — `SamplewiseCalculator`: abstract base class defining the interface for per-sample gradient-norm calculators (`compute`, network- and loss-level norm helpers), plus shared helpers. +- `samplewise_functorch.py` — `SamplewiseCalculatorFunctorch`: per-sample gradient norms via `torch.func` (`vmap` + `jacrev`). +- `samplewise_opacus.py` — `SamplewiseCalculatorOpacus`: per-sample gradient norms via Opacus ghost clipping (`GradSampleModuleFastGradientClipping`), with custom BatchNorm grad/norm samplers for eval-mode frozen stats and an optional Hutchinson trace-estimator approximation (`approximate_with_n`) for the network-level norm. + +`perspic/logger.py` — `LogarithmicWindowSchedule` (dataclass) and `logarithmic_windows(max_steps, points_per_decade=10, base_window=5, adaptive_scale=0.0)`, for building logarithmically-spaced step schedules so analysis can run sparsely over long training runs (pass as `analyzer(..., analysis_schedule=...)`). + +`perspic/utils.py` — `BatchStatSnapshot`, a context manager that freezes BatchNorm running stats (with Bessel's correction) so per-sample `vmap` gradients match a train-mode forward pass; `MultiEpochsDataLoader` and `RepeatSampler`, a `DataLoader` subclass that reuses its workers/iterator across epochs. + +Class design relies on: ABCs for the calculator hierarchy, a dataclass for the logging schedule, and a factory-returns-dynamic-subclass pattern for `analyzer()` (it builds `class Analyzer(lightning_module): ...` at call time rather than requiring users to subclass anything themselves). + +## Public API + +`from perspic import ...`: +- `analyzer` — the primary entry point; wraps and instantiates an `Analyzer` from a `LightningModule`. +- `Linearizer` — standalone linear-response calculator. +- `SamplewiseCalculatorFunctorch`, `SamplewiseCalculatorOpacus` — standalone per-sample gradient-norm calculators, also selectable inside `analyzer()` via `sample_wise_engine`. +- `LogarithmicWindowSchedule`, `logarithmic_windows` — sparse analysis scheduling. +- `MultiEpochsDataLoader` — performance-oriented DataLoader. + +Also available via `perspic.calculator`: `CouplingCalculator`, `SamplewiseCalculator` (base class). + +## Design notes + +`analyzer()` sets `automatic_optimization = False` because its analysis metrics require extra forward/backward passes separate from the training pass: the `Linearizer` does its own `zero_grad -> forward -> backward` to get `||grad_L||^2`, and `SamplewiseCalculatorOpacus` runs `output_dim` extra forward+backward passes through the Opacus ghost-clipping hooks for `chi_net` (plus one extra forward for `chi_loss`). Manual optimization keeps these extra passes from corrupting Lightning's own gradient/optimizer state. If the wrapped module already uses manual optimization itself, `analyzer()` delegates to it instead of double-handling the optimizer step. + +## Development + +Install: `pip install -r requirements.txt -r dev-requirements.txt` + +Test: `pytest` +- `tests/unit/` mirrors each `perspic` module 1:1 (`test_analyzer.py`, `test_linearizer.py`, `test_logger.py`, `test_samplewise.py`, `test_samplewise_functorch.py`, `test_samplewise_opacus.py`, `test_utils.py`), using `unittest.mock` and small synthetic Lightning modules. +- `tests/integration/` covers end-to-end/deployment scenarios: `test_analyzer_deployment.py`, `test_hutchinson_approximation.py`, `test_linearizer_deployment.py`, `test_lna_ntk_ground_truth.py`, `test_normalization_scaling.py`, `test_training_deployment.py`. + +Lint/format: `pre-commit run --all-files` (runs `black`, `isort`, `flake8` in that order, `fail_fast: true`), or individually `black .`, `isort .`, `flake8`. + +## Conventions + +- Line length 88 (`black`, preview mode; `flake8` matches). +- `isort` uses the `black` profile. +- `flake8` ignores `W503, E203, E402, C901`; `examples/` is excluded from flake8 checks. +- Docstrings are predominantly Google-style (`Args:`, `Returns:`, `Raises:`). +- Type hints are used throughout function/method signatures. + +## Examples + +`examples/`: +- `cifar10.ipynb` — basic `analyzer()` usage: wrap a `LightningModule`, train on CIFAR-10, plot logged metrics. +- `cifar10_CrossReseponse.ipynb` — `cross_response=True` demo, measuring linear response against a held-out batch via a `CombinedLoader`. +- `batch_accumulation.ipynb` / `batch_accumulation_notes.md` — `micro_batch_size`/`effective_batch_size` gradient-accumulation demo and the design notes explaining the χ_loss aggregation fix and `effective_step`-based logging. +- `batch_size_scaling_analysis.ipynb` — synthetic sweep over batch size (repeated identical samples) verifying `chi_net`/`chi_loss` batch-size invariance directly against `SamplewiseCalculatorFunctorch`, independent of `analyzer()`. +- `measurement_batch_sweep.md` — design notes for `measure_dataloader`/`measure_batch_size`/`measure_subset_seed`, the independent (and sweepable) cross-response measurement batch size. +- `logging_scheduler.ipynb` — `logarithmic_windows` / `LogarithmicWindowSchedule` demo combined with LR scheduling. +- `mup_integration.ipynb` — integrating Maximal Update Parametrization (mup) with perspic. +- `core/hutchinson_convergence.py` — convergence of the Opacus Hutchinson trace-estimator approximation toward the exact per-sample gradient norm. +- `models/` — shared model zoo used by the notebooks above (`cnns.py`, `mlps.py`, `lightning_modules.py`, `utils.py`). diff --git a/PHASE1_batch_accumulation_review.md b/PHASE1_batch_accumulation_review.md new file mode 100644 index 0000000..deefabe --- /dev/null +++ b/PHASE1_batch_accumulation_review.md @@ -0,0 +1,44 @@ +# PHASE 1 — Batch-Accumulation Feature Audit + +## Data-flow trace + +Per micro-batch, [analyzer.py:262-365](perspic/analyzer.py#L262-L365) executes: `opt.zero_grad()` only when `_accumulation_count == 0` → analysis before-hook → wrapped `training_step` → `manual_backward(output / K)` → increment count → when `count >= K`: `opt.step()`, increment `_optimizer_step_count`, reset count, step per-step schedulers. + +The analysis path ([_analyze_accumulated_step](perspic/analyzer.py#L503-L584)) decides `_analysis_active` on micro-batch 0 using `effective_step`, saves/restores `p.grad` clones around the grad-clobbering analysis passes, accumulates chi metrics and unscaled linearizer gradients per micro-batch, and finalizes on micro-batch K−1 — **before** the optimizer step, consistent with the single-step path, which also measures pre-step state. Exactly K entries accumulate per cycle; **no off-by-one in the accumulation or flush ordering itself**. + +Math verified end-to-end: +- Gradient: `Σₖ ∇(Lₖ/K) = ∇(mean loss over N=K·B)` ✓ (assuming mean-reduced criterion, equal micro-batch sizes) +- `grad_norm_squared = ‖Σgₖ‖²/K² = ‖∇L_full‖²` ✓ — confirmed numerically by [test G](tests/unit/test_analyzer.py#L1248-L1307) +- `chi_net_eff = chi_loss_eff = mean over K` ✓ per the derivation in [batch_accumulation_notes.md](examples/batch_accumulation_notes.md). Note: **the old `K·sum` chi_loss bug (a K² error, 16× at K=4) is fixed only in the uncommitted working tree** — the committed branch tip still contains it. That fix needs to be committed. + +## Verdict: ⚠️ Minor issues — core math correct; one label misalignment and several edge cases + +## Correctness issues + +1. **`effective_step` is off by one against everything it's compared with** — [analyzer.py:329-334](perspic/analyzer.py#L329-L334). Cycle N (0-indexed) tags its micro-batches `effective_step = N+1`, but the same cycle's analysis metrics log `analysis_step = N` ([analyzer.py:765](perspic/analyzer.py#L765)), and the full-batch comparison run's `train_loss` lands at Lightning `step = N`. The notebook plots loss by `effective_step` and chi by `analysis_step` on shared axes, so the accumulation loss curve is shifted +1 — visible at early steps on log axes. + **Fix**: `self.log("effective_step", float(self._optimizer_step_count), ...)` — drop the `+1`. + +2. **`_accum_step_losses` is dead code** — initialized at [analyzer.py:251](perspic/analyzer.py#L251), appended at [323](perspic/analyzer.py#L323), cleared at [345-346](perspic/analyzer.py#L345-L346) and [735](perspic/analyzer.py#L735), never read. Retains detached GPU scalars for nothing. **Fix**: delete all four sites. + +3. **No guard against ragged micro-batches** — `output / K` assumes every micro-batch has exactly `micro_batch_size` samples. With `drop_last=False`, a smaller final batch makes the accumulated gradient a weighted (wrong) mean, and [analyzer.py:628](perspic/analyzer.py#L628) uses the *last* micro-batch's `x.shape[0]` for the logged `batch_size`/`effective_batch_size`. **Fix**: warn once in `training_step` when `batch[0].shape[0] != micro_batch_size`; document `drop_last=True`. + +4. **Interrupted / partial cycles**: `_accumulation_count` isn't reset at epoch end, so cycles span epoch boundaries (self-consistent — gradients and analysis both span — but undocumented); a trailing partial cycle at end of training silently drops its gradients; `on_train_epoch_end` steps epoch-interval schedulers even mid-cycle. Additionally `_optimizer_step_count`/`_accumulation_count` are **not checkpointed**, so a resumed run resets `effective_step` to 0 and misaligns `analysis_schedule`. **Fix**: `on_save_checkpoint`/`on_load_checkpoint` for both counters; document epoch-spanning behavior. + +5. **Question — mean-reduction assumption**: loss/K scaling *and* the mean-aggregation of `chi_loss` are only valid for a mean-reduced criterion (your notes acknowledge this). Nothing checks `criterion.reduction`; a sum-reduced criterion silently yields wrong gradients and a K²-wrong `chi_loss_eff`. Suggest a best-effort warning via `getattr(criterion, "reduction", "mean")`. + +6. **Question**: `effective_step` is logged even when `log_metrics=False` — intentional (training metadata, not an analysis metric)? It's the only unconditional `self.log` in the class. + +No race conditions: single-process Lightning, no threading/async anywhere in the accumulation flow. + +## Efficiency issues + +1. **Train-side linearizer passes are redundant** — [_accumulate_linearizer_grads](perspic/analyzer.py#L586-L623) adds one extra forward+backward per micro-batch, but at cycle end `p.grad` already holds `Σ∇(Lₖ)/K`, whose squared norm equals exactly the `‖Σgₖ‖²/K²` that [finalize](perspic/analyzer.py#L644-L647) computes. Reading `p.grad` just before `opt.step()` eliminates **K forward+backward passes per analysis cycle plus one full-model gradient buffer**. (Matches your existing hook-based-refactor design notes; measure side genuinely needs its own passes.) ~15% of analysis cost at 10 output dims; much more for low-output-dim models. +2. **`BatchStatSnapshot` pays a full dummy forward even with zero BatchNorm layers** — [utils.py:145-146](perspic/utils.py#L145-L146). For BN-free models: one wasted full-batch forward per analysis micro-batch. Early-out (but keep the `model.eval()` switch so dropout behavior is unchanged) — cheapest win in the feature. +3. **One GPU sync per parameter** — `.item()` inside per-parameter generator sums at [analyzer.py:644-647](perspic/analyzer.py#L644-L647), [675-682](perspic/analyzer.py#L675-L682), and [linearizer.py:78-79](perspic/calculator/linearizer.py#L78-L79), [103-107](perspic/calculator/linearizer.py#L103-L107). Sum on-device, call `.item()` once. +4. The loop itself is **O(K)** with in-place `acc.add_()` — no quadratic behavior, no unneeded copies beyond the required `saved_grads` clone (necessary because `sample_calc.compute` calls `model.zero_grad()`). Peak: ~3 concurrent full-gradient buffers (`saved_grads`, `_accum_grad_train`, `_accum_grad_measure`) — acceptable, worth a docstring note. `itertools`/`deque`/vectorization don't apply here; the buffers are parameter-shaped tensors handled correctly. + +## Best-practice suggestions + +- The train/measure branches of `_accumulate_linearizer_grads` are copy-paste duplicates; unify. +- Several tests drive `_before_training_step` by poking `_accumulation_count` directly — brittle coupling to private state; the `training_step`-driven style of `test_analysis_does_not_corrupt_training_gradients` is more robust. +- Typo "the the" in the manual-optimization warning ([analyzer.py:185](perspic/analyzer.py#L185)). diff --git a/PHASE2_full_project_review.md b/PHASE2_full_project_review.md new file mode 100644 index 0000000..ec79d9c --- /dev/null +++ b/PHASE2_full_project_review.md @@ -0,0 +1,73 @@ +# PHASE 2 — Full Project Review + +**[SEVERITY: HIGH]** [samplewise_opacus.py:287-340](perspic/calculator/samplewise_opacus.py#L287-L340) +- **Issue**: `gs_model.remove_hooks()` and `_cleanup_opacus_leftovers()` are not in the `finally` block (only `_restore_inplace_ops` is). +- **Impact**: any exception mid-computation (CUDA OOM, the `NotImplementedError` for tied params) leaves ghost-clipping hooks attached — every subsequent training forward accumulates activations and per-sample state, silently corrupting training and growing memory. +- **Fix**: +```python +gs_model = None +try: + ... + gs_model = _GhostNormFastGradientClipping(...) + ... + return total_sq_norms.sum() if reduce else total_sq_norms +finally: + if gs_model is not None: + gs_model.remove_hooks() + _cleanup_opacus_leftovers(model) + _restore_inplace_ops(inplace_states) +``` + +**[SEVERITY: MEDIUM]** [samplewise_opacus.py:150-153](perspic/calculator/samplewise_opacus.py#L150-L153) +- **Issue**: `_cleanup_opacus_leftovers` deletes `param.norm_sample`, but opacus 1.5.4 (verified against installed source of `get_norm_sample`) stores `param._norm_sample`; the current check is dead. +- **Impact**: after the final output-dim iteration, `_norm_sample` tensors stay attached to every parameter indefinitely. +- **Fix**: add `if hasattr(param, "_norm_sample"): delattr(param, "_norm_sample")`. + +**[SEVERITY: MEDIUM]** [utils.py:106-207](perspic/utils.py#L106-L207) +- **Issue**: `BatchStatSnapshot.__enter__` is not exception-safe — if the dummy forward raises, `__exit__` never runs. +- **Impact**: model left with `momentum=1.0`, BN layers in train mode, running stats clobbered, forward hooks still registered. +- **Fix**: wrap the body in `try/except`, restore saved state and remove hooks before re-raising (and add the no-BN early-out from Phase 1). + +**[SEVERITY: MEDIUM]** [samplewise_opacus.py:366-367](perspic/calculator/samplewise_opacus.py#L366-L367) +- **Issue**: `_compute_per_sample_gradient_norm_loss` runs `model(inputs)` with grad enabled, then immediately detaches the output. +- **Impact**: builds and stores the full parameter autograd graph (all activations) that is never used — wasted memory every analysis step. +- **Fix**: +```python +with torch.no_grad(): + outputs = model(inputs) +outputs = outputs.requires_grad_(True) +``` + +**[SEVERITY: MEDIUM]** [coupling.py:33](perspic/calculator/coupling.py#L33) +- **Issue**: unguarded division `grad_norm_squared / (chi_loss * chi_net)`. +- **Impact**: zero gradients → `ZeroDivisionError` (floats) or silently logged `inf`/`nan` (tensors). +- **Fix**: return `float("nan")` when the denominator is 0. + +**[SEVERITY: MEDIUM]** [logger.py:134-137](perspic/logger.py#L134-L137) +- **Issue**: every window whose tail exceeds `max_steps` is skipped — for `max_steps < base_window` even the step-0 window is dropped. +- **Impact**: `logarithmic_windows(max_steps=4)` returns an empty schedule; analysis silently never runs. +- **Fix**: warn when the resulting schedule is empty (or clamp the step-0 window). Related: overlapping windows keep their full step lists in `windows` even when `step_to_window` reassigns steps to a later window, so `window_width` overcounts. + +**[SEVERITY: MEDIUM]** [analyzer.py:239-240](perspic/analyzer.py#L239-L240) +- **Issue**: accumulation/step counters not persisted across checkpoints (see Phase 1 #4). +- **Impact**: schedule misalignment and `zero_grad` boundary desync on resume. +- **Fix**: save/restore `_optimizer_step_count` and `_accumulation_count` in `on_save_checkpoint`/`on_load_checkpoint`. + +**[SEVERITY: LOW]** — briefly: +- [analyzer.py:349-357](perspic/analyzer.py#L349-L357): scheduler stepping ignores `config.frequency`; relies on private `self._trainer`. +- [analyzer.py:315-317](perspic/analyzer.py#L315-L317): `output / K` breaks if the wrapped `training_step` returns Lightning's `{"loss": ...}` dict (pre-existing). +- [analyzer.py:415-501](perspic/analyzer.py#L415-L501): full analysis compute runs even when `log_metrics=False`, results discarded. +- [samplewise_opacus.py:178](perspic/calculator/samplewise_opacus.py#L178): Rademacher vectors hardcoded `float32` — mismatches float64/bf16 models. +- [samplewise_opacus.py:17-19](perspic/calculator/samplewise_opacus.py#L17-L19): import-time mutation of Opacus's global sampler registry affects any other Opacus user in the process. +- [mlps.py:103](examples/models/mlps.py#L103): mutable default argument `hidden_sizes=[1024, 512, 256]`. +- [lightning_modules.py:173](examples/models/lightning_modules.py#L173): `AdvancedClassificationModule` has no `criterion` attribute → incompatible with `analyzer()` (and its label smoothing wouldn't be visible to analysis regardless). +- [analyzer.py:17](perspic/analyzer.py#L17): `Optional[str]` on `sample_wise_engine` is misleading (`None` is rejected); the dynamically created `Analyzer` class also can't be unpickled / `load_from_checkpoint`-ed — worth a docstring note. +- Repo hygiene: `examples/cifar-10-python.tar.gz` (~170 MB) is untracked and **not** covered by the new `.gitignore` entries (only `cifar-10-batches-py/*` is); same for `examples/MNIST/` and `first_wrong_test_batch_accumulation.png` — one careless `git add .` away from a 170 MB commit. + +## Top 5 prioritized actions + +1. **Make the Opacus wrapper exception-safe** (`remove_hooks` + cleanup in `finally`) and fix the `_norm_sample` cleanup name — prevents silent training corruption after any analysis failure. +2. **Drop the `+1` from `effective_step`** and remove dead `_accum_step_losses` — restores x-axis alignment across runs and metrics. Also **commit the pending chi_loss `sum/K` fix**, which currently exists only in the working tree. +3. **Warn on ragged micro-batches and checkpoint the accumulation counters** — closes the remaining accumulation edge cases. +4. **`BatchStatSnapshot`: skip the dummy forward for BN-free models and make `__enter__` exception-safe** — cheapest meaningful speed + robustness win. +5. **Replace train-side `_accumulate_linearizer_grads` with a read of `p.grad` at cycle end** — K fewer forward+backward passes per analysis cycle and one fewer full-gradient buffer; first step of the hook-based refactor already sketched in your design notes. diff --git a/examples/batch_accumulation.ipynb b/examples/batch_accumulation.ipynb new file mode 100644 index 0000000..7198c5d --- /dev/null +++ b/examples/batch_accumulation.ipynb @@ -0,0 +1,1612 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "630912d7", + "metadata": {}, + "source": [ + "# Batch Accumulation with Perspic\n", + "\n", + "This notebook demonstrates how to use **gradient accumulation** with the `perspic` analyzer.\n", + "Gradient accumulation lets you simulate a large effective batch size while only fitting a small micro-batch in GPU memory.\n", + "\n", + "We train a small Vision Transformer on CIFAR-10 and compare training **with** and **without** batch accumulation." + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "e2d7ffb7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 7\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "import pytorch_lightning as pl\n", + "import torch\n", + "import torch.nn as nn\n", + "import torchvision\n", + "from pytorch_lightning.callbacks import LearningRateMonitor\n", + "from pytorch_lightning.loggers import CSVLogger\n", + "from torch.utils.data import DataLoader, random_split\n", + "from torchvision.datasets import CIFAR10, MNIST\n", + "from torchvision.models import VisionTransformer\n", + "\n", + "from perspic.analyzer import analyzer\n", + "from examples.models import ClassificationModule\n", + "\n", + "pl.seed_everything(7)\n", + "\n", + "PATH_DATASETS = os.environ.get(\"PATH_DATASETS\", \".\")\n", + "\n", + "# CIFAR-10 / ViT experiment\n", + "MICRO_BATCH_SIZE = 64\n", + "HALF_BATCH_SIZE = 128 # Run 3: direct batch, no accumulation\n", + "EFFECTIVE_BATCH_SIZE = 256 # 4x accumulation\n", + "\n", + "# MNIST / MLP experiment\n", + "MNIST_MICRO_BATCH = 64\n", + "MNIST_EFFECTIVE_BATCH = 256\n", + "\n", + "NUM_WORKERS = int(os.cpu_count() / 2)\n", + "\n", + "torch.set_float32_matmul_precision(\"highest\")" + ] + }, + { + "cell_type": "markdown", + "id": "d0818a3e", + "metadata": {}, + "source": [ + "## Data Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "7ef5f919", + "metadata": {}, + "outputs": [], + "source": [ + "stats = ((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n", + "train_transform = torchvision.transforms.Compose([\n", + " #torchvision.transforms.RandomCrop(32, padding=4),\n", + " #torchvision.transforms.RandomHorizontalFlip(),\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize(*stats),\n", + "])\n", + "test_transform = torchvision.transforms.Compose([\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize(*stats),\n", + "])\n", + "\n", + "train_dataset_full = CIFAR10(PATH_DATASETS, train=True, download=True, transform=train_transform)\n", + "val_dataset_full = CIFAR10(PATH_DATASETS, train=True, download=True, transform=test_transform)\n", + "test_set = CIFAR10(PATH_DATASETS, train=False, download=True, transform=test_transform)\n", + "\n", + "generator = torch.Generator().manual_seed(42)\n", + "train_set, _ = random_split(train_dataset_full, [45000, 5000], generator=generator)\n", + "_, val_set = random_split(val_dataset_full, [45000, 5000], generator=generator)\n", + "\n", + "# Use micro-batch size for the DataLoader — accumulation handles the rest\n", + "train_dataloader = DataLoader(train_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)\n", + "val_dataloader = DataLoader(val_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)\n", + "test_dataloader = DataLoader(test_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "969579dc", + "metadata": {}, + "outputs": [], + "source": [ + "mnist_transform = torchvision.transforms.ToTensor()\n", + "\n", + "mnist_full = MNIST(PATH_DATASETS, train=True, download=True, transform=mnist_transform)\n", + "mnist_test = MNIST(PATH_DATASETS, train=False, download=True, transform=mnist_transform)\n", + "\n", + "generator_mnist = torch.Generator().manual_seed(42)\n", + "mnist_train, _ = random_split(mnist_full, [55000, 5000], generator=generator_mnist)\n", + "generator_mnist2 = torch.Generator().manual_seed(42)\n", + "_, mnist_val = random_split(mnist_full, [55000, 5000], generator=generator_mnist2)\n", + "\n", + "mnist_train_dl = DataLoader(mnist_train, batch_size=MNIST_MICRO_BATCH, shuffle=True, num_workers=NUM_WORKERS, drop_last=True)\n", + "mnist_train_dl_full = DataLoader(mnist_train, batch_size=MNIST_EFFECTIVE_BATCH, shuffle=True, num_workers=NUM_WORKERS, drop_last=True)\n", + "mnist_val_dl = DataLoader(mnist_val, batch_size=MNIST_EFFECTIVE_BATCH, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)" + ] + }, + { + "cell_type": "markdown", + "id": "321b9a72", + "metadata": {}, + "source": [ + "## Model Definition\n", + "\n", + "A small Vision Transformer suitable for CIFAR-10." + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "06369a8f", + "metadata": {}, + "outputs": [], + "source": [ + "model_vit = VisionTransformer(\n", + " image_size=32,\n", + " patch_size=8,\n", + " num_layers=2,\n", + " num_heads=4,\n", + " hidden_dim=128,\n", + " mlp_dim=256,\n", + " num_classes=10,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "4cb8a66b", + "metadata": {}, + "outputs": [], + "source": [ + "class MLP(nn.Module):\n", + " \"\"\"Three-layer MLP for MNIST (784 → 256 → 128 → 10).\"\"\"\n", + "\n", + " def __init__(self, input_dim=784, hidden_dim1=256, hidden_dim2=1028, hidden_dim3=128, output_dim=10):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(input_dim, hidden_dim1)\n", + " self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)\n", + " self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)\n", + " self.fc4 = nn.Linear(hidden_dim3, output_dim)\n", + "\n", + " def forward(self, x):\n", + " x = x.flatten(start_dim=1) # (B, 1, 28, 28) → (B, 784)\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return self.fc4(x)" + ] + }, + { + "cell_type": "markdown", + "id": "563f9db6", + "metadata": {}, + "source": [ + "## Training Setup\n", + "\n", + "We run four experiments with the same model architecture:\n", + "\n", + "- **Accum (`micro=64, effective=256`)**: DataLoader B=64, gradient accumulation over 4 steps simulates B=256.\n", + "- **Full batch (`B=256`)**: DataLoader B=256 directly, no accumulation.\n", + "- **Half batch (`B=128`)**: DataLoader B=128, no accumulation. Intermediate reference.\n", + "- **Single batch (`B=64`)**: DataLoader B=64, no accumulation. Shows the raw noise level of the micro-batch size; makes 4× more optimizer steps per epoch than the B=256 runs.\n", + "\n", + "If accumulation is correct, Run 1 and Run 2 should overlay. Run 4 lets you see the noise floor a B=64 batch introduces — the accumulation run should match B=256 noise, not B=64 noise." + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "998cb095", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Run 1 — accumulation steps: 4 (micro=64, effective=256)\n", + "Run 2 — accumulation steps: 1 (full batch=256)\n", + "Run 3 — accumulation steps: 1 (half batch=128)\n", + "Run 4 — accumulation steps: 1 (single batch=64, no accum)\n" + ] + } + ], + "source": [ + "import copy\n", + "\n", + "# --- Run 1: gradient accumulation (micro_batch=64, effective=256) ---\n", + "model_vit_accum = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", + "vit_accum = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=MICRO_BATCH_SIZE,\n", + " effective_batch_size=EFFECTIVE_BATCH_SIZE,\n", + " model=model_vit_accum,\n", + " lr=0.005,\n", + ")\n", + "\n", + "# --- Run 2: full batch (batch=256, no accumulation) ---\n", + "model_vit_full = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", + "vit_full = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=EFFECTIVE_BATCH_SIZE, # DataLoader batch IS the effective batch\n", + " model=model_vit_full,\n", + " lr=0.005,\n", + ")\n", + "\n", + "# --- Run 3: half batch (batch=128, no accumulation) ---\n", + "model_vit_half = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", + "vit_half = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=HALF_BATCH_SIZE,\n", + " model=model_vit_half,\n", + " lr=0.005,\n", + ")\n", + "\n", + "# --- Run 4: single micro-batch (batch=64, no accumulation) — noise baseline ---\n", + "model_vit_single = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", + "vit_single = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=MICRO_BATCH_SIZE, # no effective_batch_size → no accumulation\n", + " model=model_vit_single,\n", + " lr=0.005,\n", + ")\n", + "\n", + "print(f\"Run 1 — accumulation steps: {vit_accum.accumulation_steps} (micro={MICRO_BATCH_SIZE}, effective={EFFECTIVE_BATCH_SIZE})\")\n", + "print(f\"Run 2 — accumulation steps: {vit_full.accumulation_steps} (full batch={EFFECTIVE_BATCH_SIZE})\")\n", + "print(f\"Run 3 — accumulation steps: {vit_half.accumulation_steps} (half batch={HALF_BATCH_SIZE})\")\n", + "print(f\"Run 4 — accumulation steps: {vit_single.accumulation_steps} (single batch={MICRO_BATCH_SIZE}, no accum)\")" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "930b6ba5", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 7\n", + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params | Mode \n", + "----------------------------------------------------\n", + "0 | model | VisionTransformer | 293 K | train\n", + "----------------------------------------------------\n", + "293 K Trainable params\n", + "0 Non-trainable params\n", + "293 K Total params\n", + "1.174 Total estimated model params size (MB)\n", + "32 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1d5e87ec911344cebe5d3e5ff84ad3e7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Sanity Checking: | | 0/? [00:00:0: UserWarning: Full backward hook is firing when gradients are computed with respect to module outputs since no inputs require gradients. See https://docs.pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.register_full_backward_hook for more details.\n", + "/tikhome/jscheunemann/usr/miniconda3/envs/mast/lib/python3.13/site-packages/torch/autograd/graph.py:829: UserWarning: There is a performance drop because we have not yet implemented the batching rule for aten::_scaled_dot_product_efficient_attention_backward. Please file us an issue on GitHub so that we can prioritize its implementation. (Triggered internally at /pytorch/aten/src/ATen/functorch/BatchedFallback.cpp:81.)\n", + " return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6ca8838b582a4c6a8d4e78c8a5ea63d5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Validation: | | 0/? [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "metrics_accum = pd.read_csv(f\"{trainer_accum.logger.log_dir}/metrics.csv\")\n", + "metrics_full = pd.read_csv(f\"{trainer_full.logger.log_dir}/metrics.csv\")\n", + "metrics_half = pd.read_csv(f\"{trainer_half.logger.log_dir}/metrics.csv\")\n", + "metrics_single = pd.read_csv(f\"{trainer_single.logger.log_dir}/metrics.csv\")\n", + "\n", + "EMA_SPAN = 20\n", + "\n", + "COLORS = {\n", + " \"accum\": \"steelblue\",\n", + " \"full\": \"tomato\",\n", + " \"half\": \"seagreen\",\n", + " \"single\": \"olive\",\n", + "}\n", + "\n", + "ZORDER = {\n", + " \"accum\": 25,\n", + " \"full\": 30,\n", + " \"half\": 20,\n", + " \"single\": 10,\n", + "}\n", + "\n", + "def plot_metric(ax, metrics, col, step_col, label, color, alpha_raw=0.2, zorder=None):\n", + " subset = metrics[[col, step_col]].dropna()\n", + " if subset.empty:\n", + " return\n", + " ax.plot(subset[step_col], subset[col], alpha=alpha_raw, color=color, zorder=zorder or ZORDER.get(label.split()[-1], 0))\n", + " ax.plot(subset[step_col], subset[col].ewm(span=EMA_SPAN).mean(),\n", + " label=label, color=color, linewidth=1.8, zorder=zorder or ZORDER.get(label.split()[-1], 0))\n", + "\n", + "def loss_per_opt_step(metrics, loss_col=\"train_loss\"):\n", + " \"\"\"Return (x, y) for train_loss aligned to one point per optimizer step.\n", + "\n", + " For accumulation runs: effective_step is logged on every micro-batch, so\n", + " groupby(effective_step).mean() averages exactly the K micro-batches of each\n", + " cycle — the correct per-cycle mean loss.\n", + "\n", + " For non-accumulation runs: no effective_step column; use plain step.\n", + " \"\"\"\n", + " if \"effective_step\" in metrics.columns:\n", + " sub = metrics[[\"effective_step\", loss_col]].dropna()\n", + " sub = sub.groupby(\"effective_step\")[loss_col].mean().reset_index()\n", + " return sub[\"effective_step\"], sub[loss_col]\n", + " else:\n", + " sub = metrics[[\"step\", loss_col]].dropna()\n", + " return sub[\"step\"], sub[loss_col]\n", + "\n", + "fig, axes = plt.subplots(2, 2, figsize=(14, 8))\n", + "fig.suptitle(\n", + " \"ViT: \"\n", + " f\"Accum (micro={MICRO_BATCH_SIZE}→eff={EFFECTIVE_BATCH_SIZE}) vs \"\n", + " f\"Full B={EFFECTIVE_BATCH_SIZE} vs Half B={HALF_BATCH_SIZE} vs Single B={MICRO_BATCH_SIZE}\",\n", + " fontsize=12,\n", + ")\n", + "\n", + "step_accum = \"analysis_step\" if \"analysis_step\" in metrics_accum.columns else \"step\"\n", + "step_full = \"analysis_step\" if \"analysis_step\" in metrics_full.columns else \"step\"\n", + "step_half = \"analysis_step\" if \"analysis_step\" in metrics_half.columns else \"step\"\n", + "step_single = \"analysis_step\" if \"analysis_step\" in metrics_single.columns else \"step\"\n", + "\n", + "# --- Train Loss ---\n", + "ax = axes[0, 0]\n", + "x_a, y_a = loss_per_opt_step(metrics_accum)\n", + "x_f, y_f = loss_per_opt_step(metrics_full)\n", + "x_h, y_h = loss_per_opt_step(metrics_half)\n", + "x_s, y_s = loss_per_opt_step(metrics_single)\n", + "for x, y, col, lbl in [\n", + " (x_a, y_a, COLORS[\"accum\"], f\"Accum micro={MICRO_BATCH_SIZE}→eff={EFFECTIVE_BATCH_SIZE}\"),\n", + " (x_f, y_f, COLORS[\"full\"], f\"Full B={EFFECTIVE_BATCH_SIZE}\"),\n", + " (x_h, y_h, COLORS[\"half\"], f\"Half B={HALF_BATCH_SIZE}\"),\n", + " (x_s, y_s, COLORS[\"single\"], f\"Single B={MICRO_BATCH_SIZE} (no accum)\"),\n", + "]:\n", + " ax.plot(x, y, alpha=0.15, color=col, zorder=ZORDER.get(lbl.split()[-1], 0))\n", + " ax.plot(x, y.ewm(span=EMA_SPAN).mean(), color=col, linewidth=1.8, label=lbl, zorder=ZORDER.get(lbl.split()[-1], 0))\n", + "ax.set_title(\"Train Loss\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# --- χ_net ---\n", + "ax = axes[0, 1]\n", + "plot_metric(ax, metrics_accum, \"chi_net\", step_accum, f\"Accum micro={MICRO_BATCH_SIZE}\", COLORS[\"accum\"], zorder=ZORDER[\"accum\"])\n", + "plot_metric(ax, metrics_full, \"chi_net\", step_full, f\"Full B={EFFECTIVE_BATCH_SIZE}\", COLORS[\"full\"], zorder=ZORDER[\"full\"])\n", + "plot_metric(ax, metrics_half, \"chi_net\", step_half, f\"Half B={HALF_BATCH_SIZE}\", COLORS[\"half\"], zorder=ZORDER[\"half\"])\n", + "plot_metric(ax, metrics_single, \"chi_net\", step_single, f\"Single B={MICRO_BATCH_SIZE}\", COLORS[\"single\"], zorder=ZORDER[\"single\"])\n", + "ax.set_title(\"χ_net\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.set_ylim(1e2,5e3)\n", + "ax.set_xlim(1, 5e3)\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# --- χ_loss ---\n", + "ax = axes[1, 0]\n", + "plot_metric(ax, metrics_accum, \"chi_loss\", step_accum, f\"Accum micro={MICRO_BATCH_SIZE}\", COLORS[\"accum\"], zorder=ZORDER[\"accum\"])\n", + "plot_metric(ax, metrics_full, \"chi_loss\", step_full, f\"Full B={EFFECTIVE_BATCH_SIZE}\", COLORS[\"full\"], zorder=ZORDER[\"full\"])\n", + "plot_metric(ax, metrics_half, \"chi_loss\", step_half, f\"Half B={HALF_BATCH_SIZE}\", COLORS[\"half\"], zorder=ZORDER[\"half\"])\n", + "plot_metric(ax, metrics_single, \"chi_loss\", step_single, f\"Single B={MICRO_BATCH_SIZE}\", COLORS[\"single\"], zorder=ZORDER[\"single\"])\n", + "ax.set_title(\"χ_loss\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# --- Coupling ---\n", + "ax = axes[1, 1]\n", + "plot_metric(ax, metrics_accum, \"chi_coup\", step_accum, f\"Accum micro={MICRO_BATCH_SIZE}\", COLORS[\"accum\"], zorder=ZORDER[\"accum\"])\n", + "plot_metric(ax, metrics_full, \"chi_coup\", step_full, f\"Full B={EFFECTIVE_BATCH_SIZE}\", COLORS[\"full\"], zorder=ZORDER[\"full\"])\n", + "plot_metric(ax, metrics_half, \"chi_coup\", step_half, f\"Half B={HALF_BATCH_SIZE}\", COLORS[\"half\"], zorder=ZORDER[\"half\"])\n", + "plot_metric(ax, metrics_single, \"chi_coup\", step_single, f\"Single B={MICRO_BATCH_SIZE}\", COLORS[\"single\"], zorder=ZORDER[\"single\"])\n", + "ax.set_title(\"Coupling (χ_coup)\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0a62bfce", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## MLP — MNIST with Gradient Accumulation\n", + "\n", + "Same accumulation comparison on a simpler model and dataset: a three-layer MLP (784→256→128→10) trained on MNIST.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "a51fb353", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 7\n", + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params | Mode \n", + "---------------------------------------\n", + "0 | model | MLP | 598 K | train\n", + "---------------------------------------\n", + "598 K Trainable params\n", + "0 Non-trainable params\n", + "598 K Total params\n", + "2.393 Total estimated model params size (MB)\n", + "5 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MLP accum: 4 steps (micro=64 → eff=256)\n", + "MLP full: 1 step (batch=256)\n", + "MLP half: 1 step (batch=128)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "032453cc60a741a3a5852a28ace98b21", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Sanity Checking: | | 0/? [00:00:0: UserWarning: Full backward hook is firing when gradients are computed with respect to module outputs since no inputs require gradients. See https://docs.pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.register_full_backward_hook for more details.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1bc3a766a8214c54b5159733bf7aa512", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Validation: | | 0/? [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import glob\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "def _load_latest_csv(log_name):\n", + " matches = sorted(glob.glob(f\"logs/{log_name}/version_*/metrics.csv\"))\n", + " if not matches:\n", + " raise FileNotFoundError(\n", + " f\"No logs found under logs/{log_name}/. Run the MLP training cell first.\"\n", + " )\n", + " return pd.read_csv(matches[-1])\n", + "\n", + "metrics_mlp_accum = _load_latest_csv(\"mlp_accumulation\")\n", + "metrics_mlp_full = _load_latest_csv(\"mlp_full_batch\")\n", + "metrics_mlp_half = _load_latest_csv(\"mlp_half_batch\")\n", + "\n", + "_EMA = 10\n", + "\n", + "def _loss_per_opt_step(metrics, loss_col=\"train_loss\"):\n", + " if \"effective_step\" in metrics.columns:\n", + " sub = metrics[[\"effective_step\", loss_col]].dropna()\n", + " sub = sub.groupby(\"effective_step\")[loss_col].mean().reset_index()\n", + " return sub[\"effective_step\"], sub[loss_col]\n", + " sub = metrics[[\"step\", loss_col]].dropna()\n", + " return sub[\"step\"], sub[loss_col]\n", + "\n", + "def _plot_metric(ax, metrics, col, step_col, label, color):\n", + " subset = metrics[[col, step_col]].dropna()\n", + " if subset.empty:\n", + " return\n", + " ax.plot(subset[step_col], subset[col], alpha=0.2, color=color)\n", + " ax.plot(subset[step_col], subset[col].ewm(span=_EMA).mean(),\n", + " label=label, color=color, linewidth=1.8)\n", + "\n", + "fig, axes = plt.subplots(2, 2, figsize=(13, 8))\n", + "fig.suptitle(\n", + " f\"MLP/MNIST: Accum (micro={MNIST_MICRO_BATCH}→eff={MNIST_EFFECTIVE_BATCH}) \"\n", + " f\"vs Full batch ({MNIST_EFFECTIVE_BATCH})\"\n", + " f\" vs Half batch ({HALF_BATCH_SIZE})\",\n", + " fontsize=13,\n", + ")\n", + "\n", + "step_ma = \"analysis_step\" if \"analysis_step\" in metrics_mlp_accum.columns else \"step\"\n", + "step_mf = \"analysis_step\" if \"analysis_step\" in metrics_mlp_full.columns else \"step\"\n", + "step_mh = \"analysis_step\" if \"analysis_step\" in metrics_mlp_half.columns else \"step\"\n", + "\n", + "ax = axes[0, 0]\n", + "x_a, y_a = _loss_per_opt_step(metrics_mlp_accum)\n", + "x_f, y_f = _loss_per_opt_step(metrics_mlp_full)\n", + "x_h, y_h = _loss_per_opt_step(metrics_mlp_half)\n", + "ax.plot(x_a, y_a, alpha=0.2, color=\"steelblue\")\n", + "ax.plot(x_a, y_a.ewm(span=_EMA).mean(), color=\"steelblue\", linewidth=1.8,\n", + " label=f\"Accum micro={MNIST_MICRO_BATCH}\")\n", + "ax.plot(x_f, y_f, alpha=0.2, color=\"tomato\")\n", + "ax.plot(x_f, y_f.ewm(span=_EMA).mean(), color=\"tomato\", linewidth=1.8,\n", + " label=f\"Full batch={MNIST_EFFECTIVE_BATCH}\")\n", + "ax.plot(x_h, y_h, alpha=0.2, color=\"green\")\n", + "ax.plot(x_h, y_h.ewm(span=_EMA).mean(), color=\"green\", linewidth=1.8,\n", + " label=f\"Half batch={HALF_BATCH_SIZE}\")\n", + "ax.set_title(\"Train Loss\"); ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\"); ax.set_xscale(\"log\")\n", + "ax.legend(); ax.grid(True, alpha=0.3)\n", + "\n", + "for ax, col, title in [\n", + " (axes[0, 1], \"chi_net\", \"χ_net\"),\n", + " (axes[1, 0], \"chi_loss\", \"χ_loss\"),\n", + " (axes[1, 1], \"chi_coup\", \"Coupling (χ_coup)\"),\n", + "]:\n", + " _plot_metric(ax, metrics_mlp_accum, col, step_ma, f\"Accum micro={MNIST_MICRO_BATCH}\", \"steelblue\")\n", + " _plot_metric(ax, metrics_mlp_full, col, step_mf, f\"Full batch={MNIST_EFFECTIVE_BATCH}\", \"tomato\")\n", + " _plot_metric(ax, metrics_mlp_half, col, step_mh, f\"Half batch={HALF_BATCH_SIZE}\", \"green\")\n", + " ax.set_title(title); ax.set_xlabel(\"Effective optimizer step\")\n", + " ax.set_yscale(\"log\"); ax.set_xscale(\"log\")\n", + " ax.legend(); ax.grid(True, alpha=0.3)\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40bef94b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/batch_accumulation_notes.md b/examples/batch_accumulation_notes.md new file mode 100644 index 0000000..74e5a19 --- /dev/null +++ b/examples/batch_accumulation_notes.md @@ -0,0 +1,243 @@ +# Batch Accumulation in Perspic — Implementation Notes + +## What gradient accumulation does + +Instead of feeding a large batch of size N=K·B through the model in one forward pass, we +feed K micro-batches of size B and accumulate the resulting gradients before calling +`opt.step()`. The loss of each micro-batch is divided by K before `backward()` so that +the gradient that lands in `p.grad` after K steps is identical to the gradient you would +have obtained from one forward pass on the full N-sample batch. + +``` +gradient update = Σ_k grad( L_k / K ) + = (1/K) Σ_k grad( L_k ) # linearity of differentiation + = grad( (1/K) Σ_k L_k ) # same as mean loss over all N samples + = grad( L_full ) # ✓ +``` + +This is verified numerically: with Adam and SGD the max parameter-gradient difference +between the accumulated run and the full-batch run is < 2 × 10⁻⁸. + +--- + +## Issue 1 — χ_loss aggregation was wrong (K² error, now fixed) + +### Background: how χ_loss is normalised + +`SamplewiseCalculatorOpacus.compute()` (and the functorch variant) return two metrics with +`normalize=True`: + +| metric | formula returned | meaning | +|---|---|---| +| `chi_net` | `Σᵢ ‖∇_θ f(xᵢ)‖² / B` | average per-sample squared network-gradient norm | +| `chi_loss` | `Σᵢ ‖∇_f L(f(xᵢ),yᵢ)‖² · B` | scaled sum of per-sample squared loss-gradient norms | + +The `· B` factor in `chi_loss` looks like it makes it "extensive" (growing with B), but +this is cancelled by the `1/B` that appears inside `∇_f L` when the loss is mean-reduced +(PyTorch `cross_entropy` default). + +#### Why chi_loss is effectively intensive + +With mean-reduced cross-entropy, the per-sample loss gradient w.r.t. the network output +scales as **1/B**: + +``` +∂L_mean/∂f(xᵢ) = (1/B) · ∂L_individual(xᵢ)/∂f(xᵢ) +``` + +Substituting into the formula: + +``` +chi_loss(B) = Σᵢ ‖ (1/B) · gᵢ ‖² · B + = Σᵢ ‖gᵢ‖² / B² · B + = Σᵢ ‖gᵢ‖² / B +``` + +where `gᵢ = ∂L_individual(xᵢ)/∂f(xᵢ)` is independent of B. So `chi_loss(B)` scales as +`1/B`, and the product `(sum) · B` that the calculator returns is **constant w.r.t. B** +(at fixed data distribution). + +This can be verified empirically: `chi_loss(B=64) ≈ chi_loss(B=256) ≈ 0.9` for the same +model and data distribution. + +### The original wrong formula + +The original accumulation code used: + +```python +chi_loss_eff = K * sum(chi_loss_k) # WRONG +``` + +The reasoning was that `chi_loss` scales with B, so for N=K·B it should be multiplied by +K. But as shown above, chi_loss does *not* grow with B for mean-reduced losses. The +correct aggregation for an intensive quantity is the **mean**: + +``` +chi_loss(N=K·B) = Σ_{all N} ‖gᵢ‖² / N + = (K · Σ_{i in micro} ‖gᵢ‖²) / (K·B) + = (Σ_{i in micro} ‖gᵢ‖²) / B + = chi_loss(B) + = mean_k( chi_loss_k ) +``` + +### The error it caused + +| quantity | correct formula | old formula | error factor | +|---|---|---|---| +| `chi_net_eff` | `mean(chi_net_k)` | `mean(chi_net_k)` | 1 (correct) | +| `chi_loss_eff` | `mean(chi_loss_k)` | `K · sum(chi_loss_k)` | K² too large | +| `chi_coup` | `‖∇L‖² / (chi_loss · chi_net)` | (derived) | K² too small | + +With K=4 this produced a **16× discrepancy** in `chi_loss` and `chi_coup` compared to the +full-batch run, clearly visible in the bottom row of the comparison plot. + +### The fix + +```python +# was: +chi_loss_eff = K * sum(self._accum_chi_loss) +# now: +chi_loss_eff = sum(self._accum_chi_loss) / K +``` + +Same correction applies to `chi_loss_cross_eff` for the cross-response path. + +### Generalisation note + +If you use a **sum-reduced** loss (e.g. `reduction='sum'` in `cross_entropy`), then +`chi_loss` *would* scale as B and the correct formula would be `K · sum(chi_loss_k)`. +The right formula depends on the loss normalisation convention. With the default +mean-reduced loss, `mean` is correct. + +--- + +## Issue 2 — The train_loss curve looks different between runs (not a bug) + +The bottom-left panel of the comparison plot uses `global_step` (Lightning's micro-batch +counter) on the x-axis. This causes a visual mismatch between the two runs: + +### X-axis misalignment + +| run | steps per opt.step() | opt steps after N global steps | +|---|---|---| +| full batch (B=256) | 1 | N | +| accumulation (K=4) | 4 | N/4 | + +After 1000 global steps, the full-batch run has taken 1000 optimizer steps, while the +accumulation run has only taken 250. Both runs make the same number of optimizer steps +eventually (at step 4000 for accumulation vs step 1000 for full batch), but the x-axis +stretches the accumulation curve by a factor of K=4. + +**This is a display artifact, not a computation error.** The analysis metrics (χ_net, +χ_loss, χ_coup) use `analysis_step = effective_step` (the optimizer step count) and are +therefore correctly aligned. Only `train_loss` uses raw `global_step`. + +### Higher noise in the loss curve — a logging artifact, not a training difference + +The accumulation run *looks* noisier because `ClassificationModule.training_step` calls +`self.log("train_loss", loss)` on **every** call, which fires once per micro-batch. So +for K=4 accumulation the CSV gets four separate loss values per optimizer step, each +computed on a fresh B=64 mini-batch drawn from the DataLoader. + +The full-batch run logs once per optimizer step, each value computed on a B=256 batch. + +This creates two compounding visual effects: + +1. **Sampling variance.** A B=64 loss estimate has variance ≈ σ²/64; a B=256 estimate + has variance ≈ σ²/256 — four times smaller. So every individual logged point in the + accumulation run is noisier, which is reflected in the raw (faint) trace. + +2. **Four times as many points.** The EMA smoother sees four data points per optimizer + step instead of one, so it gets reset by each new noisy value before it can settle — + making the smoothed curve appear noisier too. + +**Neither effect reflects a real difference in what the optimizer is doing.** The +gradient that `opt.step()` acts on is the mean over all four micro-batches, which is +mathematically identical to the gradient from a single B=256 pass (verified numerically: +max parameter-gradient difference < 2 × 10⁻⁸). The apparent noise is entirely a +consequence of *what is being logged*, not of what is being computed. + +**Why `train_loss` can't be re-logged as a mean — Lightning key de-duplication:** + +Lightning's `CSVLogger` writes one row per `global_step` (micro-batch counter). When +`self.log(...)` is called anywhere during a `training_step`, the value is accumulated +into the *current step's* metrics dict and flushed to the CSV at the end of that step. +The wrapped model's `training_step` logs `train_loss` (a single micro-batch loss) first; +any subsequent call to `self.log("train_loss", mean_loss)` at cycle-end loses because +Lightning does not let a later log overwrite an earlier one for the same key and step. +The cycle-end mean is silently dropped — the raw micro-batch loss always appears in the +CSV. + +**First plot fix attempted — `.last()` per `effective_step` group:** + +When `effective_step` is logged only at the cycle end, Lightning **forward-fills** its +value into the next `K-1` rows (the first three micro-batches of cycle N+1). So +`effective_step=N` appears in four CSV rows: + +- `global_step` = 4N (micro-batch 4 of cycle N — cycle end, `effective_step` logged here) +- `global_step` = 4N+1 (micro-batch 1 of cycle N+1 — **forward-filled**) +- `global_step` = 4N+2 (micro-batch 2 of cycle N+1 — **forward-filled**) +- `global_step` = 4N+3 (micro-batch 3 of cycle N+1 — **forward-filled**) + +This means `.groupby("effective_step").last()` picks `global_step = 4N+3` — a +micro-batch from the *next* cycle, not the mean. `.first()` picks `global_step = 4N`, +which is the 4th micro-batch loss of cycle N (not a mean either, since the mean log +was dropped). Neither `.first()` nor `.last()` gives the true per-cycle mean. + +**Second fix — `.mean()` per `effective_step` group:** + +`.groupby("effective_step").mean()` averages the four rows that share `effective_step=N`. +Even though three of them belong to cycle N+1 (due to forward-fill), this is a +sliding-window average over four consecutive micro-batches and reduces noise by roughly +4×. Empirically: accumulation EMA residual std = 0.052 vs full-batch 0.053 — essentially +equal. But the averaging window is shifted by one micro-batch relative to the optimizer +step boundary. + +**Correct fix — log `effective_step` on every micro-batch:** + +The cleanest solution is to tag each micro-batch with its cycle's `effective_step` at +the time it runs, eliminating any dependence on Lightning's forward-fill: + +```python +# Inside training_step, for every micro-batch during accumulation: +# opt.step() has not fired yet so +1 gives the current cycle number +self.log("effective_step", float(self._optimizer_step_count + 1), + on_step=True, on_epoch=False) +``` + +With this in place, all K micro-batches of cycle N carry `effective_step=N` directly. +`groupby("effective_step").mean()` then averages exactly those K micro-batches — +the true mean loss over the effective batch of size K·B. No forward-fill, no shifted +window, no dropped values. + +### Is the optimizer step counter affected? + +No. PyTorch optimizers (Adam, SGD, …) maintain an internal step counter `t` that +increments by 1 on each call to `opt.step()`. Adam uses `t` for bias correction: + +``` +m̂ = m / (1 - β₁ᵗ), v̂ = v / (1 - β₂ᵗ) +``` + +With gradient accumulation, `opt.step()` is called exactly as many times as with a +full-batch run (once per effective batch), so `t` is identical between the two runs at +any given optimizer step. Adam's bias correction is unaffected. The gradient accumulation +does not introduce any error via the optimizer's internal state. + +### Summary of the implemented fix + +`analyzer.py` now logs `effective_step` on **every** micro-batch (not just at +cycle-end), using `_optimizer_step_count + 1` so all K micro-batches of cycle N share +the same `effective_step = N` value without relying on Lightning's forward-fill. + +The notebook's `loss_per_opt_step()` helper then applies: + +```python +sub = metrics[["effective_step", "train_loss"]].dropna() +sub = sub.groupby("effective_step")["train_loss"].mean().reset_index() +``` + +This averages exactly the K micro-batch losses from the same optimizer step, giving one +data point per optimizer step with variance σ²/(K·B) — the same as a direct B·K batch. +The full-batch run needs no special treatment. All four plot panels use `effective_step` +on the x-axis, making the curves directly comparable. diff --git a/examples/batch_size_scaling_analysis.ipynb b/examples/batch_size_scaling_analysis.ipynb index 0395363..624f1be 100644 --- a/examples/batch_size_scaling_analysis.ipynb +++ b/examples/batch_size_scaling_analysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "297431a4", "metadata": {}, "outputs": [], @@ -495,7 +495,7 @@ ], "metadata": { "kernelspec": { - "display_name": "lightning", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -509,7 +509,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.11" + "version": "3.13.7" } }, "nbformat": 4, diff --git a/examples/measurement_batch_sweep.ipynb b/examples/measurement_batch_sweep.ipynb new file mode 100644 index 0000000..f58ccd1 --- /dev/null +++ b/examples/measurement_batch_sweep.ipynb @@ -0,0 +1,725 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e6c70d6a", + "metadata": {}, + "source": [ + "# Measurement Batch-Size Sweep\n", + "\n", + "Demonstrates `measure_dataloader` / `measure_batch_size` / `measure_subset_seed`: an\n", + "independent, sweepable batch size for the cross-response measurement side, decoupled\n", + "from the training batch/accumulation. See `measurement_batch_sweep.md` for the full\n", + "design.\n", + "\n", + "`measure_dataloader.batch_size` is the GPU's max single-pass capacity. Swept sizes at\n", + "or below it are measured with a single direct pass; sizes above it are gradient-\n", + "accumulated. One sweep, one training run, one plot." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ee8fd900", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:50:56.606900Z", + "iopub.status.busy": "2026-07-15T15:50:56.606823Z", + "iopub.status.idle": "2026-07-15T15:51:03.639421Z", + "shell.execute_reply": "2026-07-15T15:51:03.639180Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 7\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import pytorch_lightning as pl\n", + "import torch\n", + "import torch.nn as nn\n", + "from pytorch_lightning.loggers import CSVLogger\n", + "from torch.utils.data import DataLoader, random_split\n", + "from torchvision.datasets import MNIST\n", + "from torchvision.transforms import ToTensor\n", + "\n", + "from examples.models import ClassificationModule\n", + "from perspic import analyzer, logarithmic_windows\n", + "\n", + "pl.seed_everything(7)\n", + "\n", + "PATH_DATASETS = os.environ.get(\"PATH_DATASETS\", \".\")\n", + "NUM_WORKERS = int(os.cpu_count() / 2)" + ] + }, + { + "cell_type": "markdown", + "id": "b3ae66f4", + "metadata": {}, + "source": [ + "## Data\n", + "\n", + "A train split and a held-out measurement split, both from MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ca0205ce", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:03.640310Z", + "iopub.status.busy": "2026-07-15T15:51:03.640134Z", + "iopub.status.idle": "2026-07-15T15:51:03.679401Z", + "shell.execute_reply": "2026-07-15T15:51:03.679040Z" + } + }, + "outputs": [], + "source": [ + "mnist_full = MNIST(PATH_DATASETS, train=True, download=True, transform=ToTensor())\n", + "\n", + "# Train / measurement split -- the measurement set stays fixed and disjoint from training\n", + "generator = torch.Generator().manual_seed(42)\n", + "train_set, measurement_set = random_split(mnist_full, [55000, 5000], generator=generator)\n", + "\n", + "TRAIN_BATCH_SIZE = 64\n", + "MEASURE_MAX_SINGLE_PASS = 16 # measure_dataloader.batch_size = GPU max-capacity chunk\n", + "\n", + "train_dataloader = DataLoader(\n", + " train_set, batch_size=TRAIN_BATCH_SIZE, shuffle=True,\n", + " num_workers=NUM_WORKERS, drop_last=True,\n", + ")\n", + "measure_dataloader = DataLoader(\n", + " measurement_set, batch_size=MEASURE_MAX_SINGLE_PASS, shuffle=True,\n", + " num_workers=NUM_WORKERS, drop_last=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ce6cf252", + "metadata": {}, + "source": [ + "## Model + analyzer\n", + "\n", + "Sweep measurement batch sizes both below and above the max single-pass size (16): 4\n", + "and 8 run as a single direct pass, 32/64/128 are gradient-accumulated. A sparse\n", + "`analysis_schedule` keeps the sweep cheap -- recommended whenever sweeping more than\n", + "one size (`analyzer()` warns if you sweep without one)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "acd4baf5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:03.680423Z", + "iopub.status.busy": "2026-07-15T15:51:03.680311Z", + "iopub.status.idle": "2026-07-15T15:51:03.686139Z", + "shell.execute_reply": "2026-07-15T15:51:03.685880Z" + } + }, + "outputs": [], + "source": [ + "model = nn.Sequential(\n", + " nn.Flatten(),\n", + " nn.Linear(28 * 28, 128),\n", + " nn.ReLU(),\n", + " nn.Linear(128, 10),\n", + ")\n", + "\n", + "MEASURE_BATCH_SIZES = [4, 8, 16, 32, 64, 128]\n", + "MAX_STEPS = 200\n", + "\n", + "analyzed_model = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " model=model,\n", + " lr=1e-3,\n", + " measure_dataloader=measure_dataloader,\n", + " measure_batch_size=MEASURE_BATCH_SIZES,\n", + " measure_subset_seed=0,\n", + " analysis_schedule=logarithmic_windows(\n", + " max_steps=MAX_STEPS, points_per_decade=3, base_window=1\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "73285f2a", + "metadata": {}, + "source": [ + "## Train" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "3f1a449b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:03.686909Z", + "iopub.status.busy": "2026-07-15T15:51:03.686829Z", + "iopub.status.idle": "2026-07-15T15:51:33.931293Z", + "shell.execute_reply": "2026-07-15T15:51:33.930993Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TPU available: False, using: 0 TPU cores\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "HPU available: False, using: 0 HPUs\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tikhome/jscheunemann/usr/miniconda3/envs/mast/lib/python3.13/site-packages/pytorch_lightning/trainer/configuration_validator.py:70: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.\n", + "You are using a CUDA device ('NVIDIA GeForce RTX 4090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + " | Name | Type | Params | Mode \n", + "---------------------------------------------\n", + "0 | model | Sequential | 101 K | train\n", + "---------------------------------------------\n", + "101 K Trainable params\n", + "0 Non-trainable params\n", + "101 K Total params\n", + "0.407 Total estimated model params size (MB)\n", + "5 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "83ba3e3aa3b84cc9811c806f9f4398ef", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00:0: UserWarning: Full backward hook is firing when gradients are computed with respect to module outputs since no inputs require gradients. See https://docs.pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.register_full_backward_hook for more details.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`Trainer.fit` stopped: `max_steps=200` reached.\n" + ] + } + ], + "source": [ + "trainer = pl.Trainer(\n", + " max_steps=MAX_STEPS,\n", + " accelerator=\"auto\",\n", + " devices=1,\n", + " logger=CSVLogger(save_dir=\"logs/\", name=\"measurement_batch_sweep\"),\n", + " log_every_n_steps=1,\n", + ")\n", + "trainer.fit(analyzed_model, train_dataloaders=train_dataloader)" + ] + }, + { + "cell_type": "markdown", + "id": "3404bf96", + "metadata": {}, + "source": [ + "## Sweep results\n", + "\n", + "Every analyzed step logs one `cross_*@bs{S}` metric set per swept size, in the same\n", + "row (no forward-fill needed). Take the last analyzed step and plot chi_net / chi_coup /\n", + "grad_dot_product against the swept batch size." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1604c946", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:33.932543Z", + "iopub.status.busy": "2026-07-15T15:51:33.932393Z", + "iopub.status.idle": "2026-07-15T15:51:34.302721Z", + "shell.execute_reply": "2026-07-15T15:51:34.302474Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABdEAAAGNCAYAAAD+VjiVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3XdYFFcXBvB3dylLRzooIgIizYaK2NCIYm+xJ9EYWyxRQ6KJsUeNSYwt1mhiixqNxsQkVuwF7BURG81GsQBKFfZ+f5Ddz5UiKLCU9/c8++jO3Jk5M+xyl7N3zpUIIQSIiIiIiIiIiIiIiCgXqaYDICIiIiIiIiIiIiIqq5hEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiIiIiIiIiIiIiCgfTKITEREREREREREREeWDSXQiIiIiIiIiIiIionwwiU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIqIyTSKRYMyYMZoOQ6VVq1bw9PQs0WPMmDEDEomkRI9BRERERESFwyQ6ERERlXvr1q2DRCKBRCLBiRMncq0XQsDe3h4SiQSdO3fWQIT0ss2bN2PRokUaO35YWBhmzJiBqKgojcVAZVN6ejpevHhRYvsPDg7GjBkzkJiYWGLHeJ39+/djyJAh8PT0hEwmQ40aNfJte/v2bfTq1QtVqlSBvr4+mjdvjsOHD+fZdunSpXBzc4Ouri6qVq2KwMBApKSklNBZEBEREZUuJtGJiIiowpDL5di8eXOu5UePHsW9e/egq6urgajoVWUhiT5z5swynUSfMmUK0tLSNB1GpXD27Fl88MEHsLa2hp6eHnR1dWFvb4+xY8fi9u3bxXqs4OBgzJw5U6NJ9M2bN2Pz5s0wMTGBnZ1dvu3u3r0LX19fnDhxAhMmTMDcuXPx/PlztGvXDseOHVNr+8UXX+CTTz6Bp6cnFi9ejHfffRdLlixBz549S/p0iIiIiEoFk+hERERUYXTs2BHbtm1DVlaW2vLNmzfD29sbNjY2Goqs+HBkZ+WgpaUFuVyu6TAqtKysLIwZMwY+Pj6Ijo7GxIkT8c8//2D79u0YNWoUjh8/Di8vLyxbtkzToRarb775BsnJyTh58iTq1q2bb7tvv/0WiYmJOHr0KL766iuMGzcOwcHBsLW1xaeffqpq9/DhQyxYsAAffPABtm3bho8//hg//vgjFi5ciP379+Off/4pjdMiIiIiKlFMohMREVGF0b9/fzx+/BhBQUGqZZmZmdi+fTsGDBiQ5zYKhQKLFi2Ch4cH5HI5rK2tMWLECDx9+lSt3c6dO9GpUyfY2dlBV1cXTk5OmDVrFrKzs9Xa3bp1C++++y5sbGwgl8tRrVo19OvXD0lJSQCAqKgoSCQSrFu3LlcsEokEM2bMUD1X1sUOCwvDgAEDUKVKFTRv3ly1fuPGjfD29oaenh7MzMzQr18/3L17V22fyvrdV65cgZ+fH/T19eHs7Izt27cDyBml7+PjAz09Pbi6uuLAgQO54rp//z4++ugjWFtbQ1dXFx4eHlizZo1amyNHjkAikeD333/HnDlzUK1aNcjlcrRp00ZtNG+rVq2wa9cuREdHq0rwFFRO4mWbNm2Cq6sr5HI5vL29c42GjY6OxqhRo+Dq6go9PT2Ym5ujd+/eaiPO161bh969ewMAWrdurYrhyJEjqjZ79uyBn58fjIyMYGxsjEaNGuV5h0NYWBhat24NfX19VK1aFd9//32hzuPFixeYOXMmXFxcIJfLYW5ujubNm6u9bl+tif7hhx+qYn318fJrJiMjA9OnT4ezs7NqRPXEiRORkZHx2rhe99rt2bMnGjRooLZNly5dIJFI8Pfff6uWnT59GhKJBHv27FEtS0xMxPjx42Fvbw9dXV04Ozvju+++g0KhUNtfYd+PNWrUQOfOnbF//37Uq1cPcrkc7u7u2LFjx2vPU2nw4MHYvHkzdu/ejWPHjuGzzz5D586d0bNnT0yaNAkXL17EypUr8fnnn2PlypWF2ueSJUvg4eEBfX19VKlSBQ0bNlS9dmbMmIEJEyYAABwdHVU/v5dfn0V5T58/fx5NmzaFnp4eHB0dCx2jnZ0dtLW1X9vu+PHjqF+/PlxdXVXL9PX10bVrV1y4cAG3bt0CAISEhCArKwv9+vVT2175fMuWLYWKi4iIiKgs09J0AERERETFpUaNGvD19cVvv/2GDh06AMhJiCYlJaFfv3748ccfc20zYsQIrFu3DoMHD8bYsWMRGRmJpUuX4uLFizh58qQq2bRu3ToYGhoiMDAQhoaGOHToEKZNm4bk5GTMmzcPQE7CPiAgABkZGfjkk09gY2OD+/fv499//0ViYiJMTEze6Lx69+4NFxcXfPPNNxBCAADmzJmDqVOnok+fPhg6dCgSEhKwZMkStGzZEhcvXoSpqalq+6dPn6Jz587o168fevfujRUrVqBfv37YtGkTxo8fj48//hgDBgzAvHnz0KtXL9y9exdGRkYAgLi4ODRp0kQ1uaelpSX27NmDIUOGIDk5GePHj1eL9dtvv4VUKsXnn3+OpKQkfP/993jvvfdw+vRpAMDkyZORlJSEe/fuYeHChQAAQ0PD116Do0ePYuvWrRg7dix0dXWxfPlytG/fHmfOnFFN8nn27FkEBwejX79+qFatGqKiorBixQq0atUKYWFh0NfXR8uWLTF27Fj8+OOP+Oqrr+Dm5gYAqn/XrVuHjz76CB4eHpg0aRJMTU1x8eJF7N27V+2LmKdPn6J9+/bo2bMn+vTpg+3bt+OLL76Al5eX6rWXnxkzZmDu3LkYOnQoGjdujOTkZJw7dw4XLlxA27Zt89xmxIgR8Pf3V1u2d+9ebNq0CVZWVgByEtBdu3bFiRMnMHz4cLi5ueHq1atYuHAhbt68ib/++ivfmArz2m3RogV27tyJ5ORkGBsbQwiBkydPQiqV4vjx4+jatSuAnOSrVCpFs2bNAACpqanw8/PD/fv3MWLECFSvXh3BwcGYNGkSHj58qFbap7DvRyAn6d+3b198/PHHGDRoENauXYvevXtj7969+V5HpV9//RV//vknTp8+DQ8PDwA5cyekpKSoXo+PHj3CBx98AAsLC/Tu3RsdOnSAg4NDvvtcvXo1xo4di169emHcuHFIT0/HlStXcPr0aQwYMAA9e/bEzZs38dtvv2HhwoWwsLAAAFhaWgIo+nu6Y8eO6NOnD/r374/ff/8dI0eOhI6ODj766KMCz72wMjIyUKVKlVzL9fX1AQDnz5+Hi4uL6gsaPT29fNsRERERlXuCiIiIqJxbu3atACDOnj0rli5dKoyMjERqaqoQQojevXuL1q1bCyGEcHBwEJ06dVJtd/z4cQFAbNq0SW1/e/fuzbVcub+XjRgxQujr64v09HQhhBAXL14UAMS2bdvyjTUyMlIAEGvXrs21DoCYPn266vn06dMFANG/f3+1dlFRUUImk4k5c+aoLb969arQ0tJSW+7n5ycAiM2bN6uWhYeHCwBCKpWKU6dOqZbv27cvV2xDhgwRtra24tGjR2rH6tevnzAxMVFdl8OHDwsAws3NTWRkZKjaLV68WAAQV69eVS3r1KmTcHBwyPca5XVdAIhz586plkVHRwu5XC569OihWpbXzygkJEQAEBs2bFAt27ZtmwAgDh8+rNY2MTFRGBkZCR8fH5GWlqa2TqFQqP6vvKYv7zMjI0PY2NiId99997XnU7duXbXXYV6UP/v83Lp1S5iYmIi2bduKrKwsIYQQv/76q5BKpeL48eNqbVeuXCkAiJMnT+a7v8K8ds+ePSsAiN27dwshhLhy5YoAIHr37i18fHxU7bp27Srq16+vej5r1ixhYGAgbt68qba/L7/8UshkMhETEyOEKNr70cHBQQAQf/zxh2pZUlKSsLW1VTt2XhQKhXB0dBSLFi1SLdu5c6ews7MTAET16tVV74XIyEghhBA9evQQX331VYH77datm/Dw8Ciwzbx589T2q/Qm7+n58+erlmVkZIh69eoJKysrkZmZWWAMLyvovdilSxdhamoqkpOT1Zb7+voKAOKHH34QQghx/vx5AUDMmjVLrZ3y52ZoaFjoeIiIiIjKKpZzISIiogqlT58+SEtLw7///otnz57h33//zbeUy7Zt22BiYoK2bdvi0aNHqoe3tzcMDQ1x+PBhVduXR1k+e/YMjx49QosWLZCamorw8HAAUI0037dvH1JTU4vtnD7++GO15zt27IBCoUCfPn3U4raxsYGLi4ta3EDOSO+XSy24urrC1NQUbm5u8PHxUS1X/j8iIgJAzsjcP/74A126dIEQQu1YAQEBSEpKwoULF9SONXjwYOjo6Kiet2jRQm2fb8rX1xfe3t6q59WrV0e3bt2wb98+VUmdl39GL168wOPHj+Hs7AxTU9NcceYlKCgIz549w5dffpmrHvnLpVWAnGv6/vvvq57r6OigcePGhTpPU1NTXLt2TVUOo6hSUlLQo0cPVKlSBb/99htkMhmAnNezm5sbateurfazeueddwAg1+viZYV57davXx+GhoaqMjrHjx9HtWrVMHDgQFy4cAGpqakQQuDEiROqn7syrhYtWqBKlSpqcfn7+yM7O1u1v6K8H4GcsiQ9evRQPTc2NsbAgQNx8eJFxMbG5nuu58+fR3x8PIYMGQIgp1xR//790bhxY/zxxx/49NNPc43m7t69u1rJn7yYmpri3r17OHv2bIHt8lLU97SWlhZGjBiheq6jo4MRI0YgPj6+2EZ+jxw5EomJiejbty8uXryImzdvYvz48Th37hwAqCa+bdCgAXx8fPDdd99h7dq1iIqKwp49ezBixAhoa2tzglwiIiKqEFjOhYiIiCoUS0tL+Pv7Y/PmzUhNTUV2djZ69eqVZ9tbt24hKSlJVQ7jVfHx8ar/X7t2DVOmTMGhQ4eQnJys1k5ZM9rR0RGBgYFYsGABNm3ahBYtWqBr1654//3337iUi3K/r8YthICLi0ue7V+td1ytWrVcSWATExPY29vnWgZAVX86ISEBiYmJWLVqFVatWpXnsV6+RkBOcvtlynIQr9a0Lqq8zrVWrVpITU1FQkICbGxskJaWhrlz52Lt2rW4f/++qvQN8P+fUUHu3LkDAKryMAXJ65pWqVIFV65cUT1/NZFrYmICPT09fP311+jWrRtq1aoFT09PtG/fHh988AHq1Knz2uMCwLBhw3Dnzh0EBwfD3NxctfzWrVu4fv26qjzIq179Wb2sMK9dmUwGX19fHD9+HEBOEr1FixZo3rw5srOzcerUKVhbW+PJkydqSfRbt27hypUrr42rKO9HAHB2ds71M6hVqxaAnLkH8ptI+Pz582jYsKGqbMumTZtQtWpVbN++XfWFhKmpKQYPHqzaxtraGgkJCXnuT+mLL77AgQMH0LhxYzg7O6Ndu3YYMGCAqqxNQYr6nrazs4OBgYHaspfPvUmTJq895ut06NABS5YswZdffqmqhe/s7Iw5c+Zg4sSJamWY/vjjD/Tt21f15YNMJkNgYCCOHj2KGzduvHUsRERERJrGJDoRERFVOAMGDMCwYcMQGxuLDh06qNUSfplCoYCVlRU2bdqU53pl0i8xMRF+fn4wNjbG119/DScnJ8jlcly4cAFffPGF2uSI8+fPx4cffoidO3di//79GDt2LObOnYtTp07lmXhVenWC0pe9WmtYoVCoJm5UJv1e9mqN8bzaFLRcmXxWntf777+PQYMG5dn21cTv6/ZZkj755BOsXbsW48ePh6+vL0xMTCCRSNCvX79cE1i+rcKcp62trdq6tWvX4sMPP0TLli1x584d1Wvk559/xsKFC7Fy5UoMHTq0wOMuXrwYv/32GzZu3Ih69eqprVMoFPDy8sKCBQvy3PbVL01e9brXLgA0b94cc+bMQXp6Oo4fP47JkyfD1NQUnp6eOH78OKytrQFALYmuUCjQtm1bTJw4Mc/jKpO/hX0/vq3Hjx/Dzs5O9TwqKgr169dX+5k2btxYbZu7d++qfWGRFzc3N9y4cQP//vsv9u7diz/++APLly/HtGnTMHPmzAK3Lep7urSMGTMGgwcPxpUrV6Cjo4N69erhl19+AfD/nxsAVK1aFSdOnMCtW7cQGxsLFxcX2NjYwM7OTq0dERERUXnFJDoRERFVOD169MCIESNw6tQpbN26Nd92Tk5OOHDgAJo1a5YrUf2yI0eO4PHjx9ixYwdatmypWh4ZGZlney8vL3h5eWHKlCkIDg5Gs2bNsHLlSsyePVs1MjsxMVFtm+jo6EKfn5OTE4QQcHR0LNEElaWlJYyMjJCdnZ1rUsu3kd8XCQXJq/TJzZs3oa+vr0qubt++HYMGDcL8+fNVbdLT03Nd6/yO7+TkBAAIDQ2Fs7NzkWN8VVBQkNpz5QSWAGBmZobBgwdj8ODBeP78OVq2bIkZM2YUmEQ/fvw4Pv/8c4wfPx7vvfdenvFfvnwZbdq0eaNrDBT82gVykuOZmZn47bffcP/+fVWyvGXLlqokeq1atVTJdGVcz58/f+1rqLDvR6Xbt29DCKF2rjdv3gSQM8lwfoyNjdXuTLCxscGZM2fU2rxclkcIgV9++aVQ7wEDAwP07dsXffv2RWZmJnr27Ik5c+Zg0qRJkMvlBb72ivKefvDgAVJSUtRGoxfm3N+EgYEBfH19Vc8PHDgAPT29PEfYu7i4qEbTh4WF4eHDh/jwww+LNR4iIiIiTWBNdCIiIqpwDA0NsWLFCsyYMQNdunTJt12fPn2QnZ2NWbNm5VqXlZWlSr4qR4a+PMo4MzMTy5cvV9smOTkZWVlZasu8vLwglUqRkZEBICeBZ2FhoaoDrfTqvgrSs2dPyGQyzJw5M9cIbyEEHj9+XOh9FUQmk+Hdd9/FH3/8gdDQ0FzrX1feIj8GBgaFKq/yspCQELW65nfv3sXOnTvRrl071c9HJpPluh5LlizJNcpfmXh8Nbnerl07GBkZYe7cuUhPT1db9yYj6f39/dUeypHpr/58DA0N4ezsrHqN5OXhw4fo06cPmjdvjnnz5uXZpk+fPrh//z5Wr16da11aWhpSUlLy3X9hXrtATt18bW1tfPfddzAzM1N9MdCiRQucOnUKR48eVRuFrowrJCQE+/bty3XcxMRE1XEL+35UevDgAf7880+1c9iwYQPq1auXbykXIGfE+NmzZ1V3J3Tr1g0XL17EtGnTEBERgePHj2PChAkAgIsXL+Ldd9/FvXv3MG7cuHz3CeT+uero6MDd3R1CCLx48QJA/q+9or6ns7Ky8NNPP6meZ2Zm4qeffoKlpaXa3AHFLTg4GDt27MCQIUMKLFGlUCgwceJE6Ovr55rTgYiIiKg84kh0IiIiqpDyKz/yMj8/P4wYMQJz587FpUuX0K5dO2hra+PWrVvYtm0bFi9ejF69eqFp06aoUqUKBg0ahLFjx0IikeDXX3/Nlew6dOgQxowZg969e6NWrVrIysrCr7/+qkpGKw0dOhTffvsthg4dioYNG+LYsWOqUaSF4eTkhNmzZ2PSpEmIiopC9+7dYWRkhMjISPz5558YPnw4Pv/888JfrAJ8++23OHz4MHx8fDBs2DC4u7vjyZMnuHDhAg4cOIAnT54UeZ/e3t7YunUrAgMD0ahRIxgaGhb4ZQeQU6c8ICAAY8eOha6urupLh5fLZHTu3Bm//vorTExM4O7ujpCQEBw4cCBXGY569epBJpPhu+++Q1JSEnR1dfHOO+/AysoKCxcuxNChQ9GoUSMMGDAAVapUweXLl5Gamor169cX+Vzz4u7ujlatWsHb2xtmZmY4d+4ctm/fjjFjxuS7zdixY5GQkICJEydiy5Ytauvq1KmDOnXq4IMPPsDvv/+Ojz/+GIcPH0azZs2QnZ2N8PBw/P7779i3bx8aNmyY5/4L+9rV19eHt7c3Tp06hS5duqhGVrds2RIpKSlISUnJlUSfMGEC/v77b3Tu3BkffvghvL29kZKSgqtXr2L79u2IioqChYVFod+PSrVq1cKQIUNw9uxZWFtbY82aNYiLi8PatWsLvP7NmzdHZmYm/v77b3Tv3h1169bF7NmzMWXKFMyaNQtaWlqYP38+xo0bh549e6Jdu3Y4duwYLCwsCtxvu3btYGNjg2bNmsHa2hrXr1/H0qVL0alTJxgZGQGAKsE9efJk9OvXD9ra2ujSpUuR39N2dnb47rvvEBUVhVq1amHr1q24dOkSVq1alat++quuXLmCv//+G0DOaP6kpCTVnQZ169ZVvRejo6PRp08fdO3aFTY2Nrh27RpWrlyJOnXq4JtvvlHb57hx45Ceno569erhxYsX2Lx5M86cOYP169fnmieBiIiIqFwSREREROXc2rVrBQBx9uzZAts5ODiITp065Vq+atUq4e3tLfT09ISRkZHw8vISEydOFA8ePFC1OXnypGjSpInQ09MTdnZ2YuLEiWLfvn0CgDh8+LAQQoiIiAjx0UcfCScnJyGXy4WZmZlo3bq1OHDggNrxUlNTxZAhQ4SJiYkwMjISffr0EfHx8QKAmD59uqrd9OnTBQCRkJCQ5/n88ccfonnz5sLAwEAYGBiI2rVri9GjR4sbN26o2vj5+QkPD49CXwsAYvTo0WrL4uLixOjRo4W9vb3Q1tYWNjY2ok2bNmLVqlWqNocPHxYAxLZt29S2jYyMFADE2rVrVcueP38uBgwYIExNTQUA4eDgkOf5vRrTxo0bhYuLi9DV1RX169dXXXelp0+fisGDBwsLCwthaGgoAgICRHh4uHBwcBCDBg1Sa7t69WpRs2ZNIZPJ1H6GQgjx999/i6ZNmwo9PT1hbGwsGjduLH777TfV+vyu6aBBg157LkIIMXv2bNG4cWNhamoq9PT0RO3atcWcOXNEZmamqo3yZ//yMQHk+Xj5NZOZmSm+++474eHhIXR1dUWVKlWEt7e3mDlzpkhKSso3psK+doUQYsKECQKA+O6779SWOzs7CwDizp07ubZ59uyZmDRpknB2dhY6OjrCwsJCNG3aVPzwww9q5y1E4d6Pytfvvn37RJ06dYSurq6oXbt2rtdffqZPny5q1qwpnjx5olp2//59cezYMREbGyuEEOLEiRMiPj6+UPsTQoiffvpJtGzZUpibmwtdXV3h5OQkJkyYkOu6z5o1S1StWlVIpVIBQERGRqrWFeU9fe7cOeHr6yvkcrlwcHAQS5cuLVScyt+XeT1efp88efJEdOvWTdjY2AgdHR3h6OgovvjiC5GcnJznPuvWrSsMDAyEkZGRaNOmjTh06FChrx0RERFRWScRohRmeSIiIiIiIiomNWrUgKenJ/7999832j49PR3NmjWDTCbDzp07c00Cq7R9+3b06NEj34lkNaFVq1Z49OhRniWWiIiIiKhksCY6ERERERFVKnK5HLt374ZEIoGrqyu++OILHDt2DNHR0QgPD8eGDRvg6+uLQYMGqdXiJyIiIqLKiTXRiYiIiIio0rG2tsbx48exdOlSLF26FN9//71qnVwuR48ePbBhwwa4uLhoMEoiIiIiKguYRCciIiIiokpJR0cHgYGBCAwMRFRUFO7fvw+5XA43Nzfo6+trOjwiIiIiKiNYE52IiIiIiIiIiIiIKB+siU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiIiIiIiIiIiIiCgfTKITEREREREREREREeWDSXQiIiIiIiIiIiIionwwiU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiIiIiIiIiIiIiCgfTKITEREREREREREREeWDSXQiIiIiIiIiIiIionwwiU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiN5aq1at4Onp+dp2UVFRkEgkWLduXckHRURERLmwzyYiIipb1q1bB4lEgqioqLfe14wZMyCRSN4+qHKoRo0a+PDDDzUdBlVgTKITUaUTHByMGTNmIDExUdOhEBEREREREWnc8uXL+eV5ATZv3oxFixZpOgzSIC1NB0BElYeDgwPS0tKgra2t0TiCg4Mxc+ZMfPjhhzA1NdVoLERERGVRWemziYiIqHQsX74cFhYWHM2dj82bNyM0NBTjx4/XdCikIRyJTlRGpKSkaDqEEieRSCCXyyGTyTQdChER0Rtjn01ERFQ0FbHvrIjnpAm8jlReMIlOVELu37+PIUOGwM7ODrq6unB0dMTIkSORmZmpqnl29OhRjBo1ClZWVqhWrZpq2+XLl8PDwwO6urqws7PD6NGjc5UeuXXrFt59913Y2NhALpejWrVq6NevH5KSklRtgoKC0Lx5c5iamsLQ0BCurq746quvinwue/bsgZ+fH4yMjGBsbIxGjRph8+bNudqFhYWhdevW0NfXR9WqVfH999+rrX+T+qpHjhyBRCLB77//jjlz5qBatWqQy+Vo06YNbt++nav96dOn0b59e5iYmEBfXx9+fn44efKkav2MGTMwYcIEAICjoyMkEkmx1Z8jIqLyiX128fTZAJCeno4ZM2agVq1akMvlsLW1Rc+ePXHnzh1Vm5SUFHz22Wewt7eHrq4uXF1d8cMPP0AIUajjSyQSzJgxQ/VcWf81PDwcffr0gbGxMczNzTFu3Dikp6cXKX4iIiqcitJ3KhQKzJgxA3Z2dtDX10fr1q0RFhaWq752QecUHR2NUaNGwdXVFXp6ejA3N0fv3r3z/Bvz2rVreOedd6Cnp4dq1aph9uzZUCgURYpZ6cSJE2jUqBHkcjmcnJzw008/5dkuKysLs2bNgpOTE3R1dVGjRg189dVXyMjIULWpUaMGrl27hqNHj6r+Rm7VqlWhY5FIJBgzZgw2bdoEV1dXyOVyeHt749ixY2rtlH12WFgYBgwYgCpVqqB58+aFjhMAhBCYPXs2qlWrpvqZXbt2LVdM+dWHz68GfUGfoVq1aoVdu3YhOjpadX1q1KhR6OtDFQPLuRCVgAcPHqBx48ZITEzE8OHDUbt2bdy/fx/bt29Hamqqqt2oUaNgaWmJadOmqb59nTFjBmbOnAl/f3+MHDkSN27cwIoVK3D27FmcPHkS2trayMzMREBAADIyMvDJJ5/AxsYG9+/fx7///ovExESYmJjg2rVr6Ny5M+rUqYOvv/4aurq6uH37tlpCuTDWrVuHjz76CB4eHpg0aRJMTU1x8eJF7N27FwMGDFC1e/r0Kdq3b4+ePXuiT58+2L59O7744gt4eXmhQ4cOb31Nv/32W0ilUnz++edISkrC999/j/feew+nT59WtTl06BA6dOgAb29vTJ8+HVKpFGvXrsU777yD48ePo3HjxujZsydu3ryJ3377DQsXLoSFhQUAwNLS8q1jJCKi8od9dvH12dnZ2ejcuTMOHjyIfv36Ydy4cXj27BmCgoIQGhoKJycnCCHQtWtXHD58GEOGDEG9evWwb98+TJgwAffv38fChQvf+Ph9+vRBjRo1MHfuXJw6dQo//vgjnj59ig0bNrzxPomIKLeK1HdOmjQJ33//Pbp06YKAgABcvnwZAQEB+X4Jm9c5nT17FsHBwejXrx+qVauGqKgorFixAq1atUJYWBj09fUBALGxsWjdujWysrLw5ZdfwsDAAKtWrYKenl6RfwZXr15Fu3btYGlpiRkzZiArKwvTp0+HtbV1rrZDhw7F+vXr0atXL3z22Wc4ffo05s6di+vXr+PPP/8EACxatAiffPIJDA0NMXnyZADIc18FOXr0KLZu3YqxY8dCV1cXy5cvR/v27XHmzJlck5r37t0bLi4u+Oabb1RfohcmTgCYNm0aZs+ejY4dO6Jjx464cOEC2rVrh8zMzCLF+7LXfYaaPHkykpKScO/ePdVnFUNDwzc+HpVTgoiK3cCBA4VUKhVnz57NtU6hUIi1a9cKAKJ58+YiKytLtS4+Pl7o6OiIdu3aiezsbNXypUuXCgBizZo1QgghLl68KACIbdu25RvDwoULBQCRkJDwxueRmJgojIyMhI+Pj0hLS8t1Hkp+fn4CgNiwYYNqWUZGhrCxsRHvvvuuallkZKQAINauXVvoGA4fPiwACDc3N5GRkaFavnjxYgFAXL16VRWPi4uLCAgIUIstNTVVODo6irZt26qWzZs3TwAQkZGRhY6DiIgqJvbZxddnr1mzRgAQCxYsyLVOGcNff/0lAIjZs2erre/Vq5eQSCTi9u3brz0+ADF9+nTV8+nTpwsAomvXrmrtRo0aJQCIy5cvF/ociIjo9SpK3xkbGyu0tLRE9+7d1ZbPmDFDABCDBg1SLcvvnITI+ZvzVSEhIbn62/HjxwsA4vTp06pl8fHxwsTEpMh/n3bv3l3I5XIRHR2tWhYWFiZkMpl4OdV36dIlAUAMHTpUbfvPP/9cABCHDh1SLfPw8BB+fn6FjuFlAAQAce7cOdWy6OhoIZfLRY8ePVTLlH12//791bYvbJzK11CnTp3UPt989dVXuX5mymO9SvmzVF7vwn6G6tSpk3BwcCjcBaEKieVciIqZQqHAX3/9hS5duqBhw4a51r98O9GwYcPUao0eOHAAmZmZGD9+PKRSqVo7Y2Nj7Nq1CwBgYmICANi3b5/aN/0vU06YuXPnzje+PSwoKAjPnj3Dl19+Cblcnu95ADnfwr7//vuq5zo6OmjcuDEiIiLe6NivGjx4MHR0dFTPW7RoAQCq/V+6dAm3bt3CgAED8PjxYzx69AiPHj1CSkoK2rRpg2PHjr3xdSAiooqJfXaO4uqz//jjD1hYWOCTTz7JtU4Zw+7duyGTyTB27Fi19Z999hmEENizZ88bH3/06NFqz5Vx7N69+433SURE6ipS33nw4EFkZWVh1KhRasvz6sfyOycAaiPJX7x4gcePH8PZ2Rmmpqa4cOGCat3u3bvRpEkTNG7cWLXM0tIS7733XpHizs7Oxr59+9C9e3dUr15dtdzNzQ0BAQFqbZV9YGBgoNryzz77DABU17w4+Pr6wtvbW/W8evXq6NatG/bt24fs7Gy1th9//PEbxal8DX3yySdqr7W3meyzKJ+hqHJjEp2omCUkJCA5OTnX7Up5cXR0VHseHR0NAHB1dVVbrqOjg5o1a6rWOzo6IjAwED///DMsLCwQEBCAZcuWqdWH69u3L5o1a4ahQ4fC2toa/fr1w++//16kDxjK+qWFOZdq1arl6mCqVKmCp0+fFvp4BXn5w4Fy3wBU+7916xYAYNCgQbC0tFR7/Pzzz8jIyFC7PkREROyz/684+uw7d+7A1dUVWlr5V4yMjo6GnZ0djIyM1Ja7ubmp1r8pFxcXtedOTk6QSqWc94SIqBhVpL5TeTxnZ2e15WZmZqq/N193TgCQlpaGadOmqeb6sLCwgKWlJRITE9Vijo6OztVXAbmvx+skJCQgLS2tUPuKjo6GVCrNdY42NjYwNTV9q373VXnFU6tWLaSmpiIhIUFteV6vjcLEqfz31WNZWlrm+zN7naJ8hqLKjUl0Ig16k9pnSvPnz8eVK1fw1VdfIS0tDWPHjoWHhwfu3bun2vexY8dw4MABfPDBB7hy5Qr69u2Ltm3b5voWuDi8+m28knhpkrCS3L/yw9K8efMQFBSU54M1y4iI6E2xzy5d+Y38Ksr14OgxIiLNqkh9p1Je5/TJJ59gzpw56NOnD37//Xfs378fQUFBMDc3LzN3Q5e1PjG/10ZxxlkcnyWIXsYkOlExs7S0hLGxMUJDQ4u8rYODAwDgxo0basszMzMRGRmpWq/k5eWFKVOm4NixYzh+/Dju37+PlStXqtZLpVK0adMGCxYsQFhYGObMmYNDhw7h8OHDhYrHyckJAN7oXEqbMlZjY2P4+/vn+dDW1gZQ9j5AEBGRZrDPLl5OTk64ceMGXrx4kW8bBwcHPHjwAM+ePVNbHh4erloP/P+Os8TERLV2BY2YU96VpnT79m0oFArUqFGjsKdARESvUZH6TuXxbt++rbb88ePHRbo7a/v27Rg0aBDmz5+PXr16oW3btmjevHmuPszBwSFXXwXkvh6vY2lpCT09vULty8HBAQqFIlfbuLg4JCYmql3zt/07Oa94bt68CX19fVhaWha4bWHjVP77aruEhIRcP7PCfpYo7Gco5hGISXSiYiaVStG9e3f8888/OHfuXK71BY3y8vf3h46ODn788Ue1dr/88guSkpLQqVMnAEBycjKysrLUtvXy8oJUKkVGRgYA4MmTJ7n2X69ePQBQtXmddu3awcjICHPnzs01O3lZGq0GAN7e3nBycsIPP/yA58+f51r/8u1jBgYGAHJ3pkREVLmwzy5e7777Lh49eoSlS5fmWqeMoWPHjsjOzs7VZuHChZBIJOjQoQOAnC/FLSwscOzYMbV2y5cvz/f4y5YtU3u+ZMkSAFDtk4iI3l5F6jvbtGkDLS0trFixQm15Xv1YQWQyWa7zXrJkSa4Rzx07dsSpU6dw5swZ1bKEhARs2rSpyMcLCAjAX3/9hZiYGNXy69evY9++fbmOCQCLFi1SW75gwQIAUF1zIOfv5Lf5GzkkJEStBvzdu3exc+dOtGvXLt+74Ioap3Jw3JIlS9Su+avbAf9Pjr/8WSIlJQXr169Xa1fYz1AGBgYsEVvJ5V+wkIje2DfffIP9+/fDz88Pw4cPh5ubGx4+fIht27bhxIkT+W5naWmJSZMmYebMmWjfvj26du2KGzduYPny5WjUqJFqErBDhw5hzJgx6N27N2rVqoWsrCz8+uuvkMlkePfddwEAX3/9NY4dO4ZOnTrBwcEB8fHxWL58OapVq4bmzZsX6jyMjY2xcOFCDB06FI0aNcKAAQNQpUoVXL58Gampqbk6H02SSqX4+eef0aFDB3h4eGDw4MGoWrUq7t+/j8OHD8PY2Bj//PMPAKgmO5k8eTL69esHbW1tdOnSRZVcJyKiyoN9dvEZOHAgNmzYgMDAQJw5cwYtWrRASkoKDhw4gFGjRqFbt27o0qULWrdujcmTJyMqKgp169bF/v37sXPnTowfP171By8ADB06FN9++y2GDh2Khg0b4tixY7h582a+x4+MjETXrl3Rvn17hISEYOPGjRgwYADq1q1b4udORFSZVJS+09raGuPGjcP8+fNV/cfly5exZ88eWFhYFHrkcefOnfHrr7/CxMQE7u7uCAkJwYEDB2Bubq7WbuLEifj111/Rvn17jBs3DgYGBli1ahUcHBxw5cqVQh1LaebMmdi7dy9atGiBUaNGISsrC0uWLIGHh4favurWrYtBgwZh1apVSExMhJ+fH86cOYP169eje/fuaN26taqtt7c3VqxYgdmzZ8PZ2RlWVlZ45513Ch2Tp6cnAgICMHbsWOjq6qq++J45c+Zrty1snJaWlvj8888xd+5cdO7cGR07dsTFixdVP7OXtWvXDtWrV8eQIUMwYcIEyGQyrFmzBpaWlmpfPhT2M5S3tze2bt2KwMBANGrUCIaGhujSpUuhrw9VAIKISkR0dLQYOHCgsLS0FLq6uqJmzZpi9OjRIiMjQ6xdu1YAEGfPns1z26VLl4ratWsLbW1tYW1tLUaOHCmePn2qWh8RESE++ugj4eTkJORyuTAzMxOtW7cWBw4cULU5ePCg6Natm7CzsxM6OjrCzs5O9O/fX9y8ebPI5/L333+Lpk2bCj09PWFsbCwaN24sfvvtN9V6Pz8/4eHhkWu7QYMGCQcHB9XzyMhIAUCsXbu20Mc+fPiwACC2bdumtjy/fV28eFH07NlTmJubC11dXeHg4CD69OkjDh48qNZu1qxZomrVqkIqlQoAIjIystAxERFRxcI+u3j6bCGESE1NFZMnTxaOjo5CW1tb2NjYiF69eok7d+6o2jx79kx8+umnws7OTmhrawsXFxcxb948oVAocu1ryJAhwsTERBgZGYk+ffqI+Ph4AUBMnz5d1W769OkCgAgLCxO9evUSRkZGokqVKmLMmDEiLS2tSPETEVHhVJS+MysrS0ydOlXY2NgIPT098c4774jr168Lc3Nz8fHHH6vaFXROT58+FYMHDxYWFhbC0NBQBAQEiPDwcOHg4CAGDRqk1vbKlSvCz89PyOVyUbVqVTFr1izxyy+/vNHfpEePHhXe3t5CR0dH1KxZU6xcuVLVJ77sxYsXYubMmaq+2d7eXkyaNEmkp6ertYuNjRWdOnUSRkZGAoDw8/MrdCwAxOjRo8XGjRuFi4uL0NXVFfXr1xeHDx9Wa6eMLyEhIdc+Chtndna2mDlzprC1tRV6enqiVatWIjQ0NM/rff78eeHj4yN0dHRE9erVxYIFC1Q/y1ev9+s+Qz1//lwMGDBAmJqaCgBqn5uocpAIUcZqMhARERERERXSjBkzMHPmTCQkJOQahUZERFRUiYmJqFKlCmbPno3JkydrOpxyQSKRYPTo0UUuhUNUnrAmOhERERERERERVTppaWm5linra7dq1ap0gyGiMo010YkqqYSEhFwTnbxMR0cHZmZmJXb8zMzMPCeDeZmJiQn09PRKLAYiIqLygH02ERFR0RS279y6dSvWrVuHjh07wtDQECdOnMBvv/2Gdu3aoVmzZqUYcY7nz5/j+fPnBbaxtLR87USdxSU2NrbA9Xp6ejAxMSmVWIg0jUl0okqqUaNGiI6Ozne9n58fjhw5UmLHDw4OVpvEJC9r167Fhx9+WGIxEBERlQfss4mIiIqmsH1nnTp1oKWlhe+//x7JycmqyUZnz55ditH+3w8//PDaiTgjIyNRo0aNUonH1ta2wPWDBg3CunXrSiUWIk1jTXSiSurkyZN53rqmVKVKFXh7e5fY8Z8+fYrz588X2MbDw+O1nTYREVFFxz6biIioaDTdd76piIgIREREFNimefPmkMvlpRLPgQMHClxvZ2cHd3f3UomFSNOYRCciIiIiIiIiIiIiygcnFiUiIiIiIiIiIiIiykeFrYmuUCjw4MEDGBkZQSKRaDocIiIiNUIIPHv2DHZ2dpBK+Z12XtiXExFRWcf+/PXYnxMRUVlW2L68wibRHzx4AHt7e02HQUREVKC7d++iWrVqmg6jTGJfTkRE5QX78/yxPyciovLgdX15hU2iGxkZAci5AMbGxhqOhuj1srOzERwcDABo2rQpZDKZhiMiopKUnJwMe3t7VX9FubEvp/KI/TlR5cL+/PXYn1N5w76cqHIpbF9eYZPoytvEjI2N2VFTudGpUydNh0BEpYy3NeePfTmVV+zPiSof9uf5Y39O5RH7cqLK53V9OYu2ERERERERERERERHlg0l0IiIiIiIiIiIiIqJ8VNhyLkTljUKhwPXr1wEAbm5uBc4ITERERGUT+3MiIqLyjX05EeWFSXSiMkIIgYSEBABA7dq1NRxN8VIoFMjMzNR0GESlSltbu9QmIVq2bBnmzZuH2NhY1K1bF0uWLEHjxo3zbb9t2zZMnToVUVFRcHFxwXfffYeOHTuq1gshMH36dKxevRqJiYlo1qwZVqxYARcXF1WbOXPmYNeuXbh06RJ0dHSQmJiY7/EeP36MunXr4v79+3j69ClMTU2L47SJyqSK3J8TERFVBuzLiSgvTKK/RrZC4EzkE8Q/S4eVkRyNHc0gk3LSGKLCyszMRGRkJBQKhaZDISp1pqamsLGxKdHJxrZu3YrAwECsXLkSPj4+WLRoEQICAnDjxg1YWVnlah8cHIz+/ftj7ty56Ny5MzZv3ozu3bvjwoUL8PT0BAB8//33+PHHH7F+/Xo4Ojpi6tSpCAgIQFhYGORyOYCc93bv3r3h6+uLX375pcAYhwwZgjp16uD+/fvFfwGIqNzj520iIqLyj/05VXRMohdgb+hDzPwnDA+T0lXLbE3kmN7FHe09bTUYGVH5IITAw4cPIZPJYG9vz9vgqNIQQiA1NRXx8fEAAFvbkuszFixYgGHDhmHw4MEAgJUrV2LXrl1Ys2YNvvzyy1ztFy9ejPbt22PChAkAgFmzZiEoKAhLly7FypUrIYTAokWLMGXKFHTr1g0AsGHDBlhbW+Ovv/5Cv379AAAzZ84EAKxbt67A+FasWIHExERMmzYNe/bsKa7TJqIKgp+3iYiIyj/251QZMImej72hDzFy4wWIV5bHJqVj5MYLWPF+A/4iIHqNrKwspKamws7ODvr6+poOh6hU6enpAQDi4+NhZWVVIqVdMjMzcf78eUyaNEm1TCqVwt/fHyEhIXluExISgsDAQLVlAQEB+OuvvwAAkZGRiI2Nhb+/v2q9iYkJfHx8EBISokqiF0ZYWBi+/vprnD59GhEREUU4MyKqDPh5m4iIqPxjf06VBYeF5iFbITDzn7BcvwAAqJbN/CcM2Yq8WhCRUnZ2NgBAR0dHw5EQaYbyy6MXL16UyP4fPXqE7OxsWFtbqy23trZGbGxsntvExsYW2F75b1H2mZeMjAz0798f8+bNQ/Xq1Qu9TXJystqDiComft4mIiIq/9ifU2XCJHoezkQ+UbsF5VUCwMOkdJyJfFJ6QRGVYyVZD5qoLKvMr/1JkybBzc0N77//fqG3mTt3LkxMTFQPe3v7EoyQiDSJn7eJiIjKP/bnVJkwiZ6H+Gf5/wJ4k3ZEREQlwcLCAjKZDHFxcWrL4+LiYGNjk+c2NjY2BbZX/luUfebl0KFD2LZtG7S0tKClpYU2bdqoYp4+fXqe20yaNAlJSUmqx927dwt9PCIqX/h5m4iIqPxjf06VCZPoebAykhdrO6LCkEqlaNGiBVq0aMEJOKlQatSogUWLFhXrPtetWwdTU9Ni3WdZVxLXsbTo6OjA29sbBw8eVC1TKBQ4ePAgfH1989zG19dXrT0ABAUFqdo7OjrCxsZGrU1ycjJOnz6d7z7z8scff+Dy5cu4dOkSLl26hJ9//hkAcPz4cYwePTrPbXR1dWFsbKz2ICpv2J8XDj9vExFRWcW+vPDYn1NlwolF89DY0Qy2JnLEJqXnWddJAsDGRI7GjmalHRpVYBKJpEQmHqSK6+zZszAwMNB0GOVeeb+OgYGBGDRoEBo2bIjGjRtj0aJFSElJweDBgwEAAwcORNWqVTF37lwAwLhx4+Dn54f58+ejU6dO2LJlC86dO4dVq1YByPldNH78eMyePRsuLi5wdHTE1KlTYWdnh+7du6uOGxMTgydPniAmJgbZ2dm4dOkSAMDZ2RmGhoZwcnJSi/PRo0cAADc3t0r3RQ1VLuzPC4eft4mIqKxiX154jR3NYGMsR2xy/iPNbdmfUwXBJHoeZFIJpndxx8iNFyAB8vxgP72LO2TSylvrlog0z9LSUtMhVAjl/Tr27dsXCQkJmDZtGmJjY1GvXj3s3btXNTFoTEyM2giapk2bYvPmzZgyZQq++uoruLi44K+//oKnp6eqzcSJE5GSkoLhw4cjMTERzZs3x969eyGX/38EybRp07B+/XrV8/r16wMADh8+jFatWpXwWRNReffy5+388PM2ERFR2SaTSuBT0ww7Lz3It011M31kKwT7dCr3eF9KPtp72mLF+w1gY5L7lpOP/ZzQ3tNWA1FRRaZQKBAeHo7w8HAoFApNh1OptWrVCp988gnGjx+PKlWqwNraGqtXr1aN7jUyMoKzszP27Nmj2iY7OxtDhgyBo6Mj9PT04OrqisWLF6vWp6enw8PDA8OHD1ctu3PnDoyMjLBmzZo84xBCYMaMGahevTp0dXVhZ2eHsWPHqta/WoZEIpHg559/Ro8ePaCvrw8XFxf8/fffavv8+++/4eLiArlcjtatW2P9+vWQSCRITEzM93rs3LkTDRo0gFwuR82aNTFz5kxkZWXl2z4qKgoSiQRbtmxB06ZNIZfL4enpiaNHjxb6egHAkSNH0LhxYxgYGMDU1BTNmjVDdHQ0AODy5cto3bo1jIyMYGxsDG9vb5w7d+6tr+O6desgkUhyPWbMmKFq//PPP8PNzQ1yuRy1a9fG8uXL870WpWXMmDGIjo5GRkYGTp8+DR8fH9W6I0eOYN26dWrte/fujRs3biAjIwOhoaHo2LGj2nqJRIKvv/4asbGxSE9Px4EDB1CrVi21NuvWrYMQItcjvwR6q1atIITgKHSq8NifF157T1sEtq2Va7m+jgwr3m/Az9tERKQR7MsL7+6TVOy7FgsAMNHTVltnoqcNqQQ4HfkEH607i+T0F5oIkajYcCR6Adp72qKtuw3ORD5B/LN0HLgej38uP0BwxGMIISCR8Fs0Kj5CCMTG5nQ+Li4uGo6mZGVnZ+e7TiKRqI2aLagtALXb7PJr+ya34q1fvx4TJ07EmTNnsHXrVowcORJ//vknevToga+++goLFy7EBx98gJiYGOjr60OhUKBatWrYtm0bzM3NERwcjOHDh8PW1hZ9+vSBXC7Hpk2b4OPjg06dOqFz5854//330bZtW3z00Ud5xvDHH39g4cKF2LJlCzw8PBAbG4vLly8XGPfMmTPx/fffY968eViyZAnee+89REdHw8zMDJGRkejVqxfGjRuHoUOH4uLFi/j8888L3N/x48cxcOBA/Pjjj2jRogXu3Lmj+iIgv8khlSZMmIBFixbB3d0dCxYsQJcuXRAZGQlzc/PXXq+srCx0794dw4YNw2+//YbMzEycOXNG9Xv3vffeQ/369bFixQrIZDJcunQJ2traecZRlOvYt29ftG/fXvX8yJEj+OCDD9CsWTMAwKZNmzBt2jQsXboU9evXx8WLFzFs2DAYGBhg0KBBBV4PIqocKlN/Xhwep2QCAFrWsoCnnQmWH7kDuZYUbd0LP5ExERFRcWJfXnhf/xuG9BcK+DiaYdNQH5yNeor4Z+mwMsop4XLsVgJGb7qAE7cfoc/KEKwd3Ai2JnqaDpvojTCJ/hoyqQS+TuYAgKZOFth/LRaX7yYi5M5jNHW20HB0ROXT8ePH811nZmaGOnXqqJ6fPHky32//TU1NUa9ePdXzU6dO4cWL3N9uv0lpibp162LKlCkAgEmTJuHbb7+FhYUFhg0bBiCnlMWKFStw5coVNGnSBNra2pg5c6Zqe0dHR4SEhOD3339Hnz59AAD16tXD7NmzMXToUPTr1w/R0dH4999/840hJiYGNjY28Pf3h7a2NqpXr47GjRsXGPeHH36I/v37AwC++eYb/Pjjjzhz5gzat2+Pn376Ca6urpg3bx4AwNXVFaGhoZgzZ06++5s5cya+/PJLVYK4Zs2amDVrFiZOnPjaJPqYMWPw7rvvAgBWrFiBvXv34pdffsHEiRNfe72Sk5ORlJSEzp07q2pru7m5qV2bCRMmoHbt2gAK/nBblOuop6cHPb2cD3V37tzB6NGj8c0336Bt27YAcr44mD9/Pnr27KmKOywsDD/99BOT6ERERaRQCOwNzUlSDPKtgZa1LLH5TAyepL7Amcgnqs/gRFS8li1bhnnz5iE2NhZ169bFkiVLCvyMuWjRIqxYsQIxMTGwsLBAr169MHfuXLUyb0RU+RwIi0NQWBy0pBLM6u4JLZk0V9/d2tUKv4/wxeB1ZxEe+ww9lgVjzYeN4G5nrKGoid4cy7kUgaWRLvo2sgcALDtyW8PREFFJejmRL5PJYG5uDi8vL9UyZb3p+Ph41bJly5bB29sblpaWMDQ0xKpVqxATE6O2388++wy1atXC0qVLsWbNGpib558g6N27N9LS0lCzZk0MGzYMf/75Z4FlVF6N28DAAMbGxqoYb9y4gUaNGqm1f11S/vLly/j6669haGioegwbNgwPHz5EamoqPv74Y7V1L/P19VX9X0tLCw0bNsT169dVywq6XmZmZvjwww8REBCALl26YPHixXj48KFq28DAQAwdOhT+/v749ttvcefOnXzP4U2uozKB36lTJ0yYMAEAkJKSgjt37mDIkCFq5zx79uwCj09ERHm7eDcRscnpMNTVQnMXC2jLpGjnntO/7gl9+JqtiehNbN26FYGBgZg+fTouXLiAunXrIiAgQO0z7cs2b96ML7/8EtOnT8f169fxyy+/YOvWrfjqq69KOXIiKkvSMrMx459rAIAhzR1Ry9oo37aeVU3w56imcLYyRGxyOvr8FILjtxJKK1SiYsOR6EU0vGVNbD4dg5O3H+NizFPUr15F0yERlTstWrTId92rZZKUZTQKo0mTJm8c06teLQ0ikUjUlinjVI6S37JlCz7//HPMnz8fvr6+MDIywrx583D69Gm1/cTHx+PmzZuQyWS4deuWWumQV9nb2+PGjRs4cOAAgoKCMGrUKMybNw9Hjx7Nt3RJXnG/TR2/58+fY+bMmaqR1y+Ty+X4+uuvX1sSJi+FuV5r167F2LFjsXfvXmzduhVTpkxBUFAQmjRpghkzZmDAgAHYtWsX9uzZg+nTp2PLli3o0aNHrmMV9TpmZ2ejb9++MDY2xqpVq9SuBQCsXr1areY48GYlg4iIKrs9V3MS5f5uVtDVyvk92sHLFr+fu4c9obGY0cUDUk5CRlSsFixYgGHDhmHw4MEAgJUrV2LXrl1Ys2YNvvzyy1ztg4OD0axZMwwYMABAzlwy/fv3z/UZl4gql+VHbuPe0zTYmsgxts3rS95Uq6KPPz5uihEbz+FUxBMMXnsWc3t6oXdD+1KIlqh4cCR6EVWroo/u9asCAJYf4chDojchk8nyfbxcD/11bV9NXBamTUk5efIkmjZtilGjRqF+/fpwdnbOc3TyRx99BC8vL6xfvx5ffPGF2sjsvOjp6aFLly748ccfceTIEYSEhODq1atvFKOrq2uuyTfPnj1b4DYNGjTAjRs34OzsnOshlUphZWWltuxlp06dUv0/KysL58+fV5VkKez1ql+/PiZNmoTg4GB4enpi8+bNqnW1atXCp59+iv3796Nnz55Yu3ZtvudRlOv46aef4urVq/jrr7/UblO2traGnZ0dIiIicl0LR0fHAq8jERGpE0Jgz3+lXDp4/X8C0WZOFjCSayHhWQbOxzzVVHhEFVJmZibOnz8Pf39/1TKpVAp/f3+EhITkuU3Tpk1x/vx5nDlzBgAQERGB3bt355qU/GUZGRlITk5WexBRxRGR8Bw/HY0AAEzv4g4D3cKNzzXR18b6jxqjWz07ZCkEJmy/gkUHbkIIUZLhEhUbjkR/Ax/7OeGPC/cQFBaHG7HP4GqT/20rRFQ5uLi4YMOGDdi3bx8cHR3x66+/4uzZs2rJ1WXLliEkJARXrlyBvb09du3ahffeew+nTp2Cjo5Orn2uW7cO2dnZ8PHxgb6+PjZu3Ag9PT04ODi8UYwjRozAggUL8MUXX2DIkCG4dOkS1q1bByD3HQBK06ZNQ+fOnVG9enX06tULUqkUly9fRmhoKGbPnl3g8ZYtWwYXFxe4ublh4cKFePr0qWoS1dddr8jISKxatQpdu3aFnZ0dbty4gVu3bmHgwIFIS0vDhAkT0KtXLzg6OuLevXs4e/asqv7621zHtWvXYvny5fjzzz8hkUhUEwopS7fMnDkTY8eOhYmJCdq3b4+MjAycO3cOT58+RWBgYKF+DkREBFy5l4T7iWnQ15HBr5alarmOlhRt3a2x48J97L76EI1qmGkwSqKK5dGjR8jOzlaVJVSytrZGeHh4ntsMGDAAjx49QvPmzSGEQFZWFj7++OMCy7nMnTtXbe4bIqo4hBCYtvMaMrMV8KtliQCPok0Erqslw8I+9VDVVA/Lj9zBogO3cO9pGub29IK2jON8qWzjK/QNOFsZooNnzi+KFayNTkTISVD37NkTffv2hY+PDx4/foxRo0ap1oeHh2PChAlYvnw57O1zbllbvnw5Hj16hKlTp+a5T1NTU6xevRrNmjVDnTp1cODAAfzzzz8F1lEviKOjI7Zv344dO3agTp06WLFiBSZPngwA0NXVzXObgIAA/Pvvv9i/fz8aNWqEJk2aYOHChYVK5H/77bf49ttvUbduXZw4cQJ///03LCxyJmR+3fXS19dHeHg43n33XdSqVQvDhw/H6NGjMWLECMhkMjx+/BgDBw5ErVq10KdPH3To0CHfP9aKch2PHj2K7OxsdO3aFba2tqrHDz/8AAAYOnQofv75Z6xduxZeXl7w8/PDunXrOBKdiKiIdv9X8/yd2laQa6vfNdbRM2dk+t7QWCgUHJ1GpElHjhzBN998g+XLl+PChQvYsWMHdu3ahVmzZuW7zaRJk5CUlKR63L17txQjJqKStOvqQ5y4/Qg6WlLM7OqR72CsgkilEkxsXxtzenhCKgG2n7+Hj9adxbP0FyUQMVHxkYgKet9EcnIyTExMkJSUBGPj4p/1N/R+EjovOQGpBDjyeWtUN9cv9mNQ5SKEwIsXOZ2Gtrb2G3VGZU16ejoiIyPh6OioVhaDyo45c+Zg5cqVxfrHTVRUFBwdHXHx4kXUq1ev2PZbHhX0Hijpfqoi4DWi8qgi9ufFTQiBVj8cQfTjVCx/rwE6vlTOBQDSX2Sj4ewDeJ6RhT9HNeUcRFSmlae+KjMzE/r6+ti+fTu6d++uWj5o0CAkJiZi586dubZp0aIFmjRpgnnz5qmWbdy4EcOHD8fz589zlWLMS3m6RkQA+/L8PM/IQpv5RxCXnIHx/i4Y71/rrfd5KDwOozddRNqLbNS2McK6wY1hY8LcAZWuwvZTHIn+hjyrmsCvliUUAlh5jLXR6e1JJBLo6OhAR0eHnTSVmOXLl+Ps2bOIiIjAr7/+innz5mHQoEGaDouIqMJgf/56YQ+TEf04FXJtKVq5WuZaL9eW4Z3aVgCgqptORG9PR0cH3t7eOHjwoGqZQqHAwYMH4evrm+c2qampec5ZBIB1jKnCYl+et0VBNxGXnAEHc3187OdULPt8p7Y1to5oAgtDXYTHPkOP5ScRHst5FKhsYhL9LYxunTOJ3vZz9xCXnK7haIiIXu/WrVvo1q0b3N3dMWvWLHz22WeYMWOGpsMiIqJKZM/VnMR4q1pW0NfJe4qmjl45pRN3X33IRB1RMQoMDMTq1auxfv16XL9+HSNHjkRKSgoGDx4MABg4cCAmTZqkat+lSxesWLECW7ZsQWRkJIKCgjB16lR06dJFlUwnooovPDYZa4OjAAAzunrkKsX2NupUM8Wfo5rCydIAD5PS0XtFCE7eflRs+ycqLkVKoq9YsQJ16tSBsbExjI2N4evriz179qjWp6enY/To0TA3N4ehoSHeffddxMXFqe0jJiYGnTp1gr6+PqysrDBhwgRkZWWptTly5AgaNGgAXV1dODs7qya+K2saO5qhUY0qyMxW4OfjEZoOh8o5hUKBmzdv4ubNm1AoFJoOhyqohQsX4sGDB0hPT8fNmzcxdepUaGkV7xzTNWrUgBCi0pdyIaLKif15wYQQ2H01px56B6/8JyPzq2UFPW0Z7j1NQ+h9jkgjKi59+/bFDz/8gGnTpqFevXq4dOkS9u7dq5psNCYmBg8fPlS1nzJlCj777DNMmTIF7u7uGDJkCAICAvDTTz9p6hSIShz7cnUKhcCUP0ORrRBo72GD1q5WxX4MezN97BjZDI0dzfAsIwuD1pzBH+fvFftxiN5GkZLo1apVw7fffovz58/j3LlzeOedd9CtWzdcu3YNAPDpp5/in3/+wbZt23D06FE8ePAAPXv2VG2fnZ2NTp06ITMzE8HBwVi/fj3WrVuHadOmqdpERkaiU6dOaN26NS5duoTx48dj6NCh2LdvXzGdcvEa9d9o9E2nY/A0JVPD0VB5JoTAgwcP8ODBA464IiIiKqfYnxfsZtxzRDxKgY6WVFWyJS96Ov8v6aKchJSIiseYMWMQHR2NjIwMnD59Gj4+Pqp1R44cURvEpqWlhenTp+P27dtIS0tDTEwMli1bBlNT09IPnKiUsC9X98eFezgX/RT6OjJM6+JeYscx0dfGr0Mao0tdO2QpBD7bdhk/HrzFnwGVGUVKonfp0gUdO3aEi4sLatWqhTlz5sDQ0BCnTp1CUlISfvnlFyxYsADvvPMOvL29sXbtWgQHB+PUqVMAgP379yMsLAwbN25EvXr10KFDB8yaNQvLli1DZmZOAnrlypVwdHTE/Pnz4ebmhjFjxqBXr15YuHBh8Z99MWhVyxLutsZIzczGuv9ubSEiIiIiotyUo9BbuljCSK5dYFvlSPU9LOlCRESkEYmpmZi7JxwAMK6NC+xM9Ur0eLpaMizuW09Vc31B0E18+cdVvMjmHQGkeW9cEz07OxtbtmxBSkoKfH19cf78ebx48QL+/v6qNrVr10b16tUREhICAAgJCYGXl5fqVjEACAgIQHJysmo0e0hIiNo+lG2U+yhrJBKJqjb6uuAoPM/Ies0WRJUP//Clyoq3fxIRqdvz36jyjgWUclFq7WoFXS0poh6nIjz2WUmHRkRERK+Yt+8GnqRkwsXKEB81dyyVY0qlEnzZoTZmdfeEVAJsPXcXQ9afY76NNK7IhXCvXr0KX19fpKenw9DQEH/++Sfc3d1x6dIl6Ojo5Lqty9raGrGxOZMHxcbGqiXQleuV6wpqk5ycjLS0NOjp5f2tV0ZGBjIyMlTPk5NLr3Zie08b1LQwQMSjFGw+HY3hLYtnlmKi8k5bWxsSiQQJCQmwtLTkzOZUaQghkJmZiYSEBEilUujo6Gg6JCIijbsd/xw3455DWyZBGzfr17Y30NVCK1dL7LsWhz1XH8LN1rgUoiQiIiIAuHw3EZvPxAAAZnX3hLbsjcfhvpEPmjjAzkSOMZsv4tjNBPRZGYK1gxvB2lheqnEQKRU5ie7q6opLly4hKSkJ27dvx6BBg3D06NGSiK1I5s6di5kzZ2rk2DKpBB+3csLE7Vew+ngkBvrWKNaZionKK5lMhmrVquHevXuIiorSdDhEpU5fXx/Vq1eHVFq6HziJiMqivf+NQm/mbAETvYJLuSh18LTFvmtx2B0ai8B2riUZHhEREf0nWyEw5a9QCAH0rF8VTWqaaySONm7W2DK8CYasP4uwh8nosewk1g5uDFcbI43EQ5VbkZPoOjo6cHbOKV/i7e2Ns2fPYvHixejbty8yMzORmJioNho9Li4ONjY5t2va2NjgzJkzavuLi4tTrVP+q1z2chtjY+N8R6EDwKRJkxAYGKh6npycDHt7+6Ke3hvrXq8qFgXdxIOkdGw/fw/vN3EotWMTlWWGhoZwcXHBixcvNB0KUamSyWTQ0tLiHRhERP/ZfTXnztOOnraF3uYdNyvoyKS4Hf8ct+KewcWafzQTERGVtM2no3H1fhKM5FqY1NFNo7HUtTfFn6OaYdDaM4hISEGvlcH46X1vNHW20GhcVPkUOYn+KoVCgYyMDHh7e0NbWxsHDx7Eu+++CwC4ceMGYmJi4OvrCwDw9fXFnDlzEB8fDysrKwBAUFAQjI2N4e7urmqze/dutWMEBQWp9pEfXV1d6Orqvu3pvDEdLSmGt6yJGf+EYeXRO+jXyB5apXyrC1FZJZPJIJPx7gwiIqLKKupRCsIeJkMmlaCt++tLuSgZy7XRwsUCB8PjsftqLMYxiU5ERFSiEp5l4Pt9NwAAEwJcYWmkuVybkr2ZPnaMbIphG87hbNRTDFp7Bt/3qoMe9atpOjSqRIqU5Z00aRKOHTuGqKgoXL16FZMmTcKRI0fw3nvvwcTEBEOGDEFgYCAOHz6M8+fPY/DgwfD19UWTJk0AAO3atYO7uzs++OADXL58Gfv27cOUKVMwevRoVQL8448/RkREBCZOnIjw8HAsX74cv//+Oz799NPiP/ti1rdRdZgb6ODe0zT8ffmBpsOhckYqlaJJkyZo0qQJSz8QERGVU+zP87YnNGcUelMnc1QxKNo8ER28bP/bx8Nij4uIiOhVlb0vn7vnOp6lZ8GzqjHe8yk7VRZM9XXw6xAfdKpjixfZAp9uvYylh25BCKHp0KiSKNJvg/j4eAwcOBCurq5o06YNzp49i3379qFt27YAgIULF6Jz585499130bJlS9jY2GDHjh2q7WUyGf7991/IZDL4+vri/fffx8CBA/H111+r2jg6OmLXrl0ICgpC3bp1MX/+fPz8888ICAgoplMuOXo6MtVsxcuP3IFCwTcyFZ5EIoFcLodcLmf5ByIionKK/XnelAnwDkUo5aLU1s0aWlIJwmOfISLheXGHRkREpKYy9+WnIh5jx4X7kEiAWd08IZOWrfOXa8uwpF99jGhZEwDww/6b+OrPq8jKVmg4MqoMJKKCfmWTnJwMExMTJCUlwdjYuPSOm/4CzeYewrOMLKx83xvtPW1K7dhERFR+aKqfKk94jYgqhrtPUtHi+8OQSoAzk/1hYVj028IHrTmDozcTMCHAFaNbO5dAlERvhn3V6/EaEZUPL7IV6PTjcdyMe44BPtXxTQ8vTYdUoA0hUZjx9zUoBNDK1RJLBzSAoe5bV62mSqiw/VTluy+lhBnLtTGwac7tLsuP3OZtJVRoCoUCd+7cwZ07d6BQ8FtUIiKi8oj9eW77ruWUcvFxNH+jBDoAdPTKGZjCki5ERFTSKmtfvvZkJG7GPYeZgQ4mBrhqOpzXGuhbAyvf94ZcW4ojNxLQ96cQxCenazosqsCYRC8Bg5s5Qq4txZV7SThx+5Gmw6FyQgiBu3fv4u7du/zyhYiIqJxif57b7qs5iW9lIvxNtHW3gUwqQej9ZMQ8Ti2u0IiIiHKpjH35w6Q0LDpwCwDwZYfaMNUv2vwlmtLOwwZbhvvC3EAH1x4ko8fyYNyKe6bpsKiCYhK9BFgY6qJfo+oAgGWHb2s4GiIiIiIizXiYlIYLMYmQSIAAjzdPopsZ6KBJTTMAHI1ORERU3Gb9G4bUzGx4O1RBrwbVNB1OkdSzN8Wfo5qhpoUB7iemoeeKYITceazpsKgCYhK9hAxvWRNaUglORTzB+egnmg6HiIiIiKjU7Q3NKeXS0KEKrIzlb7Uv5aSku//bJxEREb29IzfisftqLGRSCWZ394S0jE0mWhjVzfXxx8im8HaogmfpWRi05gx2Xrqv6bCogmESvYTYmeqhZ4OqAIDlh+9oOBoiIiIiotK352pOwluZAH8bAR42kEiAy3cTce8pS7oQERG9rfQX2Zj+9zUAwIdNa8DNtvxO/lvFQAebhvqgo5cNMrMVGLflEucqpGLFJHoJ+tjPCRIJcDA8HmEPkjUdDhERERFRqYlPTsfZ/+7IbO/55qVclCyNdNG4Rk5Jl70cjU5ERPTWfjoagejHqbAy0sV4fxdNh/PW5NoyLO3fAEObOwIAvt97A5P/CkVWduWZIJZKDpPoJaimpSE6euWMullxlKPRiYiIiKjy2HctFkIA9aubws5Ur1j2qfxszSQ6ERHR24l+nIJlR3Lm8Zva2R1Gcm0NR1Q8pFIJpnR2x/Qu7pBIgM2nYzBswzmkZGRpOjQq55hEL2GjWjkBAHZdeYDIRykajoaIiIiIqHTs/q+US8diKOWipBzRfi76KWKT0ottv0RERJWJEAIz/r6GzCwFmjtboHOd4uury4rBzRyx8n1v6GpJcfhGAvqtOoX4Z/zsQG+OSfQS5mFngtaullAI4CeORqcCSKVSNGrUCI0aNYJUyrcmERFRecT+PMfj5xk4HfkYQPGUclGyNpbD26EKgJyR7kRERMWtMvTl+67F4fCNBGjLJJjZzQMSSfmbTLQwAjxs8NvwJjAz0MHV+0nosSwYt+OfaTosKqcq5m+DMmZ0a2cAwB8X7uFhUpqGo6GySiKRwMDAAAYGBhW2AyMiIqro2J/n2B8WB4UAvKqawN5Mv1j33eG/pPzuqw+Ldb9ERERAxe/LUzOz8PU/OZOJjmjpBCdLQw1HVLIaVK+CHSObooa5Pu4npqHn8mCcjnis6bCoHGISvRQ0rGGGxo5meJEtsPpYpKbDISIiIiIqUcoEdwev4huFrtThv7roZ6KeIOFZRrHvn4iIqCL78eBtPEhKR1VTPdWgz4quhoUBdoxqhgbVTZGcnoUPfjmDvy8/0HRYVM4wiV5KlL+YfjsTg8fP+WGfclMoFIiKikJUVBQUCs4cTUREVB6xPweepmQi+E7OCK8OxVgPXamqqR7q2ptCCJZ0ISKi4leR+/Jbcc/w8/EIAMDMrh7Q05FpOKLSY2agg83DmqC9hw0ysxUY+9tFrDx6B0IITYdG5QST6KWkpYsFPKsaI+1FNtYFR2k6HCqDhBCqjpq/xImIiMon9udA0PU4ZCsE3GyN4WhhUCLH6PhfSZc9oSzpQkRExaui9uVCCEzdGYoshYC/mzX83a01HVKpk2vLsOy9BviomSMA4Ns94TnXJLtifVlCJYNJ9FIikUgwulXOaPR1wVF4lv5CwxERERERERW/Pf+VculYjBOKvko5wv1UxBM8ScksseMQERFVFDsvPcCpiCeQa0sxvYu7psPRGJlUgmld3DG1szskEmDjqRiM+PU8UjOzNB0alXFMopeiAA8bOFka4Fl6FjaeitF0OERERERExSop7QVO3H4E4P+1y0tCdXN9eFY1RrZCICiMJV2IiIgKkpT2ArN3XQcAfPKOS7FP+l0eDWnuiOUDGkBXS4qD4fHot+oU51opJ7IVAiF3HmPnpfsIufMY2YrSuWOESfRSJJVKMPK/0ei/nIhA+otsDUdERERERFR8DoXH4UW2QC1rQzhbGZbosZSj0XdfZRKdiIioIAuDbuLR8wzUtDDA0BaOmg6nzOjgZYvNw3xQRV8bV+4locfyk7gd/1zTYVEB9oY+RPPvDqH/6lMYt+US+q8+hebfHcLeUijxxyR6KetWzw5VTfXw6Hkmfj93V9PhEBEREREVG2VCu30JTCj6qg7/lYs5efsRklJZKpGIiCgvofeTsCEkCgDwdTdP6GpVnslEC8PbwQw7RjWDg7k+7j1Nw7srgnE26ommw6I87A19iJEbL+BhUrra8tikdIzceKHEE+lMopcybZkUI/xqAgB+OhqBF5y8gIiIiIgqgOcZWTh6MwEA0NGr5OqhK9W0NERtGyNkKQSCrseV+PGIiIjKG4VCYMpfoVAIoEtdOzR3sdB0SGWSo4UBdoxsinr2pkhKe4H3fj6Nf6880HRY9JJshcDMf8KQV+EW5bKZ/4SVaGkXJtE1oE9De1gY6uJ+Yhp2XuKbkoiIiIjKv0Ph8cjMUqCmhQFcrY1K5ZjKki7KyUyJiIjo/7aeu4tLdxNhqKuFKZ3cNB1OmWZuqIvfhjVBO3drZGYpMGbzRaw6dgdClE69bcqfQiGw+UxMrhHoLxMAHial40xkyd1FwCS6Bsi1ZaoaVMuP3C61AvhUtkmlUjRo0AANGjSAVMq3JhERUXlUmftzZSK7g5cNJBJJqRxTOeL9+K1HSE5nSRciInp7FaUvf5KSie/2hgMAPm1bC9bGcg1HVPbp6ciw4n1vfNi0BgDgm93hmP73NebtNCD9RTYOhcdh0o6raDL3IKb+FVqo7eKf5Z9of1vl97dBOfeeT3UYy7UQkZCC/dc4GRIBEokExsbGMDY2LrU/PImoYli2bBlq1KgBuVwOHx8fnDlzpsD227ZtQ+3atSGXy+Hl5YXdu3errRdCYNq0abC1tYWenh78/f1x69YttTZz5sxB06ZNoa+vD1NT01zHePz4Mdq3bw87Ozvo6urC3t4eY8aMQXJy8lufL1FZVln789TMLBy+EQ/g/6PDS4OLtRGcrQyRma3AoevxpXZcIiKquCpKX/7dnnAkpr5AbRsjDPJ10HQ45YZMKsH0Lu6qkfsbQqIx4tfzSMvM1nBkFd+TlExsO3cXI349h/pfB+Gjdefw25kYxD/LgFy7cClsK6OS+7KISXQNMZJrq77ZWnbkNm8PISKiN7J161YEBgZi+vTpuHDhAurWrYuAgADEx+edTAoODkb//v0xZMgQXLx4Ed27d0f37t0RGvr/b/a///57/Pjjj1i5ciVOnz4NAwMDBAQEID39/9/qZ2Zmonfv3hg5cmSex5FKpejWrRv+/vtv3Lx5E+vWrcOBAwfw8ccfF+8FIKIy4ciNBKS/UKC6mT487IxL9dgd/5tgdE8JTyZFRERUXpyPfoKt5+4CAOb08ISWjOm/opBIJBjaoiaWv9cAOlpSHLgeh36rT+HR8wxNh1bhRCQ8x6pjd9B7ZTAazg7ChO1XsO9aHNJeZMPWRI4Pmjhgw0eNcWFqW9iayJHf11oSALYmcjR2NCuxWCWigmZvk5OTYWJigqSkJBgbl+4H+cJ6kpKJZt8eQtqLbKz/qDH8allqOiTSIIVCgXv37gEAqlWrVq5vGyOi1yuufsrHxweNGjXC0qVLAeT8LrG3t8cnn3yCL7/8Mlf7vn37IiUlBf/++69qWZMmTVCvXj2sXLkSQgjY2dnhs88+w+effw4ASEpKgrW1NdatW4d+/fqp7W/dunUYP348EhMTXxvrjz/+iHnz5uHu3buFOrfy0JcTvaqy9uef/HYR/1x+gBF+NTGpQ+nWXA17kIyOPx6HrpYUF6a2hYGuVqkenyo39lWvx2tE5U1578uzshXovOQEwmOfoU/Davi+V11Nh1SunYt6gqEbziEx9QWqm+lj3eBGqGlpqOmwyq1shcDFmKcIuh6HA2FxuJOQorbe3dYYbd2t0dbdGh526neD7A19iJEbLwCA2gSjyhYr3m+A9m9wR2Rh+6ny9ZuggjEz0MEAn+oAgGWHb2s4GtI0IQQiIiIQERHBOxOIqFAyMzNx/vx5+Pv7q5ZJpVL4+/sjJCQkz21CQkLU2gNAQECAqn1kZCRiY2PV2piYmMDHxyfffRbGgwcPsGPHDvj5+eXbJiMjA8nJyWoPovKmMvbn6S+yceh6HIDSLeWi5GZrhBrm+sjIUqhKyhAREb2p8t6XbwiJRnjsM5joaeOL9rU1HU6517CGGXaMbIrqZvqIeZKKniuCcS6q5CavrIjSMrOx/1osJmy7jMZzDqDXyhD8dDQCdxJSoC2ToIWLBb7u5oGTX76D3eNa4NO2teBZ1SRXOaX2nrZY8X4D2Jiol2yxMZG/cQK9KJhE17BhLWpCWybBmcgnOMs3IRERFcGjR4+QnZ0Na2trteXW1taIjc17vo3Y2NgC2yv/Lco+C9K/f3/o6+ujatWqMDY2xs8//5xv27lz58LExET1sLe3L/LxiKj0HbuZgJTMbNiZyFG3mkmpH18ikaCDV84fTXuucq4hotcpylwqrVq1gkQiyfXo1KlTKUZMRIUVl5yOBUE3AQBftK8Nc0NdDUdUMdS0NMSOUU1Rt5oJElNfYMDPp7H7KsvIFST+WTq2nInB0PVnUe/r/Rj+63lsO38Pj1MyYSTXQrd6dljSvz7OT22LX4f4YKBvDVQ11Xvtftt72uLEF+/gt2FNsLhfPfw2rAlOfPFOiSfQASbRNc7GRI5e3tUAAMs5Gp2IiCqYhQsX4sKFC9i5cyfu3LmDwMDAfNtOmjQJSUlJqkdhy74QkWbtCc1JXHfwstXYBGwd//vD6VB4PCf+IipAUedS2bFjBx4+fKh6hIaGQiaToXfv3qUcOREVxpxd1/E8Iwt17U3RrxEHpBQnC0Nd/Da8CfzdrJGZpcDozRfw8/HyebdCSRBC4FbcMyw7fBs9lp+EzzcH8eWOqzhwPR4ZWQpUNdXDh01rYPNQH1yY2haL+9VHl7p2MJZrF/lYMqkEvk7m6FavKnydzCGTls7nTxYMLANGtHTC1rN3cfhGAq49SIKHXemP4CEiovLHwsICMpkMcXFxasvj4uJgY2OT5zY2NjYFtlf+GxcXB1tbW7U29erVK3KMNjY2sLGxQe3atWFmZoYWLVpg6tSpavtW0tXVha4uR8sQlScZWdk4EJbzO6WjV96/d0qDZ1VjVKuih3tP03D0ZnypjEYiKo8WLFiAYcOGYfDgwQCAlStXYteuXVizZk2ec6mYmalP0LZlyxbo6+sziU5UBp28/Qh/X34AqQSY090T0lJKLFYm+jpa+OkDb8z85xo2hERj9q7ruPc0DVM7u5daIrcsycpW4Fz0UxwIi0PQ9ThEP05VW1+3mgn83azh726N2jZGGhtsUVw4Er0MqGFhgM517AAAy4/c0XA0RERUXujo6MDb2xsHDx5ULVMoFDh48CB8fX3z3MbX11etPQAEBQWp2js6OsLGxkatTXJyMk6fPp3vPgtLoVAAyKl9TkQVw8nbj/AsIwvWxrqob19FY3FIJBJ0/K+ky26WdCHK05vMpfKqX375Bf369YOBgUFJhUlEbyAjKxtTd4YCAD5o4gDPqhycWVJkUglmdvXA5I45E6mvC47CqE3nK82dcM8zsrD76kMEbr2EhnMOoN+qU/j5RCSiH6dCRyZFK1dLzOnhiVOT2mDnmOb4pI0L3GyNy30CHeBI9DJjZCsn/H35AXZffYiIhOec6ZeIiAolMDAQgwYNQsOGDdG4cWMsWrQIKSkpqhFmAwcORNWqVTF37lwAwLhx4+Dn54f58+ejU6dO2LJlC86dO4dVq1YByElEjR8/HrNnz4aLiwscHR0xdepU2NnZoXv37qrjxsTE4MmTJ4iJiUF2djYuXboEAHB2doahoSF2796NuLg4NGrUCIaGhrh27RomTJiAZs2aoUaNGqV5iYioBCkT1h08bTU+4q29pw1WHYvAofB4pL/IhlxbptF4iMqaguZSCQ8Pf+32Z86cQWhoKH755ZcC22VkZKh9Yc6JwolK3s/HIxGRkAILQ10EtnPVdDgVnkQiwbCWNWFrKkfg1svYdy0O/Vefwi+DGlbIOvSxSek4cD0OQWFxCLnzGJnZCtU6U31tvFPbCm3drNGiliUMdStuqrninlk542ZrDH83Kxy4Ho+VR+/g+151NR0SERGVA3379kVCQgKmTZuG2NhY1KtXD3v37lX9gRwTEwOp9P83njVt2hSbN2/GlClT8NVXX8HFxQV//fUXPD09VW0mTpyIlJQUDB8+HImJiWjevDn27t0Lufz/s6BPmzYN69evVz2vX78+AODw4cNo1aoV9PT0sHr1anz66afIyMiAvb09evbsmeet4kRUPr3IViDov1IuHTw1V8pFqV41U9iayPEwKR0nbj2Cv7v16zciokL75Zdf4OXlhcaNGxfYbu7cuZg5c2YpRUVEd5+kYsmhWwCAyZ1qw0Sv6DWm6c10rmMHKyM5hm04h0t3E9FzRTDWDW4MR4vyfbeOEALhsc8QFBaHA9fjcOVektp6B3N9tHWzRlt3a3g7VIGWrHIUOpGICloBPzk5GSYmJkhKSoKxsbGmwymUCzFP0XN5MLSkEhyd2LpQs9JSxSGEQFJSzi8mExOTCnGrCxHlrzz2U6WN14jKo8rUnx+7mYCBa87AwlAHp7/yLxO1QGf+cw1rT0ahZ4OqWNCnnqbDoUqgPPVVmZmZ0NfXx/bt29XuLhs0aBASExOxc+fOfLdNSUmBnZ0dvv76a4wbN67A4+Q1Et3e3r5cXCMioPz15cM2nENQWBx8HM2wZXiTMh9vRXQ7/jkGrzuDu0/SUEVfGz8PagRvB82VuXsTL7IVOBP5RJU4v/c0TbVOIgHq25vC390a7dyt4WRpWKFeZ4XtyzkSvQxpUL0KfGuaIyTiMVYfi8CMrh6aDolKkUQigampqabDICIiordQmfrzPaEPAQABHjZlIoEOAB29bLH2ZBSCwuKQmaWAjlblGBlFVBgvz6WiTKIr51IZM2ZMgdtu27YNGRkZeP/99197HE4UTuVdeerLD4TllNjQkkowu7tnhUpslifOVobYMbIZhqw/iyv3kjBg9Sks7levzE90npz+AkduJOBAWBwO34jHs/Qs1TpdLSlauFigrbs1Wte2gpWRvIA9VQ5Mopcxo1s7IyTiMbacjcGYd5xhUQFrKRERERFR+ZaVrcC+azmlXJQTepYF3tWrwMpIF/HPMnDyziO0drXSdEhEZUpR51JR+uWXX9C9e3eYm5trImwiykNaZjZm/HMNADCkhSNcrI00HFHlZmmkiy3Dm+CTzRdxMDweIzddwNRO7viouaOmQ1Nz72kqDl6PR1BYHE5FPEaW4v8FSswNdNDGzQr+btZo4WIJPR3OL/MyJtHLmGbO5qhbzQSX7yVh7clITAioremQqJQoFAo8fJgzosvW1lathjERERGVD5WlPz8T+QRPUjJRRV8bPo5mmg5HRSqVoL2nDTaERGPP1YdMohO9oqhzqQDAjRs3cOLECezfv18TIROVuvLSly8/chv3nqbBzkSOse+4aDocAqCvo4WfPvDGjH+uYeOpGHz9bxjuPU3D5E5uGrtrTwiB0PvJCLoehwNhcQh7qD7Zs5OlAdq626CtuxXq2VcpM3cXlkVMopcxEokEo1o7Y8Sv57EhOBoj/JxgLOekEJWBEAK3buVMBmJjo/nJuYiIiKjoKkt/vvulUi5lbTKpDp622BASjf1hcZiTrYB2GYuPSNPGjBmTb/mWI0eO5Frm6uqKCjqVGlGeykNfHpHwHD8djQAATOviDgNdpvfKCi2ZFLO6eaJaFX18uycca05G4kFiGhb1qwe5dumM7M7IysapiCcICovFgbB4xCanq9ZJJUBDBzP4u+eMOK9paVgqMVUEfJeVQW3drOFiZYhb8c/xa0g0Rrd21nRIREREREQAgGyFwN7QnFIuHcpQKRelxo5mMDfQweOUTJyOeILmLhaaDomIiKjYCCEwbec1ZGYr0MrVEgEeZTPRX5lJJBJ87OcEO1M9fP77Zey9FosBq0/h50GNYGagUyLHTEzNxOEb8TgQFo+jNxPwPOP/9c31tGVoWcsCbd1t0NrVEuYsHf1GmEQvg6RSCUa1dsKnWy9jzYlIfNTMkXWIiIiIiKhMOBf1BI+eZ8BETxtNncpefWSZVIJ2Hjb47UwMdoc+ZBKdiIgqlF1XH+LE7UfQ0ZJiZlcPTiZahnWtawdrI10M23AOF2IS0XP5Sawb3Bg1LAyKZf8xj1OxPywWB67H4WzUU2S/VN/c0kgX/m7WaOtuhaZOFqU2Cr4iYxK9jOpSxw7z99/Evadp2Ho2Bh82K1sTERARERFR5bQnNBYA4O9mXWZLpXT0ykmi7wuNxaxunqzvSUREFcLzjCzM+jcMADCqlRMczIsnGUslx6emOXaMaopBa84i6nEqeq4Ixs+DGqJB9SrIVgiciXyC+GfpsDKSo7GjWYGfWRQKgcv3EnHgehyCwuJwM+652npXayO0dbeGv7s16lQ1gZSff4oVk+hllJZMio/9nDDlr1D8dCwCA3wcoKNVNv9IISIiIqLKQaEQ2PtfEr2jV9m9fbxJTXOY6mvjcUomzkQ+gW8ZHDFPRERUVIuCbiIuOQMO5vr42M9J0+FQITlbGeHP0U3x0bqzCL2fjP6rTuHDpjXw9+UHeJj0/3rltiZyTO/ijvae/y+Xl/4iG8F3HiEoLA4Hrscj4VmGap1MKkHjGmbwd7dGWzdrVDfXL9XzqmyYRC/DenlXw+KDt/AwKR1/XbyPPo3sNR0SEREREVViF+8mIjY5HYa6WmW6TIq2TIp27tb4/dw97Al9yCQ6ERGVe9cfJmNtcBQAYGZXD5bnKGesjOTYOtwXYzZfwOEbCfjpWESuNrFJ6Ri58QK+71UHABAUFofjtx4h7UW2qo2hrhb8XC3R1s0arVwtYapfMjXWKTcm0cswubYMw1o44pvd4Vhx9A7e9a7GW1GJiIiISGP2XH0IAPB3s4KuVtn+472Dl+1/SfRYzOjiwVuaiYio3FIoBKb+FYpshUAHTxu0crXSdEj0Bgx0tbDyfW/UnxWE1MzsXOuVFc0nbL+ittzWRP5ffXNr+NQ0K/OfwSoqJtHLuAE+Dlh2+A4iH6VgT+hDdK5jp+mQqIRIpVJ4eXmp/k9ERETlT0Xuz4UQqnroHbxsX9Na85o5WcBIroWEZxk4H/MUjWqYaTokIiIqB8piX/7HhXs4F/0U+joyTO3srulw6C1ciEnMM4H+KgczfXSvXxVt3a3hYWfMCWTLgCL9Npg7dy4aNWoEIyMjWFlZoXv37rhx44Zamzt37qBHjx6wtLSEsbEx+vTpg7i4OLU2T548wXvvvQdjY2OYmppiyJAheP5cvRj+lStX0KJFC8jlctjb2+P7779/w1Ms3wx1tfBh0xoAgGWH70AIUfAGVG5JJBKYm5vD3NycvxyJiIjKqYrcn1+5l4T7iWnQ15HBr5alpsN5LR0tKdq6WQMA9lyN1XA0RERUXpS1vjwxNRNz94QDAMa1cYGdqZ6GI6K3Ef8s/fWNAAS2q4VP29aCZ1WTMvE6pCIm0Y8ePYrRo0fj1KlTCAoKwosXL9CuXTukpKQAAFJSUtCuXTtIJBIcOnQIJ0+eRGZmJrp06QKFQqHaz3vvvYdr164hKCgI//77L44dO4bhw4er1icnJ6Ndu3ZwcHDA+fPnMW/ePMyYMQOrVq0qptMuXz5sWgP6OjJcf5iMIzcSNB0OEREREVVCu0NzSrm8U9uq3NRhVY6Y3xP6EAoFB6MQEVH5M2/fDTxJyYSLlSE+au6o6XDoLVkZyYu1HZWeIpVz2bt3r9rzdevWwcrKCufPn0fLli1x8uRJREVF4eLFizA2NgYArF+/HlWqVMGhQ4fg7++P69evY+/evTh79iwaNmwIAFiyZAk6duyIH374AXZ2dti0aRMyMzOxZs0a6OjowMPDA5cuXcKCBQvUku2VRRUDHbznUx2rj0di6eHbaOVqyW+hKiCFQoH4+HgAgJWVVZm5bYyIiIgKr6L250II1WjuDp5lv5SLUgsXCxjoyPAwKR2X7yWifvUqmg6JiIjKuLLUl1+6m4jNZ2IAALO6e0JbVjE+V1RmjR3NYGsiR2xSOvL6el8CwMZEjsaOLENX1rzVuy8pKQkAYGaW84PNyMiARCKBrq6uqo1cLodUKsWJEycAACEhITA1NVUl0AHA398fUqkUp0+fVrVp2bIldHT+P8NsQEAAbty4gadPn75NyOXW0BY1oSOT4nz0U5yJfKLpcKgECCEQHh6O8PBwlu0hIiIqpypqfx72MBkxT1Ih15ailWvZL+WiJNeWoY2ypEsoS7oQEdHrlZW+PFshMOWvqxAC6Fm/KprUNNdYLFR8ZFIJpnfJqWv/6vBY5fPpXdwh44ToZc4bJ9EVCgXGjx+PZs2awdPTEwDQpEkTGBgY4IsvvkBqaipSUlLw+eefIzs7Gw8f5tz+GRsbCysr9VmEtbS0YGZmhtjYWFUba2trtTbK58o2r8rIyEBycrLaoyKxNpajV8NqAIBlR+5oOBoiIiIiqkyUo9Bb1bKCgW6RbmbVuI5eNgCA3VcfVqgvNoiIqGLbfDoaofeTYSTXwqSObpoOh4pRe09brHi/AWxM1Eu22JjIseL9Bmhfju76q0zeOIk+evRohIaGYsuWLapllpaW2LZtG/755x8YGhrCxMQEiYmJaNCgQYnf/jJ37lyYmJioHvb29iV6PE34uKUTpBLg2M0EXL2XpOlwiIiIiKgSEEJg99WcATEd/ktIlyd+taygpy3DvadpCL1fsQbaEBFRxZTwLAPf77sBAJgQ4ApLI93XbEHlTXtPW5z44h38NqwJFverh9+GNcGJL95hAr0Me6PM9pgxY/Dvv//i8OHDqFatmtq6du3a4c6dO4iPj8ejR4/w66+/4v79+6hZsyYAwMbGRlVbSikrKwtPnjyBjY2Nqk1cXJxaG+VzZZtXTZo0CUlJSarH3bt33+TUyrTq5vroWtcOALD8yG0NR0NERERElcHNuOeIeJQCHS0p3qlt9foNyhg9HZkqbuXkqERERGXZ3D3X8Sw9C55VjfGej4Omw6ESIpNK4Otkjm71qsLXyZwlXMq4IiXRhRAYM2YM/vzzTxw6dAiOjvnPCmxhYQFTU1McOnQI8fHx6Nq1KwDA19cXiYmJOH/+vKrtoUOHoFAo4OPjo2pz7NgxvHjxQtUmKCgIrq6uqFIl78mAdHV1YWxsrPaoiEa2cgYA7L0Wi9vxzzQcDRERERFVdMpR6C1dLGEk19ZwNG+mvWfOQJw9LOlCRERl3KmIx9hx4T4kEmB2dy8mVonKiCIl0UePHo2NGzdi8+bNMDIyQmxsLGJjY5GWlqZqs3btWpw6dQp37tzBxo0b0bt3b3z66adwdXUFALi5uaF9+/YYNmwYzpw5g5MnT2LMmDHo168f7OxyRlkPGDAAOjo6GDJkCK5du4atW7di8eLFCAwMLMZTL59cbYzQ1t0aQgArjkRoOhwiIiIiquD2/Dd6u2M5LOWi1Lq2FXS1pIh6nIrwWA5EISKisulFtgLTdoYCAPo3ro569qaaDYiIVIqURF+xYgWSkpLQqlUr2Nraqh5bt25Vtblx4wa6d+8ONzc3fP3115g8eTJ++OEHtf1s2rQJtWvXRps2bdCxY0c0b94cq1atUq03MTHB/v37ERkZCW9vb3z22WeYNm0ahg8f/panWzGMauUEAPjr0n3cfZKq4WiIiIiIqKK6Hf8cN+OeQ1smQRs3a02H88YMdbXgV8sSQM5odCIiorJo7clI3Ix7DjMDHUwMcNV0OET0Eq2iNC7MrY/ffvstvv322wLbmJmZYfPmzQW2qVOnDo4fP16U8CqN+tWroJmzOU7efozVxyPwdTdPTYdExUAqlcLd3V31fyIiIip/Klp/vve/UejNnC1golc+S7kodfSyxf6wOOwOjUVgOyYmiIgob5rqyx8mpWHRgVsAgC871Iapvk6pHZuIXq/8f7KvpEb/Vxt9y9m7iH+WruFoqDhIJBJYWVnBysoKEglrnhEREZVHFa0/3301FgDQ0dNWw5G8vXfcrKAjk+J2/HPcimNJFyIiypum+vJZ/4YhNTMbDR2qoFeDaqV2XCIqHCbRyylfJ3PUszdFZpYCa05EaTocIiIiIqpgoh6lIOxhMmRSCdq6l99SLkrGcm20cLEA8P8vB4iIiMqCIzfisftqLGRSCWZ194SUk4kSlTlMopdTEokEo1vnjEbfeCoaSakvNBwRvS0hBOLj4xEfH1+o0klERERU9lSk/nxPaE6iuamTOaoYVIxbyjt45YyoV06WSkRE9KrS7svTX2Rj+t/XAAAfNq0BN1vjEj8mERUdk+jlWJvaVnC1NsLzjCxsCInSdDj0lhQKBcLCwhAWFgaFQqHpcIiIiOgNVKT+XJlo7lABSrkotXWzhpZUgvDYZ4hIeK7pcIiIqAwq7b78p6MRiH6cCmtjXYz3dynx4xHRm2ESvRyTSiUY1doJALDmZCRSM7M0HBERERERVQR3n6Tiyr0kSCVAO4/yX8pFyURfG02dc0q6KEfaExERaUr04xQsO3IbADClkzuM5OV7Em+iioxJ9HKuk5ctHMz18TT1BX47c1fT4RARERFRBbD3vwRzY0czWBjqajia4tXR0wYAS7oQEZFmCSEw/e9ryMxSoLmzBTrXqTh3fhFVREyil3NaMik+9ssZjb76WAQysrI1HBERERERlXfKBHNHr4r3B307DxvIpBKE3k9GzONUTYdDRESV1L5rcThyIwE6Mim+7uYBiYSTiRKVZUyiVwA9G1SFtbEuYpPT8eeF+5oOh4iIiIjKsYdJabgQkwiJBAjwsNF0OMXOzEAHTWqaAeBodCIi0ozUzCx8/U/OZKLDW9ZETUtDDUdERK/DJHoFoKslw7AWNQEAK47eQVZ2+Z7EioiIiIg0R1nKpaFDFVgbyzUcTclQTpa6m3XRiYhIA348eBsPktJRrYoeRrd21nQ4RFQITKJXEP0bV0cVfW1EP07lHwNERERE9Mb2XM35LKlMNFdEAR42kEiAy3cTcT8xTdPhEBFRJXIr7hl+Ph4BAJjZ1QN6OjINR0REhcEkegVhoKuFwc0cAQDLD9+GEELDEVFRSSQS1K5dG7Vr12YtNCIionKqvPfn8cnpOBv9BADQ3rPilXJRsjTSRaMaOSVd9nIAChERvaQk+3IhBKb8FYoshYC/mzXauFkX6/6JqOQwiV6BDPKtAQMdGcJjn+FQeLymw6EikkqlsLGxgY2NDaRSvjWJiIjKo/Len++7FgshgPrVTWFnqqfpcEpUx/++JNhzlXXRiYjo/0qyL9956QFORz6BXFuK6V3ci3XfRFSyyt8ne8qXib423vd1AAAs5Wh0IiIiIiqi3f+VculYgUu5KLX/7xzPRT9FbFK6hqMhIqKKLintBWbvug4A+OQdF9ib6Ws4IiIqCibRK5ghzR2hoyXFxZhEnIp4oulwqAiEEHj8+DEeP37ML0CIiIjKqfLcnz9+noHTkY8BVOxSLko2JnJ4O1QBkDMCn4iICCi5vnxh0E08ep6BmpYGGNrCsdj2S0Slg0n0CsbKSI6+De0BAMuP3NZwNFQUCoUCV69exdWrV6FQKDQdDhEREb2B8tyf7w+Lg0IAXlVNKs3ouA7/fVmwmyVdiIjoPyXRl4feT8KGkCgAwKxuntDV4mSiROUNk+gV0PCWNSGTSnD81iNcvpuo6XCIiIiIqBxQJpI7eFX8UehKHbxySrqciXqChGcZGo6GiIgqIoUiZzJRhQC61LVDM2cLTYdERG+ASfQKyN5MH93q2QHgaHQiIiIier2nKZkIvpNTyqVDJaiHrlTVVA917U0hBEu6UOWxbNky1KhRA3K5HD4+Pjhz5kyB7RMTEzF69GjY2tpCV1cXtWrVwu7du0spWqLyb+u5u7h0NxGGulqY0slN0+EQ0RtiEr2CGtXKCRIJsO9aHG7FPdN0OERERERUhgVdj0O2QsDN1hiOFgaaDqdUKUu67A1lEp0qvq1btyIwMBDTp0/HhQsXULduXQQEBCA+Pj7P9pmZmWjbti2ioqKwfft23LhxA6tXr0bVqlVLOXKi8ulJSia+2xsOAPi0bS1YG8s1HBERvSkm0SsoZysjBLjn/EGw4sgdDUdDREQlqagjyrZt24batWtDLpfDy8sr12gyIQSmTZsGW1tb6Onpwd/fH7du3VJrM2fOHDRt2hT6+vowNTXNdYzLly+jf//+sLe3h56eHtzc3LB48eK3PlciKhl7/ivl0rESTCj6KmUSPSTiMZ6kZGo4GqKStWDBAgwbNgyDBw+Gu7s7Vq5cCX19faxZsybP9mvWrMGTJ0/w119/oVmzZqhRowb8/PxQt27dUo6cqHz6bk84ElNfoLaNEQb5Omg6HCJ6C0yiV2CjWjsBAHZefoC7T1I1HA0REZWEoo4oCw4ORv/+/TFkyBBcvHgR3bt3R/fu3REaGqpq8/333+PHH3/EypUrcfr0aRgYGCAgIADp6emqNpmZmejduzdGjhyZ53HOnz8PKysrbNy4EdeuXcPkyZMxadIkLF26tHgvABG9taS0Fzhx+xGA/9cIr0wczA3gYWeMbIVAUBhHo1PFlZmZifPnz8Pf31+1TCqVwt/fHyEhIXlu8/fff8PX1xejR4+GtbU1PD098c033yA7Ozvf42RkZCA5OVntQVQZnY9+gq3n7gIA5vTwhJaMKTii8ozv4AqsTjVTtHCxQLZC4KdjHI1ORFQRFXVE2eLFi9G+fXtMmDABbm5umDVrFho0aKBKbgshsGjRIkyZMgXdunVDnTp1sGHDBjx48AB//fWXaj8zZ87Ep59+Ci8vrzyP89FHH2Hx4sXw8/NDzZo18f7772Pw4MHYsWNHsV8DIno7B6/H4UW2gIuVIZytDDUdjkZ0/O/Lg91XmUSniuvRo0fIzs6GtbW12nJra2vExub92o+IiMD27duRnZ2N3bt3Y+rUqZg/fz5mz56d73Hmzp0LExMT1cPe3r5Yz4OoPMjKVmDynzmDVPo0rAZvBzMNR0REb4tJ9ApudGtnAMDv5+4hPjn9Na1JkyQSCVxcXODi4gKJRKLpcIioHHiTEWUhISFq7QEgICBA1T4yMhKxsbFqbUxMTODj45PvPgsrKSkJZmb8A4IqtvLYn+/5rxZ4ZRyFrqQs6XLy9iMkpb7QcDREZYdCoYCVlRVWrVoFb29v9O3bF5MnT8bKlSvz3WbSpElISkpSPe7evVuKERO9veLoyzeERCM89hlM9bXxZQdOJkpUEWhpOgAqWT6OZvB2qILz0U/xy4lITOrIX95llVQq5QQ9RFQkBY0oCw8Pz3Ob2NjYAkegKf8tyii1wggODsbWrVuxa9eufNtkZGQgIyND9Zy3f1N5VN768+cZWTh6MwEA0NGr8tVDV6ppaYjaNkYIj32GoOtx6OVdTdMhERU7CwsLyGQyxMXFqS2Pi4uDjU3e739bW1toa2tDJpOplrm5uSE2NhaZmZnQ0dHJtY2uri50dXWLN3iiUvS2fXlccjoWBN0EAHzRvjbMDHK/T4io/OFI9ApOIpFg9H+10TeeikZiKidLIiKi0hUaGopu3bph+vTpaNeuXb7tePs3Uek7FB6PzCwFaloYwNXaSNPhaFQHz5yR+MpJVokqGh0dHXh7e+PgwYOqZQqFAgcPHoSvr2+e2zRr1gy3b9+GQqFQLbt58yZsbW3zTKATETB713U8z8hCPXtT9G3Iz7NEFQWT6JVAa1cr1LYxQkpmNtYHR2s6HMqHEAKJiYlITEyEEELT4RBROfAmI8psbGwKbK/8tyj7LEhYWBjatGmD4cOHY8qUKQW25e3fVBGUt/5cmTDu4GVTbsrPlJQO/43EP37rEZ6ls6QLVUyBgYFYvXo11q9fj+vXr2PkyJFISUnB4MGDAQADBw7EpEmTVO1HjhyJJ0+eYNy4cbh58yZ27dqFb775BqNHj9bUKRCVuLfpy0/efoR/Lj+AVALM7u4JqbRy961EFQmT6JVAzmj0nNroa4MjkZKRpeGIKC8KhQKXLl3CpUuX1EZ6EBHl501GlPn6+qq1B4CgoCBVe0dHR9jY2Ki1SU5OxunTp/PdZ36uXbuG1q1bY9CgQZgzZ85r2+vq6sLY2FjtQVTelKf+PDUzC4dvxAP4/yjsyszFyhBOlgbIzFbgUHi8psMhKhF9+/bFDz/8gGnTpqFevXq4dOkS9u7dqyrjFhMTg4cP/383hr29Pfbt24ezZ8+iTp06GDt2LMaNG4cvv/xSU6dAVOLetC/PyMrG1J05k4l+0MQBnlVNSipEItIA1kSvJDp62WL+/huIepyK387EYGiLmpoOiYiIikFgYCAGDRqEhg0bonHjxli0aFGuEWVVq1bF3LlzAQDjxo2Dn58f5s+fj06dOmHLli04d+4cVq1aBSDni9fx48dj9uzZcHFxgaOjI6ZOnQo7Ozt0795dddyYmBg8efIEMTExyM7OxqVLlwAAzs7OMDQ0RGhoKN555x0EBAQgMDBQVU9dJpPB0tKy9C4QEeXryI0EpL9QoLqZPjzs+KWVRCJBRy9bLDl0G/9r787DoirbP4B/Z4Zl2BFkFWRxAwRBUBFzF9ey7Fdv5ppmWqaVWVmWuaRvvm22uqRlVmraamlC4r6LgguIoiAIKsMisi8DM+f3BzCJgjIjcGbg+7muuZThmXPueYC559zznPvsjMvAY0GG09ueSBuzZ8/G7Nmz6/ze/v3777ovLCwMx48fb+KoiAzfN4dScCW7GG0tTTF3WBexwyGiRsaV6K2ETCrBzIFVvdHXHryCsgqVyBEREVFj0HZFWZ8+fbB582asXbsWgYGB+PXXX7Ft2zb4+/trxsybNw8vvfQSZsyYgZ49e6KoqAiRkZGQy+WaMQsXLkT37t2xaNEiFBUVoXv37ujevTtOnToFAPj111+RnZ2NjRs3wsXFRXPr2bNnM80MEd3PzppWLv5s5VKjZkX+/sRsnr1JREQNlp5bgi/3XgYALHjYFzZmxiJHRESNTSIYQrNGHRQUFMDGxgb5+fk8HbyaslKNAR/tQ0Z+Gf77uD8mhHqIHRLdRqVS4dChQwCAfv36QSaTiRwRETUl5qn74xyRITKUfF5WoULI0igUK1XYNushBLnbih2SXhAEAYM+3o/UmyX4anx3PNLNVeyQSM8xV90f54gMjS65/LnvT2H3hUz09rbDT9N788NpIgPS0DzFleitiImRFNOr27isOZCMSpV+9+kkIiIioqZx8FI2ipUquNrIEejGnq01JBIJRgZUrUaPiFOIHA0RERmC3QmZ2H0hE0ZSCZY+5s8COlELxSJ6K/N0L3fYWZggPbcUO85l3P8BRERERNTiRMRXFYhHBrjwYP8Oo6pbuuy9mIVSJVsgEhFR/UqVKizefh4A8Fw/b3RyshI5IiJqKiyitzLmJkZ49iFPAMCq/UlQq1tkNx8iIiIiqkd5pQq7EzIBAKMCnEWORv/4t7OGWxszlFaocOBSttjhEBGRHlu1PwnXbpXC1UaOl4d0FDscImpCLKK3QpPCPGFpaoRLmUXYfSFT7HComkQigbe3N7y9vbkiTAsqtYBjyTfx55nrOJZ8Eyp+MERERCIyhHx+JCkHheWVcLI2RXf3NmKHo3ckEglG+ld9uBARzzM3iYham4bm8uTsInx94AoAYOHorjA3MWquEIlIBPwLb4VszIwxKcwDq/cnY+X+ZAz1c9Lbg7zWRCqVon379mKHYVAi4zOwZHsCMvLLNPe52MixaLQfRlSfik1ERNScDCGf76zu9T3S3wVSKd8D1mVkgAvWHUrBngtZKKtQQW6snxeIJSKixteQXC4IAhb9eR5KlRoDuzhgeFenZoqOiMTCleit1LMPecHUSIqz6Xk4mnxT7HCItBYZn4GZG2NrFdABQJFfhpkbYxHJlWNERER3UVaqset8VRF9hD9budQnyM0WLjZyFJVX4vDlHLHDISIiPfN3XAYOJ+XAxEiKJY925cJEolaARfRWysHKFE/3dAcArNyXJHI0BFR9kl1QUICCggIIAluS3ItKLWDJ9gTUNUs19y3ZnsDWLkRE1Oz0PZ8fu3ITBWWVaGtpgp6edmKHo7ekUonmQ4ad/GCeiKhVuV8uLyqvxNIdCQCAWQM7wsPeorlDJCIRsIjeik3v7w0jqQRHk28iNu2W2OG0emq1GrGxsYiNjYVarRY7HL0WnZJ71wr02wkAMvLLEJ2S23xBERERQf/zec2ZWsO7OkPGVi73NCqgqjVcVEImlJX697MkIqKmcb9c/lnUJWQWlMPD3hzPD/AWIUIiEgOL6K2YWxtzjOneDgCwal+yyNEQNVxWYf0FdF3GERERtQaVKjX+OV91UfmaAjHVL6R9GzhamaKwrBJHktnShYiIgAsZBfjuaCoAYMmjXXnNDKJWhEX0Vu6FAR0gkQC7L2TioqJA7HCIGsTRSt6gcbbmxk0cCRERkeGITslFbrESbcyNEerFVi73I5VKMLxrVUuXyOqLsRIRUeulVgt4d1s8VGoBI/2dMbCLo9ghEVEzYhG9levoaImR1f0eV+/nanQyDL287OBsc/9C+rxfzuGHY6koq1A1Q1RERET6bedtrVyMZDwMaIiRAVXvk/9JUKBCxZYuRESt2W+x13Dq6i2Ym8jw7iN+YodDRM2M754JLw7sCADYfvYGrt4sFjkaovuTSSUY5udU5/dqurvamhsjs7AcC/88j4Ef7cePx1JRXsliOhERtU4qtYDI+KpWLiPZyqXBennawd7CBHklFThxhddaISJqrfJKlFgecREAMCe8E1xtzUSOiIiaG4voBP92NhjQ2QFqAVhz4IrY4RDdV16JEjvOVa2ms5Ib1fqes40cayYG48TbQ7B0jD9cbORQFJThXRbTiYioFTuVmouconJYy40Q5m0vdjgGw0gmxbDqli41K/mJiKj1+eifROQWK9HZyRJTH/ISOxwiEoFWRfTly5ejZ8+esLKygqOjI8aMGYPExMRaYxQKBSZNmgRnZ2dYWFggODgYv/32W60xubm5mDBhAqytrWFra4tp06ahqKio1phz586hX79+kMvlcHd3x4cffqjjU6SGmD24ajX6bzHXoMjnxRhJv30Q+e8bmJPvhOOn6b3x+dNB+Gl6bxx+czBG+LvA1EiGSb09sP+NgVj6WFc4W8uRkc9iOhERtU4R8VU9vYf6OcPEiOtotDGqpqVLvAIqtSByNERE1NzOpOdhc3QaAGDpY/4wZks0olZJq7/8AwcOYNasWTh+/DiioqJQUVGBYcOGobj43xYgkydPRmJiIv766y/ExcXh//7v//DUU0/h9OnTmjETJkzA+fPnERUVhR07duDgwYOYMWOG5vsFBQUYNmwYPDw8EBMTg48++giLFy/G2rVrG+EpU116etqhl6cdlCo1vjnE1ehikEgk8PT0hKenJyQSyf0f0ErFpt3CT9VvYJaNCYDcWIawDvZ4LKgdwjrYQyatPXemRjJMCvPEgXn1FNOPX2UxnYiIGo0+5nO1WkBkdRG9piBMDdfb2x625sa4WaxEdApbuhARtXS353K1ACzYFgdBAP4vuB1CeTYXUaulVRE9MjISU6ZMQdeuXREYGIgNGzYgLS0NMTExmjFHjx7FSy+9hF69esHb2xsLFiyAra2tZsyFCxcQGRmJb775BqGhoejbty++/PJLbNmyBTdu3AAAbNq0CUqlEuvXr0fXrl3x9NNP4+WXX8aKFSsa8anTnV4c1AEAsOlEGm4VK0WOpvWRSqWaRC2V8pPtulSq1Hjnj3gAwJMhbujlZdfgx9ZbTN8Wz2I6ERE1Gn3M56fT86AoKIOlqRH6dmordjgGx1gmxVDfqmuxRLClCxFRiydAggyVFc7mGWN5xEXEXy+AldwI80f6ih0aEYnogd7Z5+fnAwDs7P4tZPXp0wdbt25Fbm4u1Go1tmzZgrKyMgwcOBAAcOzYMdja2qJHjx6ax4SHh0MqleLEiROaMf3794eJiYlmzPDhw5GYmIhbt27VGUt5eTkKCgpq3Ug7Azo7oKurNUorVPjuaKrY4RDd5YdjV3EhowA2ZsaYP9JHp23UFNP3vzEQ77GYTkRErUBEXFXhN9zXEaZGMpGjMUyjqi/GGhmvgJotXYiIWqzI+Az0/WAvxq07jle2nMH6I6kAgIcDXOBgZSpucEQkKp2L6Gq1GnPmzMFDDz0Ef39/zf0///wzKioqYG9vD1NTUzz//PP4448/0LFjVc9thUIBR0fHWtsyMjKCnZ0dFAqFZoyTk1OtMTVf14y50/Lly2FjY6O5ubu76/rUWi2JRIJZg6p+ThuOpKCovFLkiFoXQRBQXFyM4uJiCAIPzu6UWVCGFVGXAABvjvCBveWDvYGRG8swuZ5i+qCP9mMji+lERKQDfcvngiBo+qGPrC4Ek/b6dLSHldwIWYXliE2re1EPEREZtsj4DMzcGIuM/FLIUQE5KgBU5fKtJ9MRybORiFo1nYvos2bNQnx8PLZs2VLr/nfffRd5eXnYvXs3Tp06hblz5+Kpp55CXFzcAwd7L/Pnz0d+fr7mlp6e3qT7a6mGd3WGt4MFCsoqsen4VbHDaVXUajVOnjyJkydPQq1Wix2O3nlvRwKKyivRvb0tnu7ZeB+S3VlMd7I2xY38MixgMZ2IiHSgb/n83LV8XM8rhbmJDAM6O4gdjsEyNZJpWrrsjKt7UQ8RERkulVrAku0JEABIIcDHKAs+RlmQ4t8PxJdsT+AFpolaMZ2K6LNnz8aOHTuwb98+uLm5ae5PTk7GV199hfXr12PIkCEIDAzEokWL0KNHD6xcuRIA4OzsjKysrFrbq6ysRG5uLpydnTVjMjMza42p+bpmzJ1MTU1hbW1d60bak0klmDmgqjf6ukMpKKtg8ZDEd/BSNv4+lwGpBFg2xh9SaeNfqK2mmH7gjUFY8iiL6URE1DLsrF41N8jHEXJjtnJ5EDUr+SPiM9jShYiohYlOyUVGflm93xcAZOSX8QLTRK2YVkV0QRAwe/Zs/PHHH9i7dy+8vLxqfb+kpKRqo3dcREkmk2lW4oSFhSEvL6/WxUj37t0LtVqN0NBQzZiDBw+ioqJCMyYqKgpdunRBmzZttAmZdDCmezu0szVDTlE5fom5JnY41MqVVaiw8M+qi4k+08cTXV1tmnR/cmMZnulTfzF904mrUFaKv7KQiIjofgRBQET1qulR/mzl8qD6dWoLCxMZMvLLcPZantjhEBFRI0rLLW7QuKzC+gvtRNSyaVVEnzVrFjZu3IjNmzfDysoKCoUCCoUCpaWlAAAfHx907NgRzz//PKKjo5GcnIxPPvkEUVFRGDNmDADA19cXI0aMwPTp0xEdHY0jR45g9uzZePrpp+Hq6goAGD9+PExMTDBt2jScP38eW7duxeeff465c+c27rOnOhnLpJjR3xsA8PWBZFSoWDAk8aw5kIzUmyVwsjbF3KGdm22/9RXT3/kjHoM+ZjGdiIj0X0JGAdJySyA3lmJgF7ZyeVByYxmGVLd0qekzT0REhi23WIlPdiVi8V/nGzTe0UrexBERkb7Sqoi+evVq5OfnY+DAgXBxcdHctm7dCgAwNjbGzp074eDggNGjR6Nbt2744Ycf8P3332PUqFGa7WzatAk+Pj4YMmQIRo0ahb59+2Lt2rWa79vY2GDXrl1ISUlBSEgIXnvtNSxcuBAzZsxopKdN9zO2pzvaWprg2q1SbD97Q+xwqJVKzSnGqv3JAIB3H/GDldy42WO4vZi+eLQfHK1McT2vlMV0IiLSezWr0Ad2doSFqZHI0bQMI/2rWkvujMvQiwvHEhGRbhT5ZVi6IwEP/W8vvtybhNIKNWT3aBsqAeBiI0cvL7vmC5KI9IpW76Yb8kaxU6dO+O233+45xs7ODps3b77nmG7duuHQoUPahEeNSG4sw7N9vfBhZCJW7U/GmKB2TdKHmqg+giBg4V/noaxUo1+ntng4QNzT0OXGMkx5yAtP92qPLdFpWLU/WVNMX7UvGbMGdcSTIW4wMdL5es1ERESNRhAE7Iyr6oc+MqDuawqR9gZ2cYSZsQzXbpXi/I0C+Ldr2jZzRETUuK7eLMaaA1fwW8w1KKvPuvdvZ43ZgzpCrQZmbY7FnZWPmq8Xjfa7Z6GdiFo2VnuoXhN7e8BKboSkrCLsSsi8/wOIGtHOOAUOXsqGiZEU7z3mD4lEP96s1BTTD86rvTL97T/iMOjj/dh8Io0r04mISHSXMotwJacYJkZSDPZxFDucFsPMRIZBPlWtcWo+pCAiIv2XqCjEnC2nMejj/fgpOg1KlRq9vOzw/bO9sH12X4zwd8Gobi5YPTEYTta1W7Y428ixemIwRvD6IkStGs/rpHpZy43xTJgnvtqXhFX7kzC8q5PeFDJbIolEAnd3d83/W7PCsgq8t6OqJ93MAR3g1dZC5IjudvvK9J+i07C6emX623/EYeW+JK5MJyJqpfQln9cUePt3chClHVpLNtLfBTvjFNgZl4E3hndp9e/bDIlKLSA6JRdZhWVwtKpqy8BVpUQt29n0PHy1LwlRty0MHNjFAbMGdURPz7tbs4zwd8EQH0dEHjuHvFIl5nl5I7RDW75WEBGL6HRvUx/yxDeHr+DctXwcTspBv068KFVTkUql6NChg9hh6IXPdl9GZkE5POzNMXOgfs+J3FiGqQ95YVw9xfTZgzviiWAW04mIWgt9yecR8dWtXPzZyqWxDfJxhKmRFKk3S3BRUQhfF2uxQ6IGiIzPwJLtCcjIL9Pc52Ijx6LRflxdStTCCIKAY1duYtW+ZBxOygEASCRVOfHFgR3v24rL2EiG0f26N0eoRGRAWNWhe7K3NMW4Xu0BACv3JYkcDbUGCTcKsOFoKgDgvcf8ITeWiRtQA9UU0w/OG4RFo/3gUN3mZf7vcf+eMsg2L0RE1AySsgpxKbMIxjIJwn2dxA6nxbE0NcKAzlULSyLY0sUgRMZnYObG2FoFdKDqwoIzN8YiMp4/R6KWQBAE7LmQiSdWH8X4dSdwOCkHMqkETwS7IerVAVg1IYTXsiAinbGITvc1vZ83jGUSHL+Si5iruWKH02IJgoCysjKUlZU16CK+LZFaLWDBtjio1AIeDnDRHKAakppi+qF5g7DwERbTiYhaG33I5xFxCgDAQx3bwsacrVyawqjqC57vjFeIHAndj0otYMn2BNT111hz35LtCVCpW+f7b6KWQKUWsP3sDYz8/BCmfX8KsWl5MDGSYnKYB/a/PhCfPBWIjo6WDd6ePuRyItI/LKLTfbnamuH/ursBAFbtSxY5mpZLrVbj+PHjOH78ONTq1llk3XoqHbFpebAwkeHdR/zEDueByI1leLZv3cX0wZ/sx5boNFSoWufPmYioJdOHfB5RXdgdxRYVTWawryOMZRIkZRXhcmah2OHQPUSn5N61Av12AoCM/DJEp3CxEJGhUVaqsfVkGsJXHMBLP53GRUUhLExkeH6ANw6/OQjvPeYPdztzrberD7mciPQPe6JTg7wwsAN+iUnHnotZSLhRAD9X9n6kxnWzqBz/i7gIAHh1aGc428jv8wjDUFNMHx/aHptPpGH1gWRcu1WKt36Pw1f7kjB7UEc8EeIGYxk/0yQiogeXmlOMhIwCyKQSDPVjK5emYi03Rr9ODth7MQsR8Qp0crISOySqR1Zh/QV0XcYRkfhKlSpsOZmGtQevaD4kszU3xtQ+XpjSx5NnYRFRk2DVhhrEq62F5rTV1Qe4Gp0a3/8iLiK/tAK+LtaY0sdT7HAa3e0r0999xA9tLU01xfRBH3NlOhERNY6aVeh9OtijjYWJyNG0bDUXbd3Jvuh6zdGqYQszGjquJVi5ciU8PT0hl8sRGhqK6Ojoesdu2LABEomk1k0ubz1zRfqloKwCK/cloe8HezUXCna0MsWCh31x5M3BeCW8EwvoRNRkWESnBntxYEcAwN/nbiAlp1jkaKglOZmai19irgEAlo3xh1ELXpUtN5Zh2j2K6VtPsphORES6i6i+QOJItnJpckP9nGAkleCiohBXsovEDofq0cvLDjZm9RfVJABcbOTo5WXXfEGJaOvWrZg7dy4WLVqE2NhYBAYGYvjw4cjKyqr3MdbW1sjIyNDcrl692owRE1WdtfzxP4l46H978dE/ibhZrIS7nRn++7g/Ds4bhOf6ecPClI0WiKhptdxKFTU6P1drDPZxhFoAvuZqdGokFSo1FvwRDwB4uqc7QjzaiBxR8zAzqbuY/uZvLKYTEZFu0nNLcO5aPqQSYFhXtnJparbmJujTsS2Af88AIP1zIaMAxeWVdX5PUv3votF+kEkldY5paVasWIHp06dj6tSp8PPzw5o1a2Bubo7169fX+xiJRAJnZ2fNzcmJry/UPDLyS/He9gT0/WAfvtqXhMKySnRytMSnYwOx77WBmBDqAbmxTOwwiaiVYBGdtDJrUAcAwG+x15CRXypyNNQSfHckBYmZhbCzMMGbI3zEDqfZ3V5MX/CwL4vpRESks8jqQm4vLzu0tTQVOZrWYVR1S5eaMwBIv+SVKPHCxhhUqgX4t7OGs3XtNiTONnKsnhiMEa3kzA2lUomYmBiEh4dr7pNKpQgPD8exY8fqfVxRURE8PDzg7u6Oxx57DOfPn2+OcKkVS80pxlu/nUP/D/dh/ZEUlFaoENDOBmsmhuCfOf3xeHe3Fn32MhHpJ57vQloJ8bBDqJcdTqTkYt3BFCwc7Sd2SGTAbuSV4rPdlwEAb430adW9W81MZHiunzcmhHpg04mrWHPgiqaY/tW+JLw0qBMeD27HC5ASEVG9dlYXcmuuY0NNb1hXZ7yzLR7x1wuQdrME7e3NxQ6JqqnVAuZsPYNrt0rR3s4cm6b1hqXcCNEpucgqLIOjVVULl9ayAh0AcnJyoFKp7lpJ7uTkhIsXL9b5mC5dumD9+vXo1q0b8vPz8fHHH6NPnz44f/483Nzc6nxMeXk5ysvLNV8XFBQ03pOgFu2iogCr9iVjx7kbUAtV94V62WH24I7o27EtJJLW8/dKRPqHRXTS2qxBHXEiJRo/Radh1qAOsOdKp0YhkUjg6uqq+X9rsGT7eZQoVejp2QZPBtf9Jry1qauYnp5binm/ncOX+y6zmE5EpOfEyucZ+aU4nZYHiQQY3tW52fbb2tlZmCDUyw5Hk28iIj4Dzw/oIHZIVO2LvZexPzEbpkZSrJ4YrLnYYFgHe5EjMyxhYWEICwvTfN2nTx/4+vri66+/xtKlS+t8zPLly7FkyZLmCpFagNNpt7ByXzJ2X8jU3DeoiwNmDeqIHp7Nf72C1nhsTkT3xyoMaa1fp7YIaGeD0goVNhxNFTucFkMqlaJz587o3LkzpNKW/6e592Im/jmfCZlUgqVj/CFtRauAGqKmmP5vmxcTTTF98Cf78fPJdLZ5ISLSQ2Ll85pWLj082sDpjpYV1LRGVq/8Z190/bEvMQuf76k62/G/jwegq6uNyBHph7Zt20ImkyEzM7PW/ZmZmXB2btiHb8bGxujevTuSkpLqHTN//nzk5+drbunp6Q8UN7VMgiDgaFIOxq87jsdXHcXuC5mQSICHu7ng75f74rupvUQpoAOt79iciBqGrwakNYlEoumNvuFoKgrLKkSOiAxNqVKFRX9V9VKc1tcLPs7WIkekv/4tpg++q5g+5JMD+PkUi+lERARExFUVcEe2kt7O+mR4VydIJMCZ9Dxcz+M1g8SWnluCOVvOQBCA8aHt8WQIz3asYWJigpCQEOzZs0dzn1qtxp49e2qtNr8XlUqFuLg4uLjU/1pjamoKa2vrWjeiGmq1gKiETDy+6ijGf3MCR5NvwkgqwX9C3LB77gCsHB/MD76ISC+xiE46GebnjA4OFigsq8TG42lih9MiCIIApVIJpVIJQRDEDqdJrdyXhPTcUrjYyPHKkE5ih2MQ6iqmp+WWYN6vLKYTEekTMfJ5VkEZTl7NBQCM8Gcrl+bmaCVHz+rVkpFcjS6qsgoVZm6KQX5pBQLdbLCI12+6y9y5c7Fu3Tp8//33uHDhAmbOnIni4mJMnToVADB58mTMnz9fM/69997Drl27cOXKFcTGxmLixIm4evUqnnvuObGeAhkolVrAn2euY9QXhzD9h1M4k54HUyMpngnzwIF5g/DRfwLRwcFS7DABtK5jcyJqOPZEJ51IpRK8OLAjXvvlLL49fAVTH/KE3FgmdlgGTa1W4+jRowCAfv36QSZrmfOZlFWErw8mAwAWjfaDhSlfhrRRU0wfH9oem46n4euDyZpi+ld7kzB7cEc83p0904mIxCJGPv/nvAKCAAS528LV1qzJ90d3G+XvjOiUXETEZWBaXy+xw2m1Fv15HvHXC9DG3BirJobA1Khlvp9+EGPHjkV2djYWLlwIhUKBoKAgREZGai42mpaWVqt9xa1btzB9+nQoFAq0adMGISEhOHr0KPz8+AEFNUx5pQp/xF7HmgPJSL1ZAgCwNDXCpDAPPPuQFxys9O8aa63l2JyItMMqC+ns0SBXtLM1Q06REltPss8d3Z8gCFj4ZzwqVAIGdXHghc8egLmJEab398bBeYPwzqjaK9PDVxzAL6fSUcmV6a3GypUr4enpCblcjtDQUERHR99z/C+//AIfHx/I5XIEBARg586dtb4vCAIWLlwIFxcXmJmZITw8HJcvX6415r///S/69OkDc3Nz2Nra1rmfl19+GSEhITA1NUVQUNCDPEUiuoed1a1cRgUwr4plRHUbnVNXb0GRXyZyNK3Tlug0bD2VDqkE+HJcMNrxA6V6zZ49G1evXkV5eTlOnDiB0NBQzff279+PDRs2aL7+9NNPNWMVCgX+/vtvdO/eXYSoydCUKCux/nAKBny4H2/9HofUmyVoY26M14Z2xpE3B+PNET56WUAnIqoPi+ikM2OZFC8M8AYAfH0gGcpKFuzo3v46ewNHk2/C1EiKJY/680rnjeDOYrq9hQmu3izBG7+ew5B6iukqtYBjyTfx55nrOJZ8Eyo1T1E0ZFu3bsXcuXOxaNEixMbGIjAwEMOHD0dWVlad448ePYpx48Zh2rRpOH36NMaMGYMxY8YgPj5eM+bDDz/EF198gTVr1uDEiROwsLDA8OHDUVb2b2FIqVTiP//5D2bOnHnP+J599lmMHTu2cZ4sEd3lZlE5TqTcBMB+6GJytpEjxKMNgKozA6h5nbuWh4XV19t5bVgX9O3UVuSIiFqv/NIKrNyXhL4f7MN7OxKgKCiDk7UpFjzsi8NvDsZLQzrBxtxY7DCJiLTGPgr0QP7Twx2f70nCjfwy/HnmOv7Tw13skEhP5ZdWYOmOCwCAlwZ3RHt7c5EjallqiukTerfHxuNX8fWBK5pi+lf7kjB7UFWbl90XMrFkewIyblsl52Ijx6LRfppVdGRYVqxYgenTp2t6ma5ZswZ///031q9fj7feeuuu8Z9//jlGjBiBN954AwCwdOlSREVF4auvvsKaNWsgCAI+++wzLFiwAI899hgA4IcffoCTkxO2bduGp59+GgCwZMkSAKi1Wu1OX3zxBQAgOzsb586da7TnTET/2pWQCbUABLSzgbsdc6uYRvo7I+bqLeyMy8AzfTzFDqfVuFWsxMyNsVBWqhHu64SZAzqIHRJRq5RTVI71h1Pw47GrKCyvBAC0tzPHCwM64ImQdmyvREQGjyvR6YHIjWV4rl9V38fVB5K5opXqtWJXInKKyuHtYIHp/b3FDqfFMjcxwoz+HXDozUF4e5RPrZXpYcv34IWNsbUK6ACgyC/DzI2xiIzPEClq0pVSqURMTAzCw8M190mlUoSHh+PYsWN1PubYsWO1xgPA8OHDNeNTUlKgUChqjbGxsUFoaGi92yQi8eyMq3rtHslWLqKruajrydRcZBeWixxN66BSC3hl6xlczyuFh705PnkqEFIpz3Qkak438kqx+K/z6PvBXqzan4zC8kp0drLE508HYe9rAzA+tD0L6ETUIrCITg9sQmh7WMuNcCW7mKevUp3iruXjx+NXAQDLHvPnm6hmcGcx3c7cGNlFyjrH1nz0tWR7Aj8IMzA5OTlQqVSai4HVcHJygkJR9+uxQqG45/iaf7XZZmMpLy9HQUFBrRsR1e9WsRJHk9nKRV+4tTFHoJsN1AKwK4HviZvD57sv4eClbMiNpVgzMQQ2ZmwRQdRcUnKK8eav5zDgo33YcDQVZRVqBLrZYO2kEES+0h+PBbWDkYwlJyJqOfiKRg/MSm6MKdWnrK7clwRBYBGO/qVSC3hnWxzUAvBYkCv6dGSPyuZUU0xfMTbonuMEABn5ZYhOyW2WuIjqsnz5ctjY2Ghu7u5sEUZ0L1EXMqFSC/B1sYZXWwuxwyEAIwOqPsyIiGMRvantvZiJL/YmAQCW/18AfF2sRY6IqHW4kFGA2ZtjMeST/dh6Kh0VKgFh3vbYOC0U22Y9hGFdnXlGCBG1SCyiU6OY8pAXzIxlOH+jAAcuZYsdjkGSSCRwdnaGs7Nzi7rg5uYTV3HuWj6sTI3wzsO+YofTauWXVjRoXGR8BgrKGjaWxNe2bVvIZDJkZmbWuj8zMxPOznW3dnB2dr7n+Jp/tdlmY5k/fz7y8/M1t/T09CbdH1FTaM58HlHTysWfrVz0Rc3P4tiVm8gtrvsMMHpwaTdLMGfLGQDApN4eeLy7m7gBEbUCMVdvYdqGkxj5+SHsOJcBtQAM8XHEbzP74KcZvdG3U9sWcxzbUo/NiejBsIhOjcLOwgTjQ9sDAFbtSxY5GsMklUrh4+MDHx8fSKUt408zu7AcH/6TCAB4fXgXOFrJRY6o9Wro3H9/7CqC34vC+HXH8e3hFFy9WdzEkdGDMDExQUhICPbs2aO5T61WY8+ePQgLC6vzMWFhYbXGA0BUVJRmvJeXF5ydnWuNKSgowIkTJ+rdZmMxNTWFtbV1rRuRoWmufJ5fWoHDSTkAgFHsh643POwt0NXVGiq1gCi2dGkSZRUqvLAxBgVlleje3hbvPuIndkhELZYgCDh8OQfj1h7HE6uPYs/FLEgkwCPdXLDz5X74dkpPhHi0ETvMRtcSj82J6MEZiR0AtRzT+3njh2OpiE7NRXRKLnp52YkdEons/Z0XUFhWiYB2NpjY20PscFq1Xl52cLGRQ5FfhvoaLlmYyuBkZYorOSU4mnwTR5NvYumOBHRwsEC4rxOG+DohuL0texvqmblz5+KZZ55Bjx490KtXL3z22WcoLi7G1KlTAQCTJ09Gu3btsHz5cgDAK6+8ggEDBuCTTz7Bww8/jC1btuDUqVNYu3YtgKqVN3PmzMGyZcvQqVMneHl54d1334WrqyvGjBmj2W9aWhpyc3ORlpYGlUqFM2fOAAA6duwIS0tLAEBSUhKKioqgUChQWlqqGePn5wcTE5PmmSCiFmrPhUxUqAR0crRER0crscOh24wKcMH5GwXYGafA2J7txQ6nRREEAe/8EY+EjALYW5hg1YRgmBjxfQlRY1OrBey+kImV+5NxNj0PAGAsk+D/urvh+QHe8HawFDdAIiIRsIhOjcbZRo4nQ9zwU3Q6Vu67DJW6I7IKy+BoJUcvLzvI2BftngRBgFqtBlD1ybehnzZ2NDkHf5y+DokEWDbGnz9/kcmkEiwa7YeZG2MhAWoV0mt+Mp/8JxAj/F2QmlOMPRezsOdCJqJTcpGcXYzk7Cv4+uAV2JobY1AXRwzxdUT/zg6wlvMCXmIbO3YssrOzsXDhQigUCgQFBSEyMlJzYdC0tLRaK2j69OmDzZs3Y8GCBXj77bfRqVMnbNu2Df7+/pox8+bNQ3FxMWbMmIG8vDz07dsXkZGRkMv/PaNh4cKF+P777zVfd+/eHQCwb98+DBw4EADw3HPP4cCBA3eNSUlJgaenZ6PPBZE+aK58vrO653ZND27SHyP8nfHRP4k4kpSD/JIK2JgzVzaWzdFp+C32GqQS4Mtx3eFiYyZ2SEQtSqVKjb/jMrBqXzISMwsBAHJjKZ7u2R4z+nvD1bZ1/M21tGNzImocEqGFXgWyoKAANjY2yM/P5+ngzejqzWIM/Gj/XStdXWzkWDTaDyP8eaBXH5VKhUOHDgEA+vXrB5lMJnJEulNWqjHy84NIzi7GxN7tsWxMgNghUbXI+Aws2Z6AjPwyzX33+vvML63AwUvZ2HMhE/sSs2v1VjeSShDqbYfBPk4I93WEhz0vaqcN5qn74xyRIWqOfF5UXongpVFQVqoROacffJz596Fvhn96EImZhfjkP4F4IoT9uhvDmfQ8PLXmGJQqNd4c4YOZAzuIHRIA5qqG4Bzpv/JKFX6LuY41B5KRllsCALAyNcKkMA8829cLbS1NRY6webWkY3Miur+G5imuRKdGdSGjoM5WEYr8MszcGIvVE4NZSG8F1h26guTsYrS1NMEbw33EDoduM8LfBUP9nBGdktugM0VszIwxOtAVowNdUalSI+bqLey9mIXdFzKRnF2MI0k3cSSpqu1LR0dLDPF1RLivE4Lbt+HZB0RETWTvxSwoK9XwbmuBLk5s5aKPRgY4IzGzEBHxGSyiN4LcYiVe3BgDpUqNYX5OeGGAt9ghERkMlVqo971/ibISm0+kYd2hK8gsKAdQdb2zZx/yxKQwT9iY8UwaIqIaLKJTo1GpBSzZnlDn9wRUtYxYsj0BQ/2cWVxrwdJzS/Dl3ssAgHce9uUbLz0kk0oQ1sFe68cZyaQI9bZHqLc95o/yRWpOMXZfyMSeC1k4mZqLpKwiJGUV4esDV9Cmuu3LYLZ9ISJqdBFxGQCqCrU8xVw/jQpwwWe7L+PgpRwUllXAinlQZyq1gJd/Oo0b+WXwamuBj58K5O89UQPVdxbq68O74MatUqw/koJbJVVnmjpbyzGjvzee7uUOcxOWioiI7sRXRmo00Sm5tZLznQQAGflliE7J1amAR/pPEAQs/us8yirU6O1thzFB7cQOiZqQZ1sLPNfPG8/180Z+aQUOVLd92Z+YjVslFfj99HX8fvq6pu3LEB8nhPs6ob29udihExEZrBJlJfYlZgEARvLsPr3VydESHRwskJxdjL0Xs/AY3xPp7NOoSziclAMzYxnWTAzhB/NEDRQZn4GZG2PvOlM8I78Mr/18VvO1h705Zg7ogMeD28HUiG1LiIjqwyI6NZqswvoL6LqMI8MTlZCJPRezYCyTYNkYf64SakVszIzxaKArHr2t7cue6rYvV25r+/LejgR0crTEEN+qPurd2faFiEgr+xOzUVahhrudGbq6srewvpJIJBgV4IIv9yZhZ1wGi+g6ikrIxFf7kgAA/3siAF2c2b6IqCFqzhK/1wXwjKQSfPRkN4wOdIWRTHqPkUREBLCITo3I0UreqOPIsJQoKzXtfKb380ZHRx7ktFa3t315e5QvUnKKsae67Ut0ai4uZxXhclYR1hxI1rR9GeLrhP6d2/J0dyKi+9hZ3cpllL8LP6zWcyP9q4ro+xOzUVxeCQtTHnppIzWnGHN/PgMAmNLHkx9EEGnhfmeJA0ClWoCzjRkL6EREDcR3ctRoennZwcVGDkV+Wb2feLvYVF3IhFqez/dcxvW8UrSzNcNLgzuJHQ7pEa8Gtn0xlkkQ6mWvuTipux3bvhAR3a6sQoW9F6tbuQSwlYu+83Wxgoe9Oa7eLMG+xCw80s1V7JAMRqlShRc2xqCwrBIhHm3w9ihfsUMiMig8S5yIqPGxiE6NRiaVYNFoP8zcGAsJUGchPcjdlq0b6iGRSODg4KD5vyFJVBTi20MpAIAlj3aFmQl76VHd7mz7curqLc0q9Ss5xTiclIPDSTlYsj0BnZ0sMdiHbV+IyLA0ZT4/eCkbJUoVXG3kCHSzadRtU+OTSCQY6e+CNQeSERGvYBG9gQRBwNt/xOGiohBtLU2wcnwwTIy4UpZIGzxL/MEY8rE5ETUdFtGpUY3wd8HqicF3XQHcWm6EgrJKRMQr8OWey3hpCFcq30kqlaJr165ih6E1QRDw7rZ4VKoFDPVzQrifk9ghkYEwkknR29sevb3t8c7DfriSXYS91X3UT6bewqXMIlzKrGr7YmdhgoFdHBDu64R+ndj2hYj0V1Pm84h4BYCqVeg8qDcMowKcseZAMvZdzEKpUsWFBg2w8fhV/HH6OmRSCb4cFwxnGxb5iLSVfqvknt+XAHDmWeL1MtRjcyJqWiyiU6Mb4e+CoX7OiE7JRVZhGRytqpLzt4ev4P2dF/FJ1CWYGksxo38HsUOlRvBb7HVEp+bCzFiGRaP9xA6HDJi3gyW8HSyr2r6UVGD/pSzsuZCF/YlZyC1W4vfY6/g9lm1fiKh1Kq9UYXdCJoCqwiwZhoB2Nmhna4breaU4cCkbI/z5s7uX2LRbeG9H1TV23hzRBWEd7EWOiMjw/B57DW/+dk7z9Z1nidd8BLtotB/P9CQi0gKL6NQkZFLJXW96Z/TvgPIKNT6JuoT3d16EqZEMz/TxFCdAahR5JUq8v/MCAODlIZ3g1obFTGocNubGeCyoHR4LaocKlRqnUqvbvlzMQkodbV+G+Fa1fQlyZ9sXImqZjiTloLC8Ek7Wpuju3kbscKiBJBIJRgU4Y92hFETEZ7CIfg85ReWYtSkWFSoBI/2dMb2ft9ghERmcP05fw2u/nIUgABNC2+OhDm2x9O/aZ4k728ixaLQfRvjz2hpERNpgEZ2a1UtDOkGpUuPLvUlY9Nd5GMukGB/aXuyw9IJKpcKhQ4cAAP369YNMpv+n+374TyJyi5Xo5GiJaX29xA6HWihjmRRhHewR1sEeCx6pavuy50JV25dTV/9t+7J6P9u+EJH4miqf74yrauUyoqszpPyw0KCMDHDBukMp2HMhC2UVKsiN9f89XnOrVKnx8k+nkZFfBm8HC3z4ZDe2LCLS0rbT1/Haz1UF9HG92mPpY/6QSiUY7n/3WeJcdHJvhnhsTkRNj0V0anZzh3ZGeaUaaw9ewTvb4mBiJMWTIW5ih0VaOp12Cz9FpwEAlo3x5wWfqNnUtH2Z3t8beSVKHLiUXW/bl97e9hji44ghbPtCRAZMWanGrvP/9kMnwxLkZgsXGzky8stw+HIOrx9Th0+iLuFo8k2Ym8jw9cQQfghOpKU/z1zH3J/PQC0A43q5479j/DUfuNZ1ljgREWmPRXRqdhKJBPNH+kBZqcaGo6mY9+tZmBhJ8Wigq9ihUQNVqtR45494CALwRLAbQr35pozEYWtucs+2L4cu5+DQ5Rws3p6ALk5WGOLriCFs+0JEBubYlZsoKKtEW0sT9PTkReAMjVQqwfCuzthwNBU74zNYRL/DP+cVWL0/GQDwwRPd0MnJSuSIiAzLX2dv4NWtVQX0p3u6479jAnjGEhFRE2ARnUQhkUiwaLQfyivV+Ck6Da9uPQMTmYR92QzED8euIiGjADZmxpg/ykfscIgA3N32JTm7CHsuZGL3hSzEXL2FxMxCJGYWYtX+ZNhbmGBgF0eE+zqiX2cHWJrePx2q1AJPhSUiUUTGZwAAhnd15uuOgRoV4IINR1OxOyETyko1z+CrdiW7CK//fBYA8OxDXhjNRTVEWtl+9gbmbDkNtQA81cMN7z/OAjoRUVPR6t3b8uXL0bNnT1hZWcHR0RFjxoxBYmKi5vupqamQSCR13n755RfNuLS0NDz88MMwNzeHo6Mj3njjDVRWVtba1/79+xEcHAxTU1N07NgRGzZseLBnSnpHIpHgv2P88USwG1RqAS/9dBp7LmSKHRbdR2ZBGVZEXQIAzBvRBW0tTUWOiKhuHRwsMaN/B/z8fBhiFoTj86eDMDrQFVZyI9wsVuK32GuYuSkWwe9FYdK3J/D90VSk55bUua3I+Az0/WAvxq07jle2nMG4dcfR94O9msIWEVFTqVSp8c/5qvdHo9jKxWCFeLSBg5UpCsoqcTQ5R+xw9EKJshIzN8aisLwSPT3bcGEGkZZ2nLuBOdUr0P8T4ob//V83FtCJiJqQVkX0AwcOYNasWTh+/DiioqJQUVGBYcOGobi4GADg7u6OjIyMWrclS5bA0tISI0eOBFB1gYaHH34YSqUSR48exffff48NGzZg4cKFmv2kpKTg4YcfxqBBg3DmzBnMmTMHzz33HP75559GfOqkD6RSCT58shtGB7qiQiVg5sZYHLyULXZYdA9LdySgqLwSge62GNeTF4Ulw1DT9uXLcd0R++5QbJ4eiml9veBpbw6lSo1Dl3Ow6K/z6PfhPgz/9CA+jLyImKu3oFILiIzPwMyNscjIL6u1TUV+GWZujGUhnYiaVHRKLnKLlWhjboxQL7ZyMVQyqQQjujoDACKqLxLbmgmCgPm/xyExsxAOVqZYOT4YxjKuzidqqL/PZeCVLWegUgt4MsQNHzzBAjoRUVPTqp1LZGRkra83bNgAR0dHxMTEoH///pDJZHB2dq415o8//sBTTz0FS0tLAMCuXbuQkJCA3bt3w8nJCUFBQVi6dCnefPNNLF68GCYmJlizZg28vLzwySefAAB8fX1x+PBhfPrppxg+fPiDPF/SQzKpBCueCoSyUoV/zmdixo+n8N2UXrz4iR46dDkbO85lQCpBrYvVEBkSY5kUfTq0RZ8ObbHgYV8kZxdj78Wqti+nUnNrtX2xMzdGaYUaQh3bEQBIACzZnoChfmyxQERNY+dtrVyMWGQ0aCMDnPHj8av4J0GBZSr/Vl00/uHYVfx55gZkUglWjg+Go7Vc7JCIDEZEXAZe3nIaKrWA/wtuxwI6EVEzeaB3bvn5+QAAO7u6V8XExMTgzJkzmDZtmua+Y8eOISAgAE5O/15QZ/jw4SgoKMD58+c1Y8LDw2tta/jw4Th27Fi9sZSXl6OgoKDWjQyHsUyKL8cFY7CPI8oq1Jj2/UmcSs0VO6xmJZFIYGdnBzs7O0gk+vcmqKxChXe3xQMAJod5wr+djcgRET04iUSCjo7/tn2JfXcoPhsbhEe6ucBKboTckgqUVqjqfbwAICO/DNEprev1iojq15j5vOpsmKpWLiP8ne8zmvRdL0872FuYIK+kAieutN68EXM1F0t3JAAA5o/0QS+eYUHUYJHxGXjpp+oCevd2+OjJQC7kaAL6fmxOROLQuYiuVqsxZ84cPPTQQ/D3969zzLfffgtfX1/06dNHc59CoahVQAeg+VqhUNxzTEFBAUpLS+vc1/Lly2FjY6O5ubu76/rUSCQmRlKsmhCMfp3aokSpwpTvTuJMep7YYTUbqVSKbt26oVu3bpBK9W9l0tcHriD1ZgkcrUzx2rDOYodD1CRszU0wpns7fDU+GLHvDsXsQR0a9LiswrL7DyKiVqEx8/mp1FzkFJXDWm6EPh3aNlKEJBYjmRTDqlu67GylrcCyC8vx4qZYVKoFPNzNBdP6eokdEpHBiIxXYPbm06hUC3i8ezt89B8W0JuKvh+bE5E4dH41mDVrFuLj47Fly5Y6v19aWorNmzfXWoXelObPn4/8/HzNLT09vVn2S41LbizD2kk90NvbDkXllZj87Qmcv5EvdlitXmpOMVbuTwIAvPuIH6zkxiJHRNT0jGVSPNTRoUFjHa14GjoRNb6I+KoFJkP9nGFixIP4lmBk9RkF/8QroFLX1Sys5apUqfHST7HILChHR0dLfPBEN67wJGqgf84rMHtz1QdQjwW54mMW0ImImp1O78Znz56NHTt2YN++fXBzc6tzzK+//oqSkhJMnjy51v3Ozs7IzMysdV/N1zX91OsbY21tDTMzszr3Z2pqCmtr61o3MkxmJjJ8+0xPhHi0QUFZJSZ+cwKJikKxw2q1BEHAwr/OQ1mpRt+ObfFINxexQyJqNr287OBiI0d9hygSAC42cp6KTkSNTq0WEFG9WnlUAFu5tBRhHexhY2aMm8XKVtcK7KN/EnH8Si4sTGRYMzEElqZaXZ6LqNXadV6BWdVncDwa6IpPWEAnIhKFVkV0QRAwe/Zs/PHHH9i7dy+8vOo//e7bb7/Fo48+CgeH2qv4wsLCEBcXh6ysLM19UVFRsLa2hp+fn2bMnj17aj0uKioKYWFh2oRLBszC1AjfTe2JQDcb3CqpwIRvjiMpq0jssJqUSqXCwYMHcfDgQahU9fdgbm4R8QocvJQNE5kU7z3WlSuGqFWRSSVYNLoqN935m1/z9aLRfjyQISKNxsrnp9PzkFlQDktTI/TtxFYuLYWxTIphflVtKyNbUUuXiLgMfH3wCgDgo/8EoqOjpcgRERmG3QmZmFW9An10oCtWPBXIi0w3A309NicicWn16jtr1ixs3LgRmzdvhpWVFRQKBRQKxV19ypOSknDw4EE899xzd21j2LBh8PPzw6RJk3D27Fn8888/WLBgAWbNmgVTU1MAwAsvvIArV65g3rx5uHjxIlatWoWff/4Zr7766gM8VTI01nJj/PBsKPxcrJFTpMSEb47j6s1iscNqUmq1Gmq1WuwwNIrKK/He9qoLP70wsAO8HXjAQ63PCH8XrJ4YDGeb2i1bnG3kWD0xGCP8eXYGEdXWGPk8Iq6qwBru6whTI1ljhEV6YlRAVd6IiFdA3QpauiRnF+GNX88BAKb389I8fyK6tz0XMjFzUwwqVAIe6eaCT1lAb1b6dmxOROLT6hy61atXAwAGDhxY6/7vvvsOU6ZM0Xy9fv16uLm5YdiwYXdtQyaTYceOHZg5cybCwsJgYWGBZ555Bu+9955mjJeXF/7++2+8+uqr+Pzzz+Hm5oZvvvkGw4cP1yZcagFszI2x8blQPL32GC5lFmH8uhPY+nxvuLUxFzu0VuHTqEtQFJShvZ05XhzYsAssErVEI/xdMNTPGdEpucgqLIOjVVULF65AJ6KmIAiCph/6SBYcW5w+He1hJTdCVmE5YtNuoYdny20JVlxeiRd+jEFReSV6ednhzRE+YodEZBD2XszEzI2xqFBVXYT3s7FBLKATEYlM63Yudd1uL6ADwPvvv4+0tLR6r2Ls4eGBnTt3oqSkBNnZ2fj4449hZFS7nj9w4ECcPn0a5eXlSE5Ovmsf1HrYWZhg03O94e1gget5pRi/7gQy8kvv/0B6IAk3CrDhaCoA4L3HukJuzFVw1LrJpBKEdbDHY0HtENbBngV0Imoy567l43peKcxNZBjQuWEXOCbDYWokw1DfqpYuO+MUIkfTdARBwJu/ncPlrCI4Wpniq/HdWQTUEytXroSnpyfkcjlCQ0MRHR3doMdt2bIFEokEY8aMadoAW7l9F7Pwwo+xUKrUeDjABZ+zgE5EpBf4SkwGwcHKFJuf6432duZIyy3BhHUnkFVYJnZYLZZaLWDBtjio1AJGBThjYBdHsUMiIiJqNXZW98oe5OPID7FbqJGali4ZLbaly3dHUrHjXAaMpBKsmhAMRyv5/R9ETW7r1q2YO3cuFi1ahNjYWAQGBmL48OG1rllWl9TUVLz++uvo169fM0XaOu1LzMLzP8ZAqVJjpL8zPnuaBXQiIn3BV2MyGM42cmyeHop2tma4klOMCetO4GZRudhhtUg/n0pHbFoeLExkePcRP7HDISIiajUEQUBE9erkUbzmQovVr1NbWJjIkJFfhrPX8sQOp9GdTM3F+zsvAADeedi3RbesMTQrVqzA9OnTMXXqVPj5+WHNmjUwNzfH+vXr632MSqXChAkTsGTJEnh7ezdjtK3L/tsK6CO6OuOLcd1hzAI6EZHe4CsyGRS3NubYPD0UztZyXM4qwsRvo5FXohQ7rBYlt1iJ/0VeBAC8OrQzXGzMRI6IiIio9Th/owBpuSWQG0sxsAtbubRUcmMZBle3dKnpf99SZBWU4cVNsahUC3g00BVT+niKHRJVUyqViImJQXh4uOY+qVSK8PBwHDt2rN7Hvffee3B0dMS0adMatJ/y8nIUFBTUutG9HbiUjRk/xkBZqcbwrk74cjwL6ERE+oavymRwPOwtsGl6KNpamuJCRgEmr49GQVmF2GE1CltbW9ja2ooaw/8iLiCvpAI+zlY86CEiItLBg+TzyOqC6sDOjrAwNbrPaDJko/ydAVS1dBGEltHSpUKlxuzNp5FdWI7OTpZY/n8BkEh4DRF9kZOTA5VKBScnp1r3Ozk5QaGo+8Ocw4cP49tvv8W6desavJ/ly5fDxsZGc3N3d3+guFu6g5eyMf2HU1BWqjHMzwlfjgtmAV0P6MOxORHpF74yk0Hq4GCJzdNDYWdhgnPX8jFlfTSKyivFDuuByGQyBAUFISgoCDKZOP1PT6Xm4udT1wAA/33cn/33iIiItPQg+VwQBOyMq+qHPjLAuSnCIz0ysIsjzIxlSM8txfkbLWOl7gcRFxGdmgtLUyOsmRjCD4IMXGFhISZNmoR169ahbdu2DX7c/PnzkZ+fr7mlp6c3YZSG7fDlHE0BfaifE74aHwwTIx6DiU0fjs2JSP/w1ZkMVmcnK2ycFgobM2PEpuVh2oaTKFWqxA7LYFWo1Hjnj3gAwNge7gjxYO9KIiKi5nQpswhXcophYiTFYB9e1LulMzORYZBPVcuemg9PDNnf5zLwzeEUAMDH/+kGbwdLkSOiO7Vt2xYymQyZmZm17s/MzISz890f3CUnJyM1NRWjR4+GkZERjIyM8MMPP+Cvv/6CkZERkpOT69yPqakprK2ta93obkeScjDt+5Mor1Qj3NcRK1lAJyLSa3yFJoPm52qNH57tBStTI5xIycWMH0+hrIKFdF1sOJKKxMxCtDE3xlsjfcQOh4iIqNWpKaT279QWVnJjkaOh5jCy+uKxO+MMu6VLUlYh3vj1LADg+QHeGMGL4uolExMThISEYM+ePZr71Go19uzZg7CwsLvG+/j4IC4uDmfOnNHcHn30UQwaNAhnzpxhm5YHcPS2AvoQH0esnMACOhGRvuOrNBm8QHdbbHi2J8xNZDh0OQczN1ZdkMXQqFQqHDlyBEeOHIFK1bwfBNzIK8Wnuy8BAOaP9EUbC5Nm3T8REVFL8SD5PCK+upULC5CtxiAfR5gYSZF6swQXFYVih6OTovJKPP9jDEqUKoR52+ONYV3EDonuYe7cuVi3bh2+//57XLhwATNnzkRxcTGmTp0KAJg8eTLmz58PAJDL5fD39691s7W1hZWVFfz9/WFiwmMGXRxNzsGz359EWYUag30csWpiMEyN2DJEn4h5bE5E+otFdGoRQjzssH5KT8iNpdiXmI2XfopFhcrwCukVFRWoqGj+i6S+tz0BJUoVQjza4MkQt2bfPxERUUuiSz5PyirEpcwiGMskCPd1uv8DqEWwNDXCgM5VLV0iDLCliyAImPfrWSRnF8PZWo4vxnXnNXX03NixY/Hxxx9j4cKFCAoKwpkzZxAZGam52GhaWhoyMgzvd9FQHEu+iWc3VBXQB3VxwGoW0PWWWMfmRKS/+A6HWoze3vb4ZnJPmBhJ8c/5TLy69QwqDbCQ3tz2XcxC5HkFZFIJlo3xh1QqETskIiKiViciTgEAeKhjW9iYs5VLazKq+iKyO+MVIkeivW8Pp2BnnALGMglWTgiGg5Wp2CFRA8yePRtXr15FeXk5Tpw4gdDQUM339u/fjw0bNtT72A0bNmDbtm1NH2QLdPzKvwX0gV0csHpiCAvoREQGhEV0alH6dmqLNRODYSyTYMe5DMz79RzUasPtL9nUyipUWPhX1cVEn33IE74uvOgPERGRGCKqC6ij2Mql1Rni6wRjmQRJWUW4nGk4LV1OXLmJ5REXAQDvPuKHEI82IkdEpL9OXLmJqd+dRGmFCgM6O2DNxBDIjVlAJyIyJCyiU4sz2McJX44Lhkwqwe+nr+PtP+JYSK/Hyn1JSM8thbO1HHPCO4sdDhERUauUmlOMhIwCyKQSDPVjK5fWxlpujH6dqlu6GMhq9MyCMszafBoqtYAxQa6Y1NtD7JCI9FZ0Si6mbqgqoPfr1BZfT2IBnYjIELGITi3SCH9nfDY2CFIJsOVkOpZsPw9BYCH9dsnZRVhzIBkAsGi0HyxMjUSOiIiIqHWqKZz26WDPi3u3UiP9q1u6GEBf9AqVGrM2xSKnqBxdnKzw/v8FQCJhO0CiupxMzcWU76JRoqwqoK+b3IMFdCIiA8UiOrVYowNd8fF/AiGRAN8fu4r3d15gIb2aIAh4d1s8KlQCBnZxwIjqAzciIiJqfhHxVYVT5uPWa6ifE4ykElxUFOJKdpHY4dzT+zsv4NTVW7AyNcKaSSEwN+FCDKK6nErNxZT1VQX0vh1ZQCciMnQsolOL9n/Bbnj/8QAAwLpDKfhk1yWRI7o3KysrWFlZNfl+/jp7A0eTb8LUSIr3HvXn6iEiIqJGpE0+T88twblr+ZBKgGF+LKK3VrbmJgjrYA9Av1u6/HX2Br47kgoA+OSpQHi1tRA3ICI9FXM1F8+sj0axUoWHOtqzgG6AmuvYnIgMB5cNUIs3rld7VKjUWPjneXy1LwmmRlK8NKST2GHdRSaTISQkpMn3U1BWgWV/XwAAzBrUEe3tzZt8n0RERK2Ftvk8srpg2svLDg5Wpk0VFhmAUQEuOHQ5BxHxGZg1qKPY4dzlUmYh3vz1HADgxYEdMKwrP/QhqkvM1Vt4Zv1JFCtV6NPBHt9M7gkzExbQDUlzHZsTkWHhSnRqFSaHeWLBw74AgE+iLuHr6l7grdEn/yQiu7Ac3m0t8PwAb7HDIaJGsHLlSnh6ekIulyM0NBTR0dH3HP/LL7/Ax8cHcrkcAQEB2LlzZ63vC4KAhQsXwsXFBWZmZggPD8fly5drjfnvf/+LPn36wNzcHLa2tnXuJy0tDQ8//DDMzc3h6OiIN954A5WVlQ/0XIlamp3VrVxGBbiIHAmJbZifE6QSIP56AdJulogdTi2FZRV44ccYlFZUrap9bVgXsUMi0kuxabfwzPpoFJVXIszbHt8+wwI6EVFLwSI6tRrP9fPGG8Or3vAvj7iI746kiBxR84u7lo8fj18FACwd4w9TI76hIzJ0W7duxdy5c7Fo0SLExsYiMDAQw4cPR1ZWVp3jjx49inHjxmHatGk4ffo0xowZgzFjxiA+Pl4z5sMPP8QXX3yBNWvW4MSJE7CwsMDw4cNRVlamGaNUKvGf//wHM2fOrHM/KpUKDz/8MJRKJY4ePYrvv/8eGzZswMKFCxt3AogMWEZ+KU6n5UEiAYZzVW+rZ29pit7eVS1dIs/rzwVGBUHAG7+cw5WcYrjYyPHF090hk7IVINGdTqfdwjPfVhXQe3vb4dspPVhAJyJqQVhEp1Zl1qCOeHlw1emxS7YnYNOJqyJH9C+VSoXjx4/j+PHjUKlUjb99tYAF2+KgFoBHA13xUMe2jb4PImp+K1aswPTp0zF16lT4+flhzZo1MDc3x/r16+sc//nnn2PEiBF444034Ovri6VLlyI4OBhfffUVgKpiyWeffYYFCxbgscceQ7du3fDDDz/gxo0b2LZtm2Y7S5YswauvvoqAgIA697Nr1y4kJCRg48aNCAoKwsiRI7F06VKsXLkSSqWy0eeBSF9ok89rWrn08GgDJ2t5c4RHem5k9RkJO+P0py/62oNXEHleAWOZBKsmBMPekm2HiO50Jj0Pk7+NRmF5JUK97LB+Sk9edNeANfWxOREZJhbRqdV5dWhnTRuTd/6Ixy+n0kWO6F9lZWW1Vno2ps3RaTh7LR9Wpkaa1jZEZNiUSiViYmIQHh6uuU8qlSI8PBzHjh2r8zHHjh2rNR4Ahg8frhmfkpIChUJRa4yNjQ1CQ0Pr3WZ9+wkICICTk1Ot/RQUFOD8+fN1Pqa8vBwFBQW1bkSGqKH5PKK6UDrSn61cqMrwrk6QSKoKctfzSsUOB0eTc/BB5EUAwMLRXdG9fRuRIyLSP2fT8zDp2xMoLK9ELy87fDeVBfSWoCmPzYnIMLGITq2ORCLBWyN8MKWPJwBg3m/n8OeZ6+IG1cSyC8vxYfUB0GvDOsORq92IWoScnByoVKpahWoAcHJygkJR9ypGhUJxz/E1/2qzTW32c/s+7rR8+XLY2Nhobu7u7g3eH5GhySoow8mruQCAEf5s5UJVHK3k6OlpB+DfMxXEosgvw8s/nYZaAP4vuB0mhrYXNR4ifXTuWh4mfnsChWWV6OVph++4Ap2IqMViEZ1aJYlEgkWj/TA+tD0EAZj781lExOlP78nGtnznBRSWVaKrqzUmhXmKHQ4RUZ3mz5+P/Px8zS09XX/OFCJqbP+cV0AQgCB3W7jamokdDumRkdUfqoj53lRZqcaLm2KQU6SEj7MV/jsmABIJ+6AT3S7uWj4mflNVQO/p2QbfTe0JC1MW0ImIWioW0anVkkgkWPaYP54McYNKLeCln05jd0Km2GE1umPJN/H76euQSID/Ph7AC0ERtSBt27aFTCZDZmbt167MzEw4O9e9stXZ2fme42v+1Wab2uzn9n3cydTUFNbW1rVuRC1VTc/rUQFchU611ZyZcOrqLSjyxWkl8N+/ExCblgcruRG+nhTCiyMS3SH+ej4mfnsCBWWV6OHRBt9N7cUCOhFRC8ciOrVqUqkEHzzRDY8GuqJSLeDFTbE4cClb7LAajbJSjXf/jAcAjO/VHkHutuIGRESNysTEBCEhIdizZ4/mPrVajT179iAsLKzOx4SFhdUaDwBRUVGa8V5eXnB2dq41pqCgACdOnKh3m/XtJy4uDllZWbX2Y21tDT8/vwZvh6glyikqx4mUmwDYD53u5mJjhuD2tgCqzlhobttOX8f3x64CAD4bGwQPe4tmj4FIn8Vfz8eEb04gv7QCIR5tsOHZXrBkAZ2IqMVjEZ1aPZlUghVPBWKkvzOUKjVm/HAKR5NzxA6rUXxz+AqSsorQ1tIE84b7iB0OETWBuXPnYt26dfj+++9x4cIFzJw5E8XFxZg6dSoAYPLkyZg/f75m/CuvvILIyEh88sknuHjxIhYvXoxTp05h9uzZAKrO0pkzZw6WLVuGv/76C3FxcZg8eTJcXV0xZswYzXbS0tJw5swZpKWlQaVS4cyZMzhz5gyKiooAAMOGDYOfnx8mTZqEs2fP4p9//sGCBQswa9YsmJqaNt8EEemhXeczoRaAgHY2cLczFzsc0kOjAqo+XImIb96WLhcVBXjr93MAgJcGd8QQX6f7PIKodTl/o2oFen5pBYLb22LD1J4soBMRtRJ8tScCYCST4vOnu6NiUwx2X8jCtA2n8MO0XpoLOzUXc/PGO5BOzy3BF3suAwDeHuULG3PjRts2EemPsWPHIjs7GwsXLoRCoUBQUBAiIyM1F/FMS0uDVPrvZ+Z9+vTB5s2bsWDBArz99tvo1KkTtm3bBn9/f82YefPmobi4GDNmzEBeXh769u2LyMhIyOX/XpR44cKF+P777zVfd+/eHQCwb98+DBw4EDKZDDt27MDMmTMRFhYGCwsLPPPMM3jvvfeaekqIRHe/fF5TGB3JVi5UjxH+zlj29wVEp+Qiu7AcDlZN/+FjQVkFXvgxBmUVavTr1BZzwjs3+T6JDEnCjQJM+OYE8koq0L29Lb5/thes5DzGaqka89iciFoGiSAIgthBNIWCggLY2NggPz+fPVWpwcorVZj+QwwOXsqGpakRfpzWC93btxE7LJ089/1J7L6QhVAvO2yZ0ZsXgyLSM8xT98c5opboVrESPf67Gyq1gH2vD4RXW7bKoLo99tVhnL2Wj/8+7o8JoR5Nui+1WsDzG2MQlZCJdrZm2P5SX9hZmDTpPlsK5qr7awlzVFVAP45bJRUIcrfFD9N6wZoFdCKiFqGheYrtXIhuY2okw9cTQxDmbY+i8kpMXh+N+Ov5YoeltV3nFdh9IQtGUgmWjfFnAZ2IiEhPRF3IhEotwMfZigV0uqeRNS1d4pq+L/qag8mISsiEiUyKVROCWUAnus2FjH8L6IEsoBMRtVosohPdwcxEhm+e6YEeHm1QWFaJSd+ewEVFgdhhNViJshJLticAAKb390YnJyuRIyIiIqIaEXFVrVxqel4T1Wekf1W7n2NXbiK3WNlk+zmSlIOP/0kEACx+tCsCeSF6Io2LiqoWLrdKKhDoZoMfnmUBnYiotWIRnagOFqZG+G5qTwS62+JWSQUmfnMCSVlFTbpPlUqF6OhoREdHQ6VS6bydL/Yk4XpeKdrZmuGlwR0bMUIiIiK6n3vl8/zSChxOqrp4+Sj2Q6f78LC3gJ+LNVRqAVEJTbMa/UZeKV766TTUAvCfEDeM6+XeJPshMkSJikKMX3cCucVKdHOzwQ/TQmFjxgJ6a9BYx+ZE1LKwiE5UDyu5MX6Y2gtdXa2RU6TE+HXHkZpT3KT7LCkpQUlJic6Pv5RZiG8OXQFQtZLI3ITXDiYiImpu9eXzPRcyUaES0MnREh0deaYY3V/Nhy07m6ClS3mlCi9uikVusRJdXa2xlC0AiTQuZRZi/LrjyC1WIqCdDX58lgX01uZBj82JqOVhEZ3oHmzMjfHjtFB0cbJCVmE5xq87jvRc/UykgiBgwbZ4VKoFhPs6Yaifk9ghERER0W1qCqEj2cqFGqjmd+Vocg7ySyoaddtLdyTgTHoebMyMsXpCCOTGskbdPpGhulxdQL9ZrIR/O2tsnBYKG3MW0ImIWjsW0Ynuw87CBBufC0UHBwvcyC/D+G+OIyO/VOyw7vJ77HVEp+RCbizF4kf9xA6HiIiIblNUXomDl7MBsJULNVwHB0t0cbJChUrA7guZjbbd32OvYePxNEgkwGdjg9De3rzRtk1kyJKyCjFu3QnkFFWdocECOhER1WARnagBHKxMsXl6b3jYmyM9txTj151AVkGZ2GFp5JUo8f7OCwCAl4d0glsbHggRERHpk70Xs6CsVMO7rQW68KLfpIWR1R+6RMRnNMr2Em4U4O0/4gAALw/uhEE+jo2yXSJDl5RVhKfXnkBOUTn8XKyx6blQ2JqbiB0WERHpCRbRiRrIyVqOzdN7o52tGVJyijH+m6o3WPrgw38ScbNYiU6Olniur7fY4RAREdEdIuKqCqAj/J3Zd5q0Mqq6pcvBSzkoLHuwli75pRWYuSkGZRVqDOjsgFeGdGqMEIkMXlJWEcatO46conL4soBORER1YBGdSAvtbM3w0/TecLGRIymrCBO/OYG8EqWoMZ1Ou4WfotMAAEvH+MPEiH/WRERE+qREWYl9iVkA/i2IEjVUJ0dLeDtYQKlSY+/FLJ23o1YLeO3nM7h6swTtbM3w2dggSKX8QIcoObuqgJ5dWA4fZytsei4UbSxYQCciotpYbSPSUnt7c2x6LhQOVqa4qCjEpG+jUfCAq4JqyOVyyOXyBo+vVKmxYFs8BAH4v+7t0NvbvlHiICIiIt3dmc/3J2ajrEINdzszdHW1FjEyMkQSiQSj/Ks+fNkZp3tLl1X7k7D7QhZMjKRYMzGERUIiAFeyizBu7b8F9M3Te8OOfxsE7Y/NiajlYxGdSAfeDpbY/Fwo7CxMEHc9H8+sj0ZReeUDbVMmk6F3797o3bs3ZDJZgx7z4/GrOH+jANZyI7z9sO8D7Z+IiIgeXF35vKbwOcrfha1cSCc1fdH3J2ajWIf3nIcuZ+OTqEsAgKWPdUWAm02jxkdkiFJyijFu3XFkFZaji1PVCnQW0AnQ7diciFo+FtGJdNTJyarqau1mxjidlodnvzuJEuWDFdK1kVlQhk92VR0MzRvhg7aWps22byIiImqYsgqVpgXHSLZyIR35uVjDw94c5ZVq7E/M1uqx1/NK8fJPpyEIwNM93TG2Z/smipLIcKTmFGPc2uPILChHZydLbJoeCnseTxER0T2wiE70APxcrfHjtF6wMjVCdGoupv9wCmUVqmbZ97K/L6CovBKB7rYY14sHQ0RERProwKVslChVcLWRI5Crf0lHEokEI2tausQ3vKVLeaUKL26Mwa2SCgS0s8HiR7s2VYhEBiM1pxhPrz0ORUEZOjlaYvP03lyQRERE98UiOtED6uZmiw3P9oKFiQxHkm7ihY0xKK/UvpCuUqkQExODmJgYqFT3fvyhy9nYfvYGpBLgv2P8IeNFoYiIiPTCnfk8Ml4BoGoVOlu50IMYVd3SZd/FLJQqG/Zec8n2BJy9lg9bc2OsmhAMuTHbElDrdvVmVQsXFtDpXrQ5Niei1kOrIvry5cvRs2dPWFlZwdHREWPGjEFiYuJd444dO4bBgwfDwsIC1tbW6N+/P0pLSzXfz83NxYQJE2BtbQ1bW1tMmzYNRUVFtbZx7tw59OvXD3K5HO7u7vjwww91fIpETS/Eow3WT+kJubEU+xOzMXvzaVSo1Fpvp7CwEIWFhfccU1ahwsI/zwMAJod5wr8dV7URERHpk5p8Xl6pwu6ETAD/FkCJdBXQzgbtbM1QolThwKX7t3T55VQ6Np9Ig0QCfP50d7jbmTdDlET6K+1mCcatPY6M/DJ0rC6gO1ixgE51a8ixORG1LloV0Q8cOIBZs2bh+PHjiIqKQkVFBYYNG4bi4mLNmGPHjmHEiBEYNmwYoqOjcfLkScyePRtS6b+7mjBhAs6fP4+oqCjs2LEDBw8exIwZMzTfLygowLBhw+Dh4YGYmBh89NFHWLx4MdauXdsIT5moaYR62+ObyT1hYiRFVEIm5mw5g0odCun3s/bgFaTkFMPByhRzh3Vu9O0TERFR4zialIPC8ko4WZuiu3sbscMhA1fV0qXqw5iI+7R0ib+ejwXb4gEAc4Z0xoDODk0eH5E+S88twbh1x3EjvwwdHCyweXooC+hERKQVrYrokZGRmDJlCrp27YrAwEBs2LABaWlpiImJ0Yx59dVX8fLLL+Ott95C165d0aVLFzz11FMwNa1KUBcuXEBkZCS++eYbhIaGom/fvvjyyy+xZcsW3LhxAwCwadMmKJVKrF+/Hl27dsXTTz+Nl19+GStWrGjEp07U+Pp2aouvJ4XAWCbB33EZeOPXc1CphUbb/tWbxfhqXxIAYMHDvrCWGzfatomIiKhxRZ6vWoU+oqszpGy9Ro2g5uK0ey5k1XsdnvySCszcFIPySjUGdXHAS4M7NmeIZABWrlwJT09PyOVyhIaGIjo6ut6xv//+O3r06AFbW1tYWFggKCgIP/74YzNG++DSc0vw9NrjuJ5XCm8HC/w0vTccreRih0VERAbmgXqi5+fnAwDs7OwAAFlZWThx4gQcHR3Rp08fODk5YcCAATh8+LDmMceOHYOtrS169OihuS88PBxSqRQnTpzQjOnfvz9MTEw0Y4YPH47ExETcunXrQUImanKDujhi5fhgGEkl+OP0dbz9exzUjVBIFwQBC/88D2WlGg91tMejga6NEC0RERE1BZVaQFTCv/3QiRpDd3dbOFvLUVReicOXc+76vlotYM7W00jPLYW7nRk+G9udH+BQLVu3bsXcuXOxaNEixMbGIjAwEMOHD0dWVlad4+3s7PDOO+/g2LFjOHfuHKZOnYqpU6fin3/+aebIdVOrgN7WAlum94ajNQvoRESkPZ2L6Gq1GnPmzMFDDz0Ef39/AMCVK1cAAIsXL8b06dMRGRmJ4OBgDBkyBJcvXwYAKBQKODo61tqWkZER7OzsoFAoNGOcnJxqjan5umbMncrLy1FQUFDrRiSWYV2d8fnT3SGVAFtPpWPRX+chCA9WSI+IV+DApWyYyKRY+pg/L05GRESkx9JvlaCwrBJtLU3Q09NO7HCohZBKJRihaely93HRV/uSsC8xG6ZGUqyeEAIbc561SLWtWLEC06dPx9SpU+Hn54c1a9bA3Nwc69evr3P8wIED8fjjj8PX1xcdOnTAK6+8gm7dutVaKKevrt2qauFSU0D/aQYL6EREpDudi+izZs1CfHw8tmzZorlPra7q//z8889j6tSp6N69Oz799FN06dKl3qTcWJYvXw4bGxvNzd3dvUn3R3Q/D3dzwSdPBUIiAX48fhXL/r6gcyG9qLwS721PAAA8P8Ab3g6WjRkqERERNbKkzCIAwPCuzpBxJTA1olHVZzZEJSigrPz3+jv7E7Pw6e5LAIBlY/x58Xm6i1KpRExMDMLDwzX3SaVShIeH49ixY/d9vCAI2LNnDxITE9G/f/+mDPWBXc8rxbh1x3HtVim8qgvoTiygExHRAzDS5UGzZ8/WXBDUzc1Nc7+LS9UbOj8/v1rjfX19kZaWBgBwdna+61SxyspK5ObmwtnZWTMmMzOz1piar2vG3Gn+/PmYO3eu5uuCggIW0kl0j3d3g7JSjTd/i8O3h1NgaiTFG8O71LuK3Ni47tVCn0VdgqKgDO3tzDFrEPtaEhFR01OpBUSn5CKrsAyOVnL08rJjMbgBVGoB1/OVSMgsBmCOEV3rfu9KpKsQjzZwsDJFdmE5vj18Ba62ZpAAePfPeAgCMD60Pf7Tg8dBdLecnByoVKo6z/q+ePFivY/Lz89Hu3btUF5eDplMhlWrVmHo0KH1ji8vL0d5ebnm6+Y+S/xGXimeXnsM6bml8LQ3x0/TWUAn7dV3bE5ErZdWRXRBEPDSSy/hjz/+wP79++Hl5VXr+56ennB1dUViYmKt+y9duoSRI0cCAMLCwpCXl4eYmBiEhIQAAPbu3Qu1Wo3Q0FDNmHfeeQcVFRWaF66oqCh06dIFbdq0qTM2U1NTzcVLifTJ2J7toaxU490/z2PV/mSYGsnwSninu8bJZDI89NBDd91/IaMA3x1NBQAseawr5Maypg6ZiIhaucj4DCzZnoCM/DLNfS42ciwa7YcR/uzvXZ9/560CQFWR6o1fz2Hxo5w3ajwyqQR+LtY4UJiNDyJrH3d52Jtj0Wi/eh5JpBsrKyucOXMGRUVF2LNnD+bOnQtvb28MHDiwzvHLly/HkiVLmjfIalUF9ONIzy2Fh705fprRG842LKCTduo7Niei1k2rdi6zZs3Cxo0bsXnzZlhZWUGhUEChUKC0tBQAIJFI8MYbb+CLL77Ar7/+iqSkJLz77ru4ePEipk2bBqBqVfqIESMwffp0REdH48iRI5g9ezaefvppuLpWXShx/PjxMDExwbRp03D+/Hls3boVn3/+ea2V5kSGZFKYJxY87AsA+HT3Jazen9ygx6nVAhZsi4dKLWCkvzMGdXG8/4OIiIgeQGR8BmZujK1VQAcARX4ZZm6MRWR8hkiR6bf65i2zgPNGjSsyPgMHLmXX+b2rN0uw72LdF4gkatu2LWQyWZ1nfdd3xjdQ1fKlY8eOCAoKwmuvvYYnn3wSy5cvr3f8/PnzkZ+fr7mlp6c32nO4l4z8qhYuabklaG9XtQLdxcasWfZNREQtn1ZF9NWrVyM/Px8DBw6Ei4uL5rZ161bNmDlz5mD+/Pl49dVXERgYiD179iAqKgodOnTQjNm0aRN8fHwwZMgQjBo1Cn379sXatWs137exscGuXbuQkpKCkJAQvPbaa1i4cCFmzJjRCE+ZSBzP9fPGvBFdAAAfRF7E+sMp933MLzHpiLl6C+YmMrz7CFcVERFR01KpBSzZnoC6ruBRc9+S7QlQqR/sYtktDeeNmkvN71p9JODvGtXPxMQEISEh2LNnj+Y+tVqNPXv2ICwsrMHbUavVtdq13MnU1BTW1ta1bk1NkV+GcWuP4+rNqgL6lhm94WrLAjoRETUerdu5NMRbb72Ft956q97v29nZYfPmzffcRrdu3XDo0CFtwiPSey8O7IjyCjU+33MZ7+1IgImRFBN7ewAAVCoV4uLiAAABAQHIL1NheURVb8JXwzvzTSARETUKQRBQolThVokSeSUVuFWixK2SCtwqVuLctby7VlLXeiyAjPwy9P3fXpiZyGoVjWveJwqar29/nHD3fXW8rbxzG7ePE26799/76tre/cbdYx+3BVXH5m57brW3oVILKK++wKMEanSQ5QIAklV2ECDVzFt0Si7COtiDSFfRKbkN+hvl7xrVZ+7cuXjmmWfQo0cP9OrVC5999hmKi4sxdepUAMDkyZPRrl07zUrz5cuXo0ePHujQoQPKy8uxc+dO/Pjjj1i9erWYT6MWRX4Zxq07jtSbJXC3M8NPLKDTA7rz2FwmY0tVItLxwqJEpLs54Z1QXqnGmgPJWLAtHiYyKZ7qWXXxp7y8PM24/0VcQF5JBXycrTDlIU9xgiUiIr2mUgsoKK1AbokSeSVK3CquuKs4nleiRG7xv/fllVRAqVI/0H4zCuov4rV2EgCWknLN/28v1mcVct7owTT0d4i/a1SfsWPHIjs7GwsXLoRCoUBQUBAiIyM1FxtNS0uDVPrvCevFxcV48cUXce3aNZiZmcHHxwcbN27E2LFjxXoKtWQWVBXQU3KK4dbGDD9N7412LKBTI7j92JyICGARnajZSSQSvDmiC5SVaqw/koI3fz8HU2MpRnZ1QnpuCYqVlUg5moKfT10DACwb4w9jmVadl4iI6A4qtYDolFxkFZbB0UqOXl52kEklYodVS1mF6rbid+2i961iZXWhvHaRPL+0os4V3Q1hYiRFG3NjtDE3gW31v+WVauxtQD/lRaP94N/OBkBVobiGpNaUSu66T6IZJ7nrvtvHSm67V1LHj+l+4+71/caK9/ZxZ9Ly8MrWM3cHegdHK17cjh5MQ3+H+LtG9zJ79mzMnj27zu/t37+/1tfLli3DsmXLmiGq+7szl3vam2PCNyeQklOMdrZVBXS3NuZih0lERC0Ui+hEIpBIJHj3EV8oVSpsPJ6GV7eewWK5DB4VVYXzc2ckAKTo08EePTztxA2WiMjARcZnYMn2hFotEFxs5Fg02g8j/F0afX+CIKCwvBJ5xbUL4lWrwatbp9RREC9RqnTep5WpEWwtagriJprieBtzE7SxMK51X03B3NxEVqs4DFQVKPp+sBeK/LI6+3tLADjbyDE5zFPvPoQQk1sbc/wv8iIU9bTZqJm3Xl7M6fRgennZwcVGft+/Uf6uUUtTVy6XSSVQqQW0szXDlhm94W7HAjoRETUdFtGJRCKRSPDeo/5IzirGsSs3kV9acddf5LHkm4iMz2iSIg8RUWsQGZ+BmRtj7yo2KfLLMHNjLFZPDL7na2ylSo280pqWKDVF79sK4be1T6lpqZJXUoFKHS/qJ5NKYGtmDFtzY9hZmNxR/K76f82/Nd+3NTdutDOWZFIJFo32w8yNsXe1IqkpmS8a7ccC+h3unLfbcd6oMfFvlFqj+nJ5zQV0Zw7swAI6ERE1ORbRiUQkAEjJKb7nmCXbEzDUz5kHQ0REWlKpBSzZnlDnas2a+9787RwuKgqRX1rVMqWmh3hNkbywrFLn/cuNpbCrKX7ftRq8+v8W1avFq4vjVqZGkIr8ej/C3wWrJwbfteLPuQlX77cENfP23l/xQMm/93PeqLHxb5Rak3vl8hor9yVhXK/2PF4iIqImxSI6kYiiU3KhuMfF2QQAGflliE7JRVgH++YLjIioBYhOya1VYKpLfmklPtt9+b7bsjEzrrUKvKYQbqcpjt+2Sry6rYrcWNZYT6XZjfB3wVA/Z73vI69vRvi7YHAXB2zdXoFiZSXeCA5F744OnDdqdPwbpdaiIbmcx0tERNQcWEQnElFWYe03hOq7TgKvexwREd1fQ187wzrYIci9zW1F8n8L4nYWJrAxM26VhSmZVMKChA5kUgk82loCAHp3sG+VvzvUPPg3Sq1BQ3M5j5eosUmljdMqj4haDhbRiUTkaCXX/F8NKc5Vut53HBERNUxDXztfHtyZhShqNDKZDP379xc7DCKiFqGhuZzHS9SYmMuJqC78aI1IRL287OBiI69n/XnVBaJcbKpOzyUiIu3wNZaIiMiwMZcTEZG+YBGdSEQyqQSLRvsBwF1vDGu+XjTaj6eCExHpgK+xREREho25nIiI9AWL6EQiG+HvgtUTg+FibQpv2U14y25CAgHONnKsnhiMEf4uYodIRGSwal5jnW1qn+bN11hqKmq1GufOncO5c+egVqvFDoeIyOAxl1NzYy4norqwJzqRHhjh74LBXRywdfsuFCsrERDcC707OnBFBRFRIxjh74Khfs6ITslFVmEZHK2qTvvmayw1BUEQkJubq/k/ERE9OOZyak7M5URUFxbRifSETCqBu505AKB3B3u+ISQiakQyqYQXDyUiIjJgzOVERCQmtnMhIiIiIiIiIiIiIqoHi+hERERERERERERERPVgEZ2IiIiIiIiIiIiIqB4sohMRERERERERERER1aPFXli05grKBQUFIkdC1DAqlQrFxcUAqn5vZTKZyBERUVOqyU81+YruxlxOhoj5nKh1YT6/P+ZzMjTM5UStS0NzeYstohcWFgIA3N3dRY6EiIiofoWFhbCxsRE7DL3EXE5ERIaC+bx+zOdERGQI7pfLJUIL/chcrVbjxo0bsLKygkQieaBtFRQUwN3dHenp6bC2tm6kCFs+zpv2OGe64bxpj3Omm8acN0EQUFhYCFdXV0il7K5Wl8bM5QB/73XBOdMN5017nDPdcN6019hzxnx+fzw2Fx/nTXucM91w3rTHOdONGMfmLXYlulQqhZubW6Nu09ramr/QOuC8aY9zphvOm/Y4Z7pprHnjirV7a4pcDvD3XhecM91w3rTHOdMN5017jTlnzOf3xmNz/cF50x7nTDecN+1xznTTnMfm/KiciIiIiIiIiIiIiKgeLKITEREREREREREREdWDRfQGMDU1xaJFi2Bqaip2KAaF86Y9zpluOG/a45zphvNm2Pjz0x7nTDecN+1xznTDedMe58yw8eenG86b9jhnuuG8aY9zphsx5q3FXliUiIiIiIiIiIiIiOhBcSU6EREREREREREREVE9WEQnIiIiIiIiIiIiIqoHi+hERERERERERERERPVgEZ2IiIiIiIiIiIiIqB4somshPT0dAwcOhJ+fH7p164ZffvlF7JAMQl5eHnr06IGgoCD4+/tj3bp1YodkMEpKSuDh4YHXX39d7FAMhqenJ7p164agoCAMGjRI7HAMQkpKCgYNGgQ/Pz8EBASguLhY7JD0XmJiIoKCgjQ3MzMzbNu2TeywqIGYz7XHXK475nLtMZdrj7lce8zlho25XHvM5Q+G+Vw7zOW6YT7XTnPncokgCEKTbb2FycjIQGZmJoKCgqBQKBASEoJLly7BwsJC7ND0mkqlQnl5OczNzVFcXAx/f3+cOnUK9vb2Yoem99555x0kJSXB3d0dH3/8sdjhGARPT0/Ex8fD0tJS7FAMxoABA7Bs2TL069cPubm5sLa2hpGRkdhhGYyioiJ4enri6tWrzAcGgvlce8zlumMu1x5zufaYyx8Mc7nhYS7XHnP5g2E+1w5zuW6Yz3XXHLmcK9G14OLigqCgIACAs7Mz2rZti9zcXHGDMgAymQzm5uYAgPLycgiCAH52c3+XL1/GxYsXMXLkSLFDoRbs/PnzMDY2Rr9+/QAAdnZ2TNJa+uuvvzBkyBAetBkQ5nPtMZfrhrmcmgNz+YNjLjc8zOXaYy7XHfM5NQfm8wfTHLmcRfQ7LF++HD179oSVlRUcHR0xZswYJCYm3jUuJiYGKpUK7u7uIkSpf+43b3l5eQgMDISbmxveeOMNtG3bVsRo9cP95uz111/H8uXLRYxQP91v3iQSCQYMGICePXti06ZNIkaqP+41Z5cvX4alpSVGjx6N4OBgvP/++yJHqz8amg9+/vlnjB07VoQI6V6Yz7XHXK495nLdMJdrj7lcN8zlho25XHvM5bphPtcec7lumM+1p0+5nEX0Oxw4cACzZs3C8ePHERUVhYqKCgwbNqxWH6Lc3FxMnjwZa9euFTFS/XK/ebO1tcXZs2eRkpKCzZs3IzMzU+SIxXevOfvzzz/RuXNndO7cWeww9c79ftcOHz6MmJgY/PXXX3j//fdx7tw5kSMW373mrLKyEocOHcKqVatw7NgxREVFISoqSuyQ9UJD8kFBQQGOHj2KUaNGiRgp1YX5XHvM5dpjLtcNc7n2mMt1w1xu2JjLtcdcrhvmc+0xl+uG+Vx7epXLBbqnrKwsAYBw4MABQRAEoaysTOjXr5/www8/iByZfrtz3m43c+ZM4ZdffhEhKv12+5y99dZbgpubm+Dh4SHY29sL1tbWwpIlS8QOUS/d63ft9ddfF7777rvmD0rP3T5nR48eFYYNG6b53ocffih8+OGHIkanv+r6Xfvhhx+ECRMmiBgVNRTzufaYy7XHXK4b5nLtMZfrhrncsDGXa4+5XDfM59pjLtcN87n2xMzlXIl+H/n5+QCqehEJgoApU6Zg8ODBmDRpksiR6bfb5y0zMxOFhYWa+w8ePIguXbqIGZ5eun3Oli9fjvT0dKSmpuLjjz/G9OnTsXDhQpEj1E+3z1txcbHmd62oqAh79+5F165dxQxPL90+Zz179kRWVhZu3boFtVqNgwcPwtfXV+QI9dPt81aDp38bDuZz7TGXa4+5XDfM5dpjLtcNc7lhYy7XHnO5bpjPtcdcrhvmc+2JmcslgsArSdRHrVbj0UcfRV5eHg4fPozDhw+jf//+6Natm2bMjz/+iICAABGj1D93zlt0dDRmzJihuXDJrFmz8Pzzz4sdpl65c85ut2HDBsTHx/MK4HW4c96uXLmCxx9/HEDV1eenT5+OV155ReQo9Utdv2sRERGYN28eBEHAsGHDsGLFCpGj1D91zVt+fj46d+6M9PR0mJiYiBwh3QvzufaYy7XHXK4b5nLtMZfrhrncsDGXa4+5XDfM59pjLtcN87n2xM7lLKLfw8yZMxEREYHDhw/Dzc1N7HAMBudNe5wz3XDetMc50w3nzbDx56c9zpn2OGe64bxpj3OmG86bYePPT3ucM91w3rTHOdMN5017Ys+ZUbPv0UDMnj0bO3bswMGDB/nLrAXOm/Y4Z7rhvGmPc6Ybzpth489Pe5wz7XHOdMN50x7nTDecN8PGn5/2OGe64bxpj3OmG86b9vRizpq867qBUavVwqxZswRXV1fh0qVLYodjMDhv2uOc6Ybzpj3OmW44b4aNPz/tcc60xznTDedNe5wz3XDeDBt/ftrjnOmG86Y9zpluOG/a06c540r0O8yaNQubN2/Gn3/+CSsrKygUCgCAjY0NzMzMRI5Of3HetMc50w3nTXucM91w3gwbf37a45xpj3OmG86b9jhnuuG8GTb+/LTHOdMN5017nDPdcN60p09zxp7od5BIJHXe/91332HKlCnNG4wB4bxpj3OmG86b9jhnuuG8GTb+/LTHOdMe50w3nDftcc50w3kzbPz5aY9zphvOm/Y4Z7rhvGlPn+aMRXQiIiIiIiIiIiIionpIxQ6AiIiIiIiIiIiIiEhfsYhORERERERERERERFQPFtGJiIiIiIiIiIiIiOrBIjoRERERERERERERUT1YRCciIiIiIiIiIiIiqgeL6ERERERERERERERE9WARnYiIiIiIiIiIiIioHiyiExERERERERERERHVg0V0IiIiIiIiIiIiIqJ6sIhORM1mypQpGDNmjCj7lkgk2LZtW6Ntb+DAgZgzZ06jbY+IiMhQMJ8TEREZNuZyIu2xiE5ED2zx4sUICgpqln2lpqZCIpHgzJkzzbK/+vz+++9YunSpqDEQERE1JuZzIiIiw8ZcTtR0jMQOgIjuTRAEqFQqGBnxz1Wf2NnZiR0CEREZEOZz/cR8TkREDcVcrp+Yy6m5cCU6tVgDBw7ESy+9hDlz5qBNmzZwcnLCunXrUFxcjKlTp8LKygodO3ZERERErcfFx8dj5MiRsLS0hJOTEyZNmoScnBzN9yMjI9G3b1/Y2trC3t4ejzzyCJKTkzXfVyqVmD17NlxcXCCXy+Hh4YHly5cDqPuT2ry8PEgkEuzfvx8AsH//fkgkEkRERCAkJASmpqY4fPgw1Go1li9fDi8vL5iZmSEwMBC//vqrZjs1j/vnn3/QvXt3mJmZYfDgwcjKykJERAR8fX1hbW2N8ePHo6SkRPO4hm53z5496NGjB8zNzdGnTx8kJiYCADZs2IAlS5bg7NmzkEgkkEgk2LBhwz1/NkuWLIGDgwOsra3xwgsvQKlUNnh+vby8AADdu3eHRCLBwIEDNd9bv349unbtClNTU7i4uGD27Nm19puTk4PHH38c5ubm6NSpE/766697xrlq1Sp06tQJcrkcTk5OePLJJzXfu/2UsZo5uvM2ZcoUzfg///wTwcHBkMvl8Pb2xpIlS1BZWXnP/RMREfM58znzORGRoWMuZy5nLqcWQSBqoQYMGCBYWVkJS5cuFS5duiQsXbpUkMlkwsiRI4W1a9cKly5dEmbOnCnY29sLxcXFgiAIwq1btwQHBwdh/vz5woULF4TY2Fhh6NChwqBBgzTb/fXXX4XffvtNuHz5snD69Glh9OjRQkBAgKBSqQRBEISPPvpIcHd3Fw4ePCikpqYKhw4dEjZv3iwIgiCkpKQIAITTp09rtnfr1i0BgLBv3z5BEARh3759AgChW7duwq5du4SkpCTh5s2bwrJlywQfHx8hMjJSSE5OFr777jvB1NRU2L9/f63H9e7dWzh8+LAQGxsrdOzYURgwYIAwbNgwITY2Vjh48KBgb28v/O9//9Psv6HbDQ0NFfbv3y+cP39e6Nevn9CnTx9BEAShpKREeO2114SuXbsKGRkZQkZGhlBSUlLnz+SZZ54RLC0thbFjxwrx8fHCjh07BAcHB+Htt99u8PxGR0cLAITdu3cLGRkZws2bNwVBEIRVq1YJcrlc+Oyzz4TExEQhOjpa+PTTTzXbBSC4ubkJmzdvFi5fviy8/PLLgqWlpebxdzp58qQgk8mEzZs3C6mpqUJsbKzw+eef1/r9euWVVwRBEITy8nLNc8/IyBD27t0ryOVy4dtvvxUEQRAOHjwoWFtbCxs2bBCSk5OFXbt2CZ6ensLixYvr3DcREf2L+Zz5nPmciMiwMZczlzOXU0vAIjq1WAMGDBD69u2r+bqyslKwsLAQJk2apLkvIyNDACAcO3ZMEARBWLp0qTBs2LBa20lPTxcACImJiXXuJzs7WwAgxMXFCYIgCC+99JIwePBgQa1W3zVWm0S9bds2zZiysjLB3NxcOHr0aK3tTZs2TRg3blytx+3evVvz/eXLlwsAhOTkZM19zz//vDB8+PAH2u7ff/8tABBKS0sFQRCERYsWCYGBgXXOz+2eeeYZwc7OTvPGSBAEYfXq1YKlpaUmEd/pzvmtaw4FQRBcXV2Fd955p959AxAWLFig+bqoqEgAIERERNQ5/rfffhOsra2FgoKCOr9/e6K+XU5OjuDt7S28+OKLmvuGDBkivP/++7XG/fjjj4KLi0u98RIRURXmc+bz2zGfExEZHuZy5vLbMZeToWI7F2rRunXrpvm/TCaDvb09AgICNPc5OTkBALKysgAAZ8+exb59+2Bpaam5+fj4AIDmtKXLly9j3Lhx8Pb2hrW1NTw9PQEAaWlpAKqucn3mzBl06dIFL7/8Mnbt2qVT7D169ND8PykpCSUlJRg6dGit2H744Ydap1Pd+ZydnJxgbm4Ob2/vWvfVPF9dt+vi4lJr3rQRGBgIc3NzzddhYWEoKipCeno6gPvPb12ysrJw48YNDBky5J77vv05WFhYwNraut7nMHToUHh4eMDb2xuTJk3Cpk2bap1qV5eKigo88cQT8PDwwOeff665/+zZs3jvvfdqzfH06dORkZFx320SERHzOfN5bcznRESGh7mcuby+58BcToaCV0OgFs3Y2LjW1xKJpNZ9EokEQFXvMQAoKirC6NGj8cEHH9y1rZrkNHr0aHh4eGDdunVwdXWFWq2Gv7+/pndYcHAwUlJSEBERgd27d+Opp55CeHg4fv31V0ilVZ9bCYKg2W5FRUWdsVtYWGj+X1RUBAD4+++/0a5du1rjTE1N633Odz7fmvtuf766bhf4d94a0/3mty5mZmYN2va95uJOVlZWiI2Nxf79+7Fr1y4sXLgQixcvxsmTJ2Fra1vnY2bOnIn09HRER0fXuthMUVERlixZgv/7v/+76zFyubxBsRMRtWbM58znt2M+JyIyPMzlzOW3Yy4nQ8QiOtFtgoOD8dtvv8HT07POK27fvHkTiYmJWLduHfr16wcAOHz48F3jrK2tMXbsWIwdOxZPPvkkRowYgdzcXDg4OAAAMjIy0L17dwCodSGT+vj5+cHU1BRpaWkYMGDAAzzDptmuiYkJVCpVg8aePXsWpaWlmuR6/PhxWFpawt3dvUHza2JiAgC19mdlZQVPT0/s2bMHgwYN0vl53MnIyAjh4eEIDw/HokWLYGtri71799aZcFesWIGff/4ZR48ehb29fa3vBQcHIzExER07dmy02IiIqH7M57phPmc+JyLSF8zlumEuZy6npsMiOtFtZs2ahXXr1mHcuHGYN28e7OzskJSUhC1btuCbb75BmzZtYG9vj7Vr18LFxQVpaWl46623am1jxYoVcHFxQffu3SGVSvHLL7/A2dkZtra2kEql6N27N/73v//By8sLWVlZWLBgwX3jsrKywuuvv45XX30VarUaffv2RX5+Po4cOQJra2s888wzOj3fxtqup6cnUlJScObMGbi5ucHKyuquT8trKJVKTJs2DQsWLEBqaioWLVqE2bNnQyqVNmh+HR0dYWZmhsjISLi5uUEul8PGxgaLFy/GCy+8AEdHR4wcORKFhYU4cuQIXnrpJZ3mZseOHbhy5Qr69++PNm3aYOfOnVCr1ejSpctdY3fv3o158+Zh5cqVaNu2LRQKBYCqT+FtbGywcOFCPPLII2jfvj2efPJJSKVSnD17FvHx8Vi2bJlO8RERUf2Yz5nPazCfExEZJuZy5vIazOWkL9gTneg2rq6uOHLkCFQqFYYNG4aAgADMmTNHk2SlUim2bNmCmJgY+Pv749VXX8VHH31UaxtWVlb48MMP0aNHD/Ts2ROpqanYuXOn5nSx9evXo7KyEiEhIZgzZ06DX6iXLl2Kd999F8uXL4evry9GjBiBv//+G15eXg/0nBtju0888QRGjBiBQYMGwcHBAT/99FO9Y4cMGYJOnTqhf//+GDt2LB599FEsXrwYABo0v0ZGRvjiiy/w9ddfw9XVFY899hgA4JlnnsFnn32GVatWoWvXrnjkkUdw+fJl7Sekmq2tLX7//XcMHjwYvr6+WLNmDX766Sd07dr1rrGHDx+GSqXCC5/wyQQAAAEJSURBVC+8ABcXF83tlVdeAQAMHz4cO3bswK5du9CzZ0/07t0bn376KTw8PHSOj4iI6sd8znxeg/mciMgwMZczl9dgLid9IRFubwBFREREREREREREREQaXIlORERERERERERERFQPFtGJiIiIiIiIiIiIiOrBIjoRERERERERERERUT1YRCciIiIiIiIiIiIiqgeL6ERERERERERERERE9WARnYiIiIiIiIiIiIioHiyiExERERERERERERHVg0V0IiIiIiIiIiIiIqJ6sIhORERERERERERERFQPFtGJiIiIiIiIiIiIiOrBIjoRERERERERERERUT1YRCciIiIiIiIiIiIiqsf/A+MqdAoMYqW2AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n", + "\n", + "sweep_cols = [f\"cross_chi_net@bs{S}\" for S in MEASURE_BATCH_SIZES]\n", + "row = metrics.dropna(subset=sweep_cols).iloc[-1]\n", + "\n", + "fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n", + "for ax, metric in zip(axes, [\"chi_net\", \"chi_coup\", \"grad_dot_product\"]):\n", + " ys = [row[f\"cross_{metric}@bs{S}\"] for S in MEASURE_BATCH_SIZES]\n", + " ax.plot(MEASURE_BATCH_SIZES, ys, \"o-\")\n", + " ax.axvline(MEASURE_MAX_SINGLE_PASS, color=\"gray\", linestyle=\"--\", alpha=0.5,\n", + " label=\"max single-pass size\")\n", + " ax.set_xscale(\"log\", base=2)\n", + " ax.set_xlabel(\"measurement batch size\")\n", + " ax.set_title(f\"cross_{metric}\")\n", + "axes[0].legend()\n", + "\n", + "fig.suptitle(f\"Measurement batch-size sweep @ step {int(row['step'])}\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2086f0994ba148e58e57906c0b51eb68": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_592116f5074648528db363f98703056a", + "max": 1.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_5f69781c8cf44938a8df9761777d102b", + "tabbable": null, + "tooltip": null, + "value": 1.0 + } + }, + "592116f5074648528db363f98703056a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": "2", + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f69781c8cf44938a8df9761777d102b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "602335a52b5748d7acbfc07a7f611aaa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": "inline-flex", + "flex": null, + "flex_flow": "row wrap", + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "100%" + } + }, + "662062d25a374c95a430a67df0a8236c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "7b7e75172bc843ad9957a168fc37d782": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "83ba3e3aa3b84cc9811c806f9f4398ef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8e32e47451ec477bb6a2356389ac6da9", + "IPY_MODEL_2086f0994ba148e58e57906c0b51eb68", + "IPY_MODEL_8b8cac2e24cb4c9491944f2d01137041" + ], + "layout": "IPY_MODEL_602335a52b5748d7acbfc07a7f611aaa", + "tabbable": null, + "tooltip": null + } + }, + "8b8cac2e24cb4c9491944f2d01137041": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_aa81e02672cb4efcbd1bdca8d44a5e79", + "placeholder": "​", + "style": "IPY_MODEL_7b7e75172bc843ad9957a168fc37d782", + "tabbable": null, + "tooltip": null, + "value": " 200/200 [00:02<00:00, 83.48it/s, v_num=0, train_loss=0.320, train_acc=0.891]" + } + }, + "8e32e47451ec477bb6a2356389ac6da9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f4c92b02245343469d0a5111affbdb1e", + "placeholder": "​", + "style": "IPY_MODEL_662062d25a374c95a430a67df0a8236c", + "tabbable": null, + "tooltip": null, + "value": "Epoch 0: 100%" + } + }, + "aa81e02672cb4efcbd1bdca8d44a5e79": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f4c92b02245343469d0a5111affbdb1e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/measurement_batch_sweep.md b/examples/measurement_batch_sweep.md new file mode 100644 index 0000000..823c026 --- /dev/null +++ b/examples/measurement_batch_sweep.md @@ -0,0 +1,314 @@ +# Independent Measurement Batch Size — Implementation Notes + +## The problem this solves + +`analyzer(..., cross_response=True)` measures the model's linear response against a +held-out "measure" batch by expecting a dict batch `{"train": ..., "measure": ...}`, +produced with a `CombinedLoader({"train": train_loader, "measure": measure_loader}, +mode="max_size_cycle")`. See `cifar10_CrossReseponse.ipynb`. + +When gradient accumulation is active on the train side +(`micro_batch_size`/`effective_batch_size`, see `batch_accumulation_notes.md`), the +legacy cross-response code accumulates the measure gradient over the *same* K +micro-batches as the train side, and divides both by the same `K` +(`perspic/analyzer.py`, `_finalize_accumulated_analysis`). There was no way to: + +1. Give the measure batch a size independent of the train effective batch size, or +2. Accumulate the measure side on its own when that size doesn't fit in one forward + pass. + +This was an open `TODO` in the code: + +```python +# Accumulate linearizer gradients (measure side) +# TODO: How would a measure batchsize different to the effective batch size work here? +# !!! We would need to accumulate separately and then combine at the end. +``` + +This document describes the fix: `measure_dataloader` / `measure_batch_size` / +`measure_subset_seed`, a new, independent path that decouples the measurement batch +size from the training batch/accumulation entirely, and — as a second phase — lets one +analysis step sweep an **array** of measurement batch sizes. + +The legacy `cross_response=True` + `CombinedLoader` path is untouched and still works +exactly as before. The new path is a separate, opt-in mechanism. + +--- + +## Quick usage + +### A single, independently sized measurement batch + +```python +from torch.utils.data import DataLoader +from perspic import analyzer + +# Sized however you like — independent of the train DataLoader's batch_size. +measure_loader = DataLoader(measurement_set, batch_size=500, drop_last=True) + +model = analyzer( + ClassificationModule, + model=backbone, + lr=0.1, + measure_dataloader=measure_loader, + measure_batch_size=2000, # 4x measure-side accumulation (2000 // 500) +) +trainer.fit(model, train_dataloaders=train_loader) # plain (x, y) loader, no CombinedLoader +``` + +`measure_batch_size` defaults to `measure_dataloader.batch_size` (no measure-side +accumulation) if omitted. + +### Sweeping multiple measurement batch sizes + +```python +from perspic import analyzer, logarithmic_windows + +schedule = logarithmic_windows(max_steps=10_000, points_per_decade=5) + +model = analyzer( + ClassificationModule, + model=backbone, + lr=0.1, + measure_dataloader=measure_loader, + measure_batch_size=[500, 1000, 2000, 4000], + measure_subset_seed=0, # reproducible subset draws across runs + analysis_schedule=schedule, # see the warning below +) +``` + +Each analyzed step now logs one `cross_*` metric set **per swept size**, suffixed +`@bs{S}`: `cross_chi_net@bs500`, `cross_chi_net@bs1000`, ..., +`cross_grad_dot_product@bs4000`, etc. + +**Warning:** sweeping without a logarithmic `analysis_schedule` runs the *entire* +sweep at *every* analyzed step (every step, if `analyze_every`/`analysis_schedule` are +both unset), which is expensive — each swept size does its own forward+backward passes +per micro-batch chunk. `analyzer()` emits a `UserWarning` at construction time if you +set a `measure_batch_size` list without also setting `analysis_schedule`. + +Sizes below `measure_dataloader.batch_size` are valid too — they're measured as a +single direct pass instead of being rejected: + +```python +measure_loader = DataLoader(measurement_set, batch_size=128, drop_last=True) + +model = analyzer( + ClassificationModule, + model=backbone, + lr=0.1, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8, 16, 32, 64, 128, 256, 512, 1024], + measure_subset_seed=0, + analysis_schedule=schedule, +) +# 4..128 -> single direct pass each (K_measure=1) +# 256, 512, 1024 -> accumulated (K_measure=2, 4, 8) +``` + +--- + +## How it works + +### 1. An independent, persistent measure data source + +`measure_dataloader` is any `DataLoader`. Unlike the train batch, the analyzer does +**not** expect it bundled into the training batch via `CombinedLoader`. Instead the +`Analyzer` owns its own iterator over `measure_dataloader` +(`_next_measure_micro_batch`), lazily created on first use and transparently refilled +(`iter(...)` again) whenever it's exhausted — so a finite measure dataset just cycles. +A `MultiEpochsDataLoader` (see `perspic/utils.py`) works too and is recommended for +small measurement sets, since it avoids re-spawning DataLoader workers every cycle. + +Because the measure source is independent, the training batch is a **plain `(x, y)` +tuple** when `measure_dataloader` is set — no `CombinedLoader` dict, no `"measure"` +key. (The two mechanisms are mutually exclusive per run: pick `cross_response=True` + +`CombinedLoader`, *or* `measure_dataloader`.) + +The measure **micro-batch size** — the largest chunk pulled from the loader in one +forward/backward pass — is inferred from `measure_dataloader.batch_size`. Use +`drop_last=True` (or a dataset size divisible by `batch_size`) so every pulled +micro-batch is full; a short final batch raises a `ValueError` rather than silently +building an undersized pool. + +### 2. Sizing and validation + +`measure_dataloader.batch_size` is the **maximum single-pass batch size** — the +largest batch your GPU can process in one forward+backward pass. `measure_batch_size` +accepts an `int` (single size) or a `list[int]` (sweep), and each entry `S` is handled +by one of two regimes depending on how it compares to that max single-pass size +(`micro = measure_dataloader.batch_size`): + +- **`S <= micro`** — measured with a **single direct pass** on exactly `S` samples, + no accumulation (this deliberately runs the GPU below its max capacity; there's + nothing to validate here beyond `S` being a positive integer). +- **`S > micro`** — `S` must be an **exact multiple** of `micro`; measured by + accumulating `S // micro` passes of size `micro` each into one combined measurement + (the same rule as `effective_batch_size` vs `micro_batch_size` on the train side). + +``` +chunk_size = min(S, micro) +K_measure = S // chunk_size +``` + +This single formula covers both regimes: `chunk_size = S` (so `K_measure = 1`, a lone +direct pass) when `S <= micro`, and `chunk_size = micro` (so `K_measure = S // micro`, +accumulated passes) when `S > micro`. `K_measure` is computed independently per swept +size and is completely decoupled from the train side's `accumulation_steps`. A run can +combine train-side accumulation (`micro_batch_size`/`effective_batch_size`) with +measure-side accumulation (`measure_dataloader`/`measure_batch_size`) freely — the two +`K`s do not need to match, and in general won't. A sweep like +`measure_batch_size=[4, 8, 16, 32, 64, 128, 256, 512, 1024]` with a 128-sample max +single pass measures `4, ..., 128` as direct passes (`K_measure=1` each) and +`256, 512, 1024` as accumulated measurements (`K_measure=2, 4, 8` respectively) — all +in the same analyzed step, each producing exactly one measurement. + +### 3. Gather once, then subset — largest to smallest + +Naively, sweeping N sizes could mean N independent pulls from the measure loader per +analyzed step (S₁ samples for size 1, S₂ for size 2, ...) — wasteful, and it makes +smaller sizes' samples *unrelated* to larger sizes' samples, which is usually not what +you want when studying how a metric depends on batch size. + +Instead, on every analyzed step, `_measure_response` gathers **one pool** of +`S_max = max(measure_batch_size)` samples from the persistent iterator — pulling +`ceil(S_max / micro)` micro-batches (at least one, even when `S_max < micro`, i.e. the +whole sweep is below the max single-pass size) and slicing the concatenation down to +exactly `S_max` samples — and processes every requested size **largest → smallest**: + +- The largest size uses the whole pool. +- Every smaller size `S` uses a **seed-fixable random subset** of `S` samples drawn + from that *same* pool, via a single `torch.Generator` created once at `analyzer()` + construction time and seeded (optionally) by `measure_subset_seed`. The generator is + *not* reseeded between steps or between sizes, so a full run's sequence of subset + draws is reproducible end-to-end given the same seed — rerun the same training script + with the same `measure_subset_seed` and every swept metric matches. + +Processing largest-first (rather than, say, smallest-first or in list order) is what +makes the smaller sizes' samples an actual *subset* of the larger sizes' samples, +rather than an independently-drawn batch — useful when you want to see how a metric +changes as you add more samples to the same pool, not how it varies across unrelated +draws. + +``` +pool = pull(ceil(S_max / micro)) micro-batches, concatenated, sliced to S_max samples +for S in sorted(measure_batch_size, reverse=True): + if S == S_max: + subset = pool # use it all + else: + subset = pool[ randperm(S_max, generator=measure_gen)[:S] ] + chunk_size = min(S, micro) + K_measure = S // chunk_size # 1 if S <= micro + for chunk in chunks(subset, chunk_size): # K_measure chunks + accumulate gradient + per-sample chi over `chunk` + combine and log cross_* metrics (suffixed @bs{S} if sweeping) +``` + +Each chunk is wrapped in its own `BatchStatSnapshot(self.model, chunk)` +(`perspic/utils.py`), so BatchNorm statistics are frozen to *that chunk's own* +statistics before computing its per-sample gradients. This is a deliberate difference +from the legacy `cross_response=True` path, which freezes the measure computation to +the *train* batch's statistics — the independent path has no train batch to borrow +statistics from once the measure size diverges from the train size, so each measure +chunk uses its own. + +### 4. Combining the metrics + +The aggregation semantics mirror the existing **single-step** (no accumulation) +reference case — not the legacy accumulated-cross-response path's per-micro-batch +geometric-mean-then-average, which conflates the train and measure micro-batch +granularities. For each measure size `S`: + +``` +chi_net_measure(S) = mean over the S's K_measure chunks of batch_grad_norms_network +chi_loss_measure(S) = mean over the S's K_measure chunks of batch_grad_norms_loss + +cross_chi_net(S) = sqrt( chi_net_self(train) * chi_net_measure(S) ) # compute_cross_metrics +cross_chi_loss(S) = sqrt( chi_loss_self(train) * chi_loss_measure(S) ) + +grad_train_mean = (train accumulated gradient) / K_train # K_train = 1 without accumulation +grad_measure_mean(S) = (measure accumulated gradient for S) / K_measure + +cross_grad_dot_product(S) = +cross_loss(S) = mean loss over the S measure samples +cross_chi_coup(S) = cross_grad_dot_product(S) / (cross_chi_loss(S) * cross_chi_net(S)) +``` + +Same intensive/extensive reasoning as train-side accumulation applies here (see +"Issue 1" in `batch_accumulation_notes.md`): both `chi_net` and `chi_loss` are +computed with `normalize=True`, which makes the *correct* aggregate over accumulated +chunks the **mean**, not `K_measure * sum(...)`. + +In the train-side-accumulated case, `_measure_response` is called from +`_finalize_accumulated_analysis` with `grad_train_mean = accumulated_train_grad / +accumulation_steps` (the same accumulated train gradient already used for the train +`chi_coup`). In the non-accumulated (single-step) case, the train gradient normally +gets discarded inside `Linearizer.compute()`, so `_analyze_single_step` recomputes it +once explicitly (a single extra forward/backward, only on analyzed steps) before +calling `_measure_response`. + +`_measure_response` always saves and restores `self.model`'s gradients around its own +forward/backward passes (mirroring the existing analysis save/restore pattern in +`_analyze_accumulated_step`), so it never corrupts the live (possibly partially +accumulated) training gradient — this is covered by +`test_measure_backward_does_not_corrupt_training_grad` in `tests/unit/test_analyzer.py`. + +### 5. Logging keys + +| case | keys logged | +|---|---| +| single `measure_batch_size` (int, or a 1-element list) | `cross_chi_net`, `cross_chi_loss`, `cross_chi_coup`, `cross_loss`, `cross_grad_dot_product`, `cross_batch_size` — identical to the legacy `cross_response=True` keys | +| sweep (`measure_batch_size` list, length > 1) | the same set, suffixed `@bs{S}` per size, e.g. `cross_chi_net@bs500`, `cross_chi_net@bs2000`, ... | + +`cross_effective_batch_size` is intentionally **not** logged for the independent +measure path: `batch_size` in the logged metric is already the full measure size `S` — +multiplying by the *train* `accumulation_steps` (what the generic logging helper does +for the train side) would be meaningless here, since `K_measure` is independent of +`K_train`. + +--- + +## Validation rules (raised at `analyzer()` construction time) + +- `measure_dataloader.batch_size` must not be `None` (i.e. it must use automatic + batching). +- Every value in `measure_batch_size` must be a positive integer. Values `<= + measure_dataloader.batch_size` need no further constraint (single direct pass). + Values `>` it must be an exact multiple of it (measured via accumulation). +- `measure_batch_size` / `measure_subset_seed` may only be set together with + `measure_dataloader`. +- A `measure_batch_size` list of length > 1 without `analysis_schedule` triggers a + `UserWarning` (see the sweep-cost warning above). + +## Edge cases + +- **Partial final measure batch.** If `measure_dataloader` has `drop_last=False` and + its dataset size isn't divisible by `batch_size`, the last batch of an epoch is + short. Pulling it into the pool would silently build a short/misaligned pool, so + `_next_measure_micro_batch` raises a `ValueError` instead, recommending + `drop_last=True`. +- **Measure dataset smaller than `S_max`.** The persistent iterator just cycles, so the + pool may contain repeated samples. This is fine for most uses but worth knowing if + you're sizing `measure_batch_size` close to (or larger than) the measurement + dataset's size. +- **Every swept size below the max single-pass size (`S_max < micro`).** The pool + gather still pulls at least one micro-batch (`ceil(S_max / micro) = 1`) and slices + it down to `S_max` samples — it never pulls zero micro-batches. +- **`disable_analyzer=True`.** `_measure_response` is never called (the whole analysis + hook is skipped), so the measure loader is never touched — zero overhead. +- **`log_metrics=False`.** `_measure_response` is gated on `log_metrics` (matching the + "no consumer, skip the expensive sweep" intent), so no measure-side computation runs + at all in that case. + +## Where to look in the code + +- `perspic/analyzer.py`: `__init__` validation block ("Independent measure data + source"), `_next_measure_micro_batch`, `_measure_response`, and the two call + sites in `_analyze_single_step` and `_finalize_accumulated_analysis`. +- `tests/unit/test_analyzer.py`: `TestIndependentMeasureResponse` — validation, + single-measure logging, the `K_measure`-vs-`K_train` mean divisor, a numeric + gradient-dot-product reference test, subset-seed determinism, sweep suffixing, and + iterator-cycling/partial-batch edge cases. +- `tests/integration/test_analyzer_deployment.py`: `TestAnalyzerWithIndependentMeasure` + — end-to-end training with a real `Trainer`, combined with train-side accumulation, + a sweep under a logarithmic schedule, and cross-run reproducibility. diff --git a/first_wrong_test_batch_accumulation.png b/first_wrong_test_batch_accumulation.png new file mode 100644 index 0000000..dfdf780 Binary files /dev/null and b/first_wrong_test_batch_accumulation.png differ diff --git a/perspic/analyzer.py b/perspic/analyzer.py index 1c842e9..c2b591e 100644 --- a/perspic/analyzer.py +++ b/perspic/analyzer.py @@ -1,5 +1,6 @@ +import math import warnings -from typing import Optional +from typing import Optional, Union import pytorch_lightning as pl import torch @@ -22,6 +23,11 @@ def analyzer( analyze_every: Optional[int] = None, analysis_schedule: Optional[LogarithmicWindowSchedule] = None, cross_response: bool = False, + micro_batch_size: Optional[int] = None, + effective_batch_size: Optional[int] = None, + measure_dataloader: Optional[torch.utils.data.DataLoader] = None, + measure_batch_size: Optional[Union[int, list[int]]] = None, + measure_subset_seed: Optional[int] = None, **model_kwargs, ): """Factory function that wraps a LightningModule with analysis capabilities. @@ -52,11 +58,44 @@ def analyzer( analysis runs only at the scheduled steps. If both analyze_every and analysis_schedule are provided, analysis_schedule takes precedence. - cross_response: If True, enables cross-batch response analysis and assumes - the training batch is a dict with 'train' and 'measure' keys. - Defaults to False. - **model_kwargs: Additional keyword arguments passed to the - LightningModule constructor. + cross_response: If True, enables cross-batch response + analysis and assumes the training batch is a dict + with 'train' and 'measure' keys. Defaults to False. + micro_batch_size: The actual micro-batch size used by the + DataLoader. Required when effective_batch_size is + set. Can be provided alone (no accumulation). + effective_batch_size: The desired simulated batch size + achieved through gradient accumulation. Must be + divisible by micro_batch_size. When set, the optimizer + step is only performed every + (effective_batch_size // micro_batch_size) micro-batches. + measure_dataloader: An independent DataLoader supplying the + measurement ("cross") batch, decoupled from the training + batch/accumulation. When set, the analyzer owns a + persistent iterator over this loader and the training + batch is expected to be a plain (x, y) tuple (not a + CombinedLoader dict). The measure micro-batch size is + inferred from measure_dataloader.batch_size; use + drop_last=True (or MultiEpochsDataLoader) so every pulled + micro-batch is full. Setting this activates the + independent measure-response path and implies + cross-response-style analysis regardless of + cross_response. + measure_batch_size: The desired measurement batch size(s), + achieved through measure-side gradient accumulation when + larger than measure_dataloader.batch_size. Accepts a + single int or a list[int] to sweep multiple sizes in one + analysis step (each swept size is logged with a + "@bs{S}" suffix). Each value must be >= and divisible by + measure_dataloader.batch_size. Defaults to + measure_dataloader.batch_size (no measure accumulation). + Only valid together with measure_dataloader. + measure_subset_seed: Seed for the random generator used to + draw reproducible subsets of the measure pool when + sweeping multiple measure_batch_size values. Only valid + together with measure_dataloader. + **model_kwargs: Additional keyword arguments passed to + the LightningModule constructor. Returns: An initialized Analyzer instance that wraps the provided @@ -117,6 +156,11 @@ def __init__( analyze_every=analyze_every, analysis_schedule=analysis_schedule, cross_response=cross_response, + micro_batch_size=micro_batch_size, + effective_batch_size=effective_batch_size, + measure_dataloader=measure_dataloader, + measure_batch_size=measure_batch_size, + measure_subset_seed=measure_subset_seed, **model_kwargs, ): super().__init__(**model_kwargs) @@ -178,6 +222,131 @@ def __init__( self.delegate_optimization = False self.automatic_optimization = False # We handle optimization manually + # Gradient accumulation setup + self.micro_batch_size = micro_batch_size + self.effective_batch_size = effective_batch_size + + if effective_batch_size is not None and micro_batch_size is None: + raise ValueError( + "micro_batch_size must be specified when " + "effective_batch_size is set." + ) + + if micro_batch_size is not None and effective_batch_size is not None: + if effective_batch_size < micro_batch_size: + raise ValueError( + f"effective_batch_size " + f"({effective_batch_size}) must be " + f">= micro_batch_size ({micro_batch_size})." + ) + if effective_batch_size % micro_batch_size != 0: + raise ValueError( + f"effective_batch_size " + f"({effective_batch_size}) must be " + f"divisible by micro_batch_size " + f"({micro_batch_size})." + ) + self.accumulation_steps = effective_batch_size // micro_batch_size + else: + self.accumulation_steps = 1 + + if self.accumulation_steps > 1 and self.delegate_optimization: + raise ValueError( + "Gradient accumulation is not supported " + "when the wrapped model uses manual " + "optimization (delegate_optimization=True)." + ) + + # --- Independent measure data source (cross-response, decoupled sizing) --- + self._measure_dataloader = measure_dataloader + self._independent_measure = measure_dataloader is not None + + if not self._independent_measure and ( + measure_batch_size is not None or measure_subset_seed is not None + ): + raise ValueError( + "measure_batch_size and measure_subset_seed are only " + "valid together with measure_dataloader." + ) + + if self._independent_measure: + measure_micro = getattr(measure_dataloader, "batch_size", None) + if measure_micro is None: + raise ValueError( + "measure_dataloader must have an integer batch_size " + "(its batch_size attribute is None). Provide a " + "DataLoader whose batch_size divides every " + "measure_batch_size." + ) + self._measure_micro_batch_size = measure_micro + + if measure_batch_size is None: + sizes = [measure_micro] + elif isinstance(measure_batch_size, int): + sizes = [measure_batch_size] + else: + sizes = list(measure_batch_size) + if len(sizes) == 0: + raise ValueError("measure_batch_size list must be non-empty.") + + for s in sizes: + if not isinstance(s, int): + raise ValueError( + f"measure_batch_size entries must be integers, " + f"got {type(s)}." + ) + if s < 1: + raise ValueError( + f"measure_batch_size entries must be positive, " f"got {s}." + ) + if s > measure_micro and s % measure_micro != 0: + raise ValueError( + f"measure_batch_size ({s}) is larger than the " + f"measure_dataloader batch_size ({measure_micro}) " + f"— the maximum single-pass size — and must then " + f"be an exact multiple of it, to be measured via " + f"gradient accumulation. Sizes <= {measure_micro} " + f"need no such constraint (they run as a single " + f"direct pass)." + ) + self._measure_batch_sizes = sizes + + if len(sizes) > 1 and analysis_schedule is None: + warnings.warn( + "A measure batch-size sweep (measure_batch_size " + f"list of length {len(sizes)}) without a " + "logarithmic analysis_schedule runs the full sweep " + "at EVERY analyzed step and is very expensive. Pass " + "analysis_schedule=logarithmic_windows(...) to " + "restrict analysis to logarithmically spaced steps." + ) + + self._measure_gen = torch.Generator() + if measure_subset_seed is not None: + self._measure_gen.manual_seed(measure_subset_seed) + self._measure_iter = None + else: + self._measure_micro_batch_size = None + self._measure_batch_sizes = None + self._measure_gen = None + self._measure_iter = None + + self._accumulation_count = 0 + self._optimizer_step_count = 0 + + # Analysis accumulation buffers + self._accum_chi_net = [] + self._accum_chi_loss = [] + self._accum_cross_chi_net = [] + self._accum_cross_chi_loss = [] + self._accum_grad_train = None + self._accum_grad_measure = None + self._accum_train_loss = 0.0 + self._accum_measure_loss = 0.0 + self._accum_step_losses = [] # micro-batch losses for per-opt-step logging + # Track whether analysis is active for this cycle + self._analysis_active = False + # Check if model has criterion attribute if not hasattr(self, "criterion"): raise AttributeError( @@ -203,7 +372,7 @@ def training_step(self, batch, batch_idx): Output from the wrapped module's training_step. """ batch_measure = None - if self.cross_response: + if self.cross_response and not self._independent_measure: # Unpack batch if provided as tuple (batch, batch_idx, dataloader_idx) if type(batch) is tuple and len(batch) == 3: batch, _batch_idx, dataloader_idx = batch @@ -223,7 +392,10 @@ def training_step(self, batch, batch_idx): # Initializing manual optimization opt = self.optimizers() - opt.zero_grad() + + # Zero gradients only at start of accumulation cycle + if self._accumulation_count == 0: + opt.zero_grad() # BEFORE logic if not self.disable_analyzer: @@ -232,16 +404,40 @@ def training_step(self, batch, batch_idx): # Original training step output = super().training_step(batch, batch_idx) if not self.delegate_optimization: - # Backward pass - self.manual_backward(output) - # Optimizer step - opt.step() + # Scale loss for gradient accumulation + scaled_output = output / self.accumulation_steps + self.manual_backward(scaled_output) + + self._accumulation_count += 1 + + if self.accumulation_steps > 1: + self._accum_step_losses.append(output.detach()) + # Tag every micro-batch with its cycle's effective_step so + # groupby(effective_step).mean() in the plot averages exactly + # the K micro-batches of that cycle (no Lightning forward-fill + # ambiguity). opt.step() has not fired yet, so +1 gives the + # current cycle number. + self.log( + "effective_step", + float(self._optimizer_step_count + 1), + on_step=True, + on_epoch=False, + ) - # Step schedulers with interval='step' - if self._trainer is not None and self.trainer.lr_scheduler_configs: - for config in self.trainer.lr_scheduler_configs: - if config.interval == "step": - config.scheduler.step() + # Step optimizer only at end of accumulation cycle + if self._accumulation_count >= self.accumulation_steps: + opt.step() + self._optimizer_step_count += 1 + self._accumulation_count = 0 + + if self.accumulation_steps > 1 and self._accum_step_losses: + self._accum_step_losses.clear() + + # Step schedulers with interval='step' + if self._trainer is not None and self.trainer.lr_scheduler_configs: + for config in self.trainer.lr_scheduler_configs: + if config.interval == "step": + config.scheduler.step() # AFTER logic if not self.disable_analyzer: @@ -263,6 +459,13 @@ def on_train_epoch_end(self): super().on_train_epoch_end() + @property + def effective_step(self): + """Return the effective optimizer step count.""" + if self.delegate_optimization: + return self.global_step + return self._optimizer_step_count + def _should_analyze(self, step: int) -> bool: """Determine if analysis should run at the given step.""" # If schedule provided, use it @@ -278,8 +481,9 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): """Hook executed before the wrapped training step. Computes analysis metrics including sample-wise gradients and - linearization probes. Only runs if the scheduler determines - this step should be analyzed. + linearization probes. When gradient accumulation is active, + metrics are accumulated across micro-batches and only logged + after the full accumulation cycle. Args: batch: Training batch containing input data and labels. @@ -289,29 +493,36 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): Returns: None """ - # Check if we should run analysis at this step - if not self._should_analyze(self.global_step): + if self.accumulation_steps == 1: + return self._analyze_single_step(batch, batch_idx, cross_response_batch) + else: + return self._analyze_accumulated_step( + batch, batch_idx, cross_response_batch + ) + + def _analyze_single_step(self, batch, batch_idx, cross_response_batch=None): + """Run analysis for a single step (no accumulation).""" + if not self._should_analyze(self.effective_step): return None x, y = batch - - # Get cross-response batch if available + # Get cross-response batch if applicable x2, y2 = None, None if self.cross_response: x2, y2 = cross_response_batch samples_results = {} with BatchStatSnapshot(self.model, x): - # Compute samplewise metrics for the training batch + # Compute sample-wise metrics and self response samples_results["self"] = self.sample_calc.compute( self.model, self.criterion, x, y, ) - # Compute samplewise metrics for the cross batch if available + # Compute sample-wise metrics and cross response if applicable if x2 is not None and y2 is not None: - probe_results_cross_preliminary = self.sample_calc.compute( + cross_preliminary = self.sample_calc.compute( self.model, self.criterion, x2, @@ -319,10 +530,9 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): ) samples_results["cross"] = self.sample_calc.compute_cross_metrics( sample_wise_metrics_self=samples_results["self"], - sample_wise_metrics_cross=probe_results_cross_preliminary, + sample_wise_metrics_cross=cross_preliminary, ) - - # Linearizer compute + # Linearizer probe probe_results = self.linearizer.compute( model=self.model, criterion=self.criterion, @@ -334,22 +544,34 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): # Get "self" result for coupling calculation loss_self, _, delta_loss_self = probe_results["self"] - - # Compute coupling value (using self response) chi_coup = self.coupling_calc.calculate( delta_loss=delta_loss_self, chi_loss=samples_results["self"]["batch_grad_norms_loss"], chi_net=samples_results["self"]["batch_grad_norms_network"], ) + chi_coup_cross = None if self.cross_response and "cross" in probe_results: - # Optionally, compute coupling for cross response as well - loss_cross, _, delta_loss_cross = probe_results["cross"] + _, _, delta_loss_cross = probe_results["cross"] chi_coup_cross = self.coupling_calc.calculate( delta_loss=delta_loss_cross, chi_loss=samples_results["cross"]["batch_grad_norms_loss"], chi_net=samples_results["cross"]["batch_grad_norms_network"], ) + # Capture the train gradient for the independent measure-response + # path. Linearizer.compute() zeroes grads internally and discards + # its own gradient, so we recompute it here (K_train=1) rather than + # reaching into the linearizer's internals. + grad_train_mean = None + if self._independent_measure: + self.model.zero_grad() + loss_t = self.criterion(self.model(x), y) + loss_t.backward() + grad_train_mean = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + self.model.zero_grad() # Log results with fixed metric names if self.log_metrics: self._log_analysis_results( @@ -359,7 +581,6 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): chi_coup=chi_coup, batch_size=x.shape[0], ) - # Log cross response if available if "cross" in samples_results and samples_results["cross"] is not None: self._log_analysis_results( @@ -369,19 +590,431 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): chi_coup=chi_coup_cross, batch_size=x2.shape[0] if x2 is not None else 0, ) - # Log window tracking info if using logarithmic schedule if self.analysis_schedule is not None: window_info = self.analysis_schedule.get_window_info( - self.global_step + self.effective_step ) if window_info is not None: self.log("window_id", window_info["window_id"]) self.log("window_center", window_info["window_center"]) self.log("window_width", window_info["window_width"]) + if self._independent_measure: + self._measure_response( + grad_train_mean=grad_train_mean, + self_chi_metrics=samples_results["self"], + ) + + return None + + def _analyze_accumulated_step( + self, batch, batch_idx, cross_response_batch=None + ): + """Run analysis with gradient accumulation across micro-batches. + + On each micro-batch: accumulate sample-wise metrics and linearizer + gradients. On the last micro-batch of the cycle: finalize, log, clear. + """ + # On first micro-batch of cycle, decide whether to analyze + if self._accumulation_count == 0: + self._analysis_active = self._should_analyze(self.effective_step) + if self._analysis_active: + self._clear_accumulation_buffers() + + if not self._analysis_active: + return None + + x, y = batch + x2, y2 = None, None + if self.cross_response: + x2, y2 = cross_response_batch + + # Save training grads before any analysis backward/zero_grad calls. + # sample_calc.compute and _accumulate_linearizer_grads both call + # model.zero_grad() internally; restoring here ensures the training + # accumulation loop sees unmodified gradients after this hook. + saved_grads = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + + with BatchStatSnapshot(self.model, x): + # Accumulate sample-wise metrics + self_metrics = self.sample_calc.compute( + self.model, + self.criterion, + x, + y, + ) + self._accum_chi_net.append(self_metrics["batch_grad_norms_network"]) + self._accum_chi_loss.append(self_metrics["batch_grad_norms_loss"]) + + if x2 is not None and y2 is not None: + cross_preliminary = self.sample_calc.compute( + self.model, + self.criterion, + x2, + y2, + ) + cross_metrics = self.sample_calc.compute_cross_metrics( + sample_wise_metrics_self=self_metrics, + sample_wise_metrics_cross=cross_preliminary, + ) + self._accum_cross_chi_net.append( + cross_metrics["batch_grad_norms_network"] + ) + self._accum_cross_chi_loss.append( + cross_metrics["batch_grad_norms_loss"] + ) + + # Accumulate linearizer gradients (train side) + self._accumulate_linearizer_grads(x, y, is_train=True) + # Accumulate linearizer gradients (measure side). This legacy + # path ties the measure batch to the train accumulation cycle + # (CombinedLoader "measure" key, size = K_train * micro). For + # an independently sized (and independently accumulated) + # measure batch, use measure_dataloader/measure_batch_size + # instead (see _measure_response), which accumulates and + # combines the measure side separately with its own K. + if x2 is not None and y2 is not None: + self._accumulate_linearizer_grads(x2, y2, is_train=False) + + # Restore training grads clobbered by analysis backward passes + for p, s in zip(self.model.parameters(), saved_grads): + p.grad = s + + # On last micro-batch: finalize and log + is_last = self._accumulation_count == self.accumulation_steps - 1 + if is_last: + self._finalize_accumulated_analysis(x, x2) + return None + def _accumulate_linearizer_grads(self, x, y, is_train=True): + """Forward+backward on a micro-batch and add grads to accumulator. + + Must be called inside a BatchStatSnapshot context (caller's + responsibility). Training grads are saved/restored by the caller + (_analyze_accumulated_step) around the full analysis block. + """ + self.model.zero_grad() + loss = self.criterion(self.model(x), y) + loss.backward() + + loss_val = loss.detach().item() + if is_train: + self._accum_train_loss += loss_val + if self._accum_grad_train is None: + self._accum_grad_train = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + else: + for acc, p in zip(self._accum_grad_train, self.model.parameters()): + if acc is not None and p.grad is not None: + acc.add_(p.grad) + else: + self._accum_measure_loss += loss_val + if self._accum_grad_measure is None: + self._accum_grad_measure = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + else: + for acc, p in zip( + self._accum_grad_measure, self.model.parameters() + ): + if acc is not None and p.grad is not None: + acc.add_(p.grad) + + def _finalize_accumulated_analysis(self, x, x2): + """Combine accumulated metrics and log results.""" + K = self.accumulation_steps + B = x.shape[0] + + # Combine sample-wise metrics. + # Both chi_net and chi_loss are computed with normalize=True, which + # makes them extensive in the batch size via a 1/B or *B factor + # derived from mean-reduced loss. For an effective batch N=K*B, the + # correct aggregate for both quantities is the mean across micro-batches. + chi_net_eff = sum(self._accum_chi_net) / K + chi_loss_eff = sum(self._accum_chi_loss) / K + + samples_result_self = { + "batch_grad_norms_network": chi_net_eff, + "batch_grad_norms_loss": chi_loss_eff, + } + + # Compute self linearizer result from accumulated grads + grad_norm_sq = sum( + (g**2).sum().item() for g in self._accum_grad_train if g is not None + ) / (K**2) + + avg_train_loss = self._accum_train_loss / K + delta_loss_self = -grad_norm_sq + probe_result_self = ( + avg_train_loss, + avg_train_loss + delta_loss_self, + delta_loss_self, + ) + + chi_coup = self.coupling_calc.calculate( + delta_loss=delta_loss_self, + chi_loss=chi_loss_eff, + chi_net=chi_net_eff, + ) + + # Cross response + samples_result_cross = None + probe_result_cross = None + chi_coup_cross = None + if self._accum_grad_measure is not None: + chi_net_cross_eff = sum(self._accum_cross_chi_net) / K + chi_loss_cross_eff = sum(self._accum_cross_chi_loss) / K + samples_result_cross = { + "batch_grad_norms_network": chi_net_cross_eff, + "batch_grad_norms_loss": chi_loss_cross_eff, + } + + cross_dot = sum( + (g1 * g2).sum().item() + for g1, g2 in zip( + self._accum_grad_train, + self._accum_grad_measure, + ) + if g1 is not None and g2 is not None + ) / (K**2) + + avg_measure_loss = self._accum_measure_loss / K + delta_loss_cross = -cross_dot + probe_result_cross = ( + avg_measure_loss, + avg_measure_loss + delta_loss_cross, + delta_loss_cross, + ) + chi_coup_cross = self.coupling_calc.calculate( + delta_loss=delta_loss_cross, + chi_loss=chi_loss_cross_eff, + chi_net=chi_net_cross_eff, + ) + + # Log results + if self.log_metrics: + self._log_analysis_results( + prefix="", + samples_result=samples_result_self, + probe_result=probe_result_self, + chi_coup=chi_coup, + batch_size=B, + ) + if samples_result_cross is not None: + self._log_analysis_results( + prefix="cross_", + samples_result=samples_result_cross, + probe_result=probe_result_cross, + chi_coup=chi_coup_cross, + batch_size=x2.shape[0] if x2 is not None else 0, + ) + if self.analysis_schedule is not None: + window_info = self.analysis_schedule.get_window_info( + self.effective_step + ) + if window_info is not None: + self.log("window_id", window_info["window_id"]) + self.log("window_center", window_info["window_center"]) + self.log("window_width", window_info["window_width"]) + + if self._independent_measure: + grad_train_mean = [ + g / K if g is not None else None for g in self._accum_grad_train + ] + self._measure_response( + grad_train_mean=grad_train_mean, + self_chi_metrics=samples_result_self, + ) + + self._clear_accumulation_buffers() + + def _clear_accumulation_buffers(self): + """Reset all accumulation buffers.""" + self._accum_chi_net.clear() + self._accum_chi_loss.clear() + self._accum_cross_chi_net.clear() + self._accum_cross_chi_loss.clear() + self._accum_grad_train = None + self._accum_grad_measure = None + self._accum_train_loss = 0.0 + self._accum_measure_loss = 0.0 + self._accum_step_losses.clear() + + def _next_measure_micro_batch(self): + """Pull one measure micro-batch from the persistent iterator. + + The iterator is created lazily on first use and refilled on + exhaustion so a finite measure_dataloader cycles indefinitely + (a MultiEpochsDataLoader is already infinite and simply keeps + yielding). Returns tensors moved to self.device. + """ + if self._measure_iter is None: + self._measure_iter = iter(self._measure_dataloader) + try: + xb, yb = next(self._measure_iter) + except StopIteration: + self._measure_iter = iter(self._measure_dataloader) + xb, yb = next(self._measure_iter) + + micro = self._measure_micro_batch_size + if xb.shape[0] != micro: + raise ValueError( + f"measure_dataloader yielded a micro-batch of size " + f"{xb.shape[0]}, expected {micro}. Use drop_last=True " + f"(or a dataset size divisible by batch_size) so every " + f"measure micro-batch is full." + ) + return xb.to(self.device), yb.to(self.device) + + def _measure_response(self, grad_train_mean, self_chi_metrics): + """Compute and log cross-response metrics against an independent + measure batch, decoupled from the training accumulation. + + Gathers a single pool of size max(measure_batch_size) from the + persistent measure iterator, then processes every requested + measure batch size largest-to-smallest: the largest uses the + whole pool, each smaller size uses a seed-fixable random subset + of that same pool (so subset draws are reproducible given + measure_subset_seed). For each size, the measure gradient and + per-sample chi metrics are accumulated over its own micro-batch + chunks (measure-side gradient accumulation), then combined and + logged with a "@bs{S}" suffix when sweeping multiple sizes. + + Args: + grad_train_mean: List aligned with self.model.parameters(), + the mean training gradient (accumulated grad / K_train). + self_chi_metrics: Dict with "batch_grad_norms_network" and + "batch_grad_norms_loss", the aggregated train self chi + metrics used as the "self" side of the cross metric. + + Note: + Runs its own forward/backward passes; saves and restores + self.model gradients so the surrounding (possibly partially + accumulated) training gradient is never corrupted. + """ + saved_grads = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + + sizes = sorted(self._measure_batch_sizes, reverse=True) + S_max = sizes[0] + micro = self._measure_micro_batch_size + sweep = len(self._measure_batch_sizes) > 1 + + # Gather ONE pool of S_max samples from the persistent iterator. + # S_max need not be a multiple of micro (e.g. every swept size is + # below the max single-pass size), so pull enough micro-batches + # to cover it and slice down to exactly S_max samples. + pool_x, pool_y = [], [] + for _ in range(math.ceil(S_max / micro)): + xb, yb = self._next_measure_micro_batch() + pool_x.append(xb) + pool_y.append(yb) + x_pool = torch.cat(pool_x, dim=0)[:S_max] + y_pool = torch.cat(pool_y, dim=0)[:S_max] + + for S in sizes: + if S == S_max: + idx = torch.arange(S_max, device=x_pool.device) + else: + # Seed-fixable random subset of the same pool, drawn + # after larger sizes so the sweep is reproducible given + # measure_subset_seed regardless of how many sizes ran. + perm = torch.randperm(S_max, generator=self._measure_gen) + idx = perm[:S].to(x_pool.device) + x_sel, y_sel = x_pool[idx], y_pool[idx] + + # chunk_size = min(S, micro) unifies both regimes: when + # S <= micro this gives chunk_size=S, K_measure=1 (a single + # direct pass using less than the max single-pass capacity, + # no accumulation); when S > micro this gives + # chunk_size=micro, K_measure=S // micro (accumulated passes + # of the max single-pass size — exact, since S > micro must + # be a multiple of micro per __init__ validation). + chunk_size = min(S, micro) + K_measure = S // chunk_size + grad_measure_acc = None + measure_loss_sum = 0.0 + chi_net_chunks, chi_loss_chunks = [], [] + + for k in range(K_measure): + xc = x_sel[k * chunk_size : (k + 1) * chunk_size] + yc = y_sel[k * chunk_size : (k + 1) * chunk_size] + with BatchStatSnapshot(self.model, xc): + m = self.sample_calc.compute(self.model, self.criterion, xc, yc) + chi_net_chunks.append(m["batch_grad_norms_network"]) + chi_loss_chunks.append(m["batch_grad_norms_loss"]) + + self.model.zero_grad() + loss = self.criterion(self.model(xc), yc) + loss.backward() + measure_loss_sum += loss.detach().item() + if grad_measure_acc is None: + grad_measure_acc = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + else: + for acc, p in zip( + grad_measure_acc, self.model.parameters() + ): + if acc is not None and p.grad is not None: + acc.add_(p.grad) + + measure_chi = { + "batch_grad_norms_network": sum(chi_net_chunks) / K_measure, + "batch_grad_norms_loss": sum(chi_loss_chunks) / K_measure, + } + cross_metrics = self.sample_calc.compute_cross_metrics( + sample_wise_metrics_self=self_chi_metrics, + sample_wise_metrics_cross=measure_chi, + ) + + grad_measure_mean = [ + g / K_measure if g is not None else None for g in grad_measure_acc + ] + cross_dot = sum( + (g1 * g2).sum().item() + for g1, g2 in zip(grad_train_mean, grad_measure_mean) + if g1 is not None and g2 is not None + ) + avg_measure_loss = measure_loss_sum / K_measure + delta_loss_cross = -cross_dot + probe_result_cross = ( + avg_measure_loss, + avg_measure_loss + delta_loss_cross, + delta_loss_cross, + ) + chi_coup_cross = self.coupling_calc.calculate( + delta_loss=delta_loss_cross, + chi_loss=cross_metrics["batch_grad_norms_loss"], + chi_net=cross_metrics["batch_grad_norms_network"], + ) + + if self.log_metrics: + suffix = f"@bs{S}" if sweep else "" + self._log_analysis_results( + prefix="cross_", + samples_result=cross_metrics, + probe_result=probe_result_cross, + chi_coup=chi_coup_cross, + batch_size=S, + suffix=suffix, + log_effective_batch_size=False, + ) + + for p, s in zip(self.model.parameters(), saved_grads): + p.grad = s + def _log_analysis_results( self, prefix: str, @@ -389,28 +1022,56 @@ def _log_analysis_results( probe_result: tuple, chi_coup: Optional[float], batch_size: int, + suffix: str = "", + log_effective_batch_size: Optional[bool] = None, ): - """Helper method to log analysis metrics with a given prefix.""" + """Helper method to log analysis metrics with a given prefix. + + Args: + suffix: Appended to every logged key. Used to disambiguate + swept measure batch sizes, e.g. "@bs2000". + log_effective_batch_size: Whether to additionally log + "{prefix}effective_batch_size{suffix}" as + batch_size * accumulation_steps. Defaults to + accumulation_steps > 1 (existing behavior). Independent + measure batches pass False explicitly since the train + accumulation_steps multiplier has no meaning for them + (batch_size is already the full measure size). + """ + if log_effective_batch_size is None: + log_effective_batch_size = self.accumulation_steps > 1 + # Log sample-wise metrics if "batch_grad_norms_network" in samples_result: - self.log(f"{prefix}chi_net", samples_result["batch_grad_norms_network"]) + self.log( + f"{prefix}chi_net{suffix}", + samples_result["batch_grad_norms_network"], + ) if "batch_grad_norms_loss" in samples_result: - self.log(f"{prefix}chi_loss", samples_result["batch_grad_norms_loss"]) + self.log( + f"{prefix}chi_loss{suffix}", + samples_result["batch_grad_norms_loss"], + ) # Log coupling if provided if chi_coup is not None: - self.log(f"{prefix}chi_coup", chi_coup) + self.log(f"{prefix}chi_coup{suffix}", chi_coup) - self.log(f"{prefix}batch_size", batch_size) + self.log(f"{prefix}batch_size{suffix}", batch_size) + if log_effective_batch_size: + self.log( + f"{prefix}effective_batch_size{suffix}", + batch_size * self.accumulation_steps, + ) - # Only log analysis_step once (usually with empty prefix) - if prefix == "": - self.log("analysis_step", self.global_step) + # Only log analysis_step once (usually with empty prefix/suffix) + if prefix == "" and suffix == "": + self.log("analysis_step", self.effective_step) # Log probe results (linearization) if probe_result is not None: loss, _, delta_loss = probe_result - self.log(f"{prefix}loss", loss) + self.log(f"{prefix}loss{suffix}", loss) # For cross response, we might want to name it differently or keep # consistent. @@ -420,7 +1081,7 @@ def _log_analysis_results( metric_name = ( "grad_norm_squared" if prefix == "" else "grad_dot_product" ) - self.log(f"{prefix}{metric_name}", -delta_loss) + self.log(f"{prefix}{metric_name}{suffix}", -delta_loss) def _after_training_step(self, batch, batch_idx, output): """Hook executed after the wrapped training step. @@ -443,5 +1104,10 @@ def _after_training_step(self, batch, batch_idx, output): analyze_every=analyze_every, analysis_schedule=analysis_schedule, cross_response=cross_response, + micro_batch_size=micro_batch_size, + effective_batch_size=effective_batch_size, + measure_dataloader=measure_dataloader, + measure_batch_size=measure_batch_size, + measure_subset_seed=measure_subset_seed, **model_kwargs, ) diff --git a/tests/integration/test_analyzer_deployment.py b/tests/integration/test_analyzer_deployment.py index ed92e08..11e0711 100644 --- a/tests/integration/test_analyzer_deployment.py +++ b/tests/integration/test_analyzer_deployment.py @@ -1247,6 +1247,278 @@ def mock_log(name, value, *args, **kwargs): assert not torch.isnan(torch.tensor(logged_metrics["cross_grad_dot_product"])) +class TestAnalyzerWithIndependentMeasure: + """Integration tests for the independent measure_dataloader / + measure_batch_size path (decoupled from train batch/accumulation).""" + + class _MetricsTracker(pl.Callback): + """Collects trainer.callback_metrics after every training batch.""" + + def __init__(self): + self.metrics = [] + + def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx): + self.metrics.append(dict(trainer.callback_metrics)) + + def test_independent_measure_end_to_end(self, simple_lightning_module): + """A plain train DataLoader + measure_dataloader (no CombinedLoader) + trains end-to-end and logs cross-response metrics.""" + torch.manual_seed(42) + + train_x = torch.randn(32, 10) + train_y = torch.randint(0, 2, (32,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=8, drop_last=True + ) + + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_epochs=1, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + last = tracker.metrics[-1] + for key in ( + "cross_loss", + "cross_grad_dot_product", + "cross_chi_net", + "cross_chi_loss", + "cross_chi_coup", + ): + assert key in last + + def test_independent_measure_with_accumulation(self, simple_lightning_module): + """Independent measure batch combined with train-side gradient + accumulation exercises the _finalize_accumulated_analysis call site.""" + torch.manual_seed(42) + + train_x = torch.randn(32, 10) + train_y = torch.randint(0, 2, (32,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=4) + + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + measure_dataloader=measure_loader, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_epochs=1, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + logged_cross = [m for m in tracker.metrics if "cross_grad_dot_product" in m] + assert len(logged_cross) > 0 + + def test_independent_measure_sweep_suffixed_keys(self, simple_lightning_module): + """A measure_batch_size sweep under a logarithmic schedule logs + @bs{S}-suffixed cross metrics instead of the unsuffixed cross_* keys.""" + from perspic.logger import logarithmic_windows + + torch.manual_seed(42) + + train_x = torch.randn(64, 10) + train_y = torch.randint(0, 2, (64,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + schedule = logarithmic_windows(max_steps=8) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + measure_subset_seed=0, + analysis_schedule=schedule, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_steps=4, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + all_keys = set() + for m in tracker.metrics: + all_keys.update(m.keys()) + + assert "cross_grad_dot_product@bs4" in all_keys + assert "cross_grad_dot_product@bs8" in all_keys + assert "cross_grad_dot_product" not in all_keys + + def test_independent_measure_sweep_mixed_regime(self, simple_lightning_module): + """A sweep spanning sizes below, at, and above the measure loader's + batch_size (the max single-pass size) trains end-to-end and logs + every size: below/at it as a single direct pass, above it as an + accumulated measurement.""" + from perspic.logger import logarithmic_windows + + torch.manual_seed(42) + + train_x = torch.randn(64, 10) + train_y = torch.randint(0, 2, (64,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + measure_x = torch.randn(32, 10) + measure_y = torch.randint(0, 2, (32,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=8, drop_last=True + ) + + schedule = logarithmic_windows(max_steps=8) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[2, 8, 32], + measure_subset_seed=0, + analysis_schedule=schedule, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_steps=4, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + all_keys = set() + for m in tracker.metrics: + all_keys.update(m.keys()) + + for S in (2, 8, 32): + assert f"cross_grad_dot_product@bs{S}" in all_keys + assert f"cross_batch_size@bs{S}" in all_keys + + def test_independent_measure_reproducible(self, simple_lightning_module): + """Two fits with the same measure_subset_seed produce an identical + swept cross metric (the measure loader is shuffle=False, so only the + subset draw needs a fixed seed for full reproducibility).""" + from perspic.logger import logarithmic_windows + + def run(): + torch.manual_seed(42) + train_x = torch.randn(32, 10) + train_y = torch.randint(0, 2, (32,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + torch.manual_seed(7) + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + schedule = logarithmic_windows(max_steps=8) + torch.manual_seed(42) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + measure_subset_seed=99, + analysis_schedule=schedule, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_steps=4, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + for m in reversed(tracker.metrics): + if "cross_grad_dot_product@bs4" in m: + return m["cross_grad_dot_product@bs4"] + return None + + val_a = run() + val_b = run() + + assert val_a is not None + assert torch.allclose(torch.as_tensor(val_a), torch.as_tensor(val_b)) + + def test_measure_loader_smaller_than_train(self, simple_lightning_module): + """A measure_dataloader much smaller than the training run cycles via + the persistent iterator without raising StopIteration.""" + torch.manual_seed(42) + + train_x = torch.randn(64, 10) + train_y = torch.randint(0, 2, (64,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=4) + + measure_x = torch.randn(4, 10) + measure_y = torch.randint(0, 2, (4,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + + trainer = pl.Trainer( + max_steps=16, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + ) + + # Should complete without StopIteration + trainer.fit(model, train_loader) + assert trainer.global_step == 16 + + class TestSchedulerIntegration: """Integration tests for learning rate schedulers.""" diff --git a/tests/unit/test_analyzer.py b/tests/unit/test_analyzer.py index 6046012..c68b86e 100644 --- a/tests/unit/test_analyzer.py +++ b/tests/unit/test_analyzer.py @@ -1,5 +1,6 @@ """Unit tests for the analyzer module.""" +import warnings from unittest.mock import Mock, patch import pytest @@ -7,6 +8,7 @@ import torch import torch.nn as nn import torch.nn.functional as F +from torch.utils.data import DataLoader, TensorDataset from perspic.analyzer import analyzer from perspic.calculator.linearizer import Linearizer @@ -104,6 +106,16 @@ def sample_batch(): return x, y +def _make_measure_dataloader(n_samples, micro, seed=123, drop_last=True): + """Build a deterministic measure DataLoader (shuffle=False) for testing + the independent measure-batch-size path.""" + g = torch.Generator().manual_seed(seed) + x = torch.randn(n_samples, 10, generator=g) + y = torch.randint(0, 2, (n_samples,), generator=g) + ds = TensorDataset(x, y) + return DataLoader(ds, batch_size=micro, shuffle=False, drop_last=drop_last) + + # Test Classes class TestAnalyzerFactoryFunction: """Test the analyzer factory function.""" @@ -717,10 +729,7 @@ def test_before_hook_skipped_when_not_scheduled( ): """Test _before_training_step skips analysis when not scheduled.""" model = analyzer(simple_lightning_module, analyze_every=10) - model._global_step = 5 # Not a multiple of 10 - - # Mock global_step property - type(model).global_step = property(lambda self: 5) + model._optimizer_step_count = 5 # Not a multiple of 10 model._before_training_step(sample_batch, 0) @@ -753,7 +762,6 @@ def test_logs_window_info_with_schedule( simple_lightning_module, analysis_schedule=schedule, log_metrics=True ) model.log = Mock() - type(model).global_step = property(lambda self: 0) model._before_training_step(sample_batch, 0) @@ -779,7 +787,6 @@ def test_no_window_info_without_schedule( model = analyzer(simple_lightning_module, log_metrics=True) model.log = Mock() - type(model).global_step = property(lambda self: 0) model._before_training_step(sample_batch, 0) @@ -787,3 +794,1290 @@ def test_no_window_info_without_schedule( assert "window_id" not in logged_names assert "window_center" not in logged_names assert "window_width" not in logged_names + + +class TestGradientAccumulation: + """Test gradient accumulation functionality.""" + + # The tests are categorized into sections A-J for clarity. + + # --- A. Parameter validation --- + + def test_accumulation_steps_default(self, simple_lightning_module): + """No params → accumulation_steps=1.""" + model = analyzer(simple_lightning_module) + assert model.accumulation_steps == 1 + + def test_batch_size_only_no_accumulation(self, simple_lightning_module): + """micro_batch_size alone → no accumulation, value stored.""" + model = analyzer(simple_lightning_module, micro_batch_size=8) + assert model.accumulation_steps == 1 + assert model.micro_batch_size == 8 + assert model.effective_batch_size is None + + def test_accumulation_steps_computed(self, simple_lightning_module): + """micro=8, effective=32 → accumulation_steps=4.""" + model = analyzer( + simple_lightning_module, + micro_batch_size=8, + effective_batch_size=32, + ) + assert model.accumulation_steps == 4 + + def test_effective_without_micro_batch_raises(self, simple_lightning_module): + """effective_batch_size alone → ValueError.""" + with pytest.raises(ValueError, match="micro_batch_size must be specified"): + analyzer( + simple_lightning_module, + effective_batch_size=32, + ) + + def test_effective_less_than_micro_batch_raises(self, simple_lightning_module): + """effective=8, micro=32 → ValueError.""" + with pytest.raises(ValueError, match="must be >= micro_batch_size"): + analyzer( + simple_lightning_module, + micro_batch_size=32, + effective_batch_size=8, + ) + + def test_not_divisible_raises(self, simple_lightning_module): + """effective=30, micro=8 → ValueError (not divisible).""" + with pytest.raises(ValueError, match="must be divisible"): + analyzer( + simple_lightning_module, + micro_batch_size=8, + effective_batch_size=30, + ) + + def test_accumulation_with_delegate_raises(self, manual_optimization_module): + """Accumulation + manual optimization module → ValueError.""" + with pytest.raises( + ValueError, + match="Gradient accumulation is not supported", + ): + with pytest.warns(UserWarning, match="manual optimization"): + analyzer( + manual_optimization_module, + micro_batch_size=8, + effective_batch_size=32, + ) + + # --- B. Optimizer behavior --- + + def test_zero_grad_once_per_cycle(self, simple_lightning_module, sample_batch): + """4 micro-steps, accum=4: zero_grad called exactly 1x.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.zero_grad.call_count == 1 + + def test_step_once_per_cycle(self, simple_lightning_module, sample_batch): + """4 micro-steps, accum=4: opt.step() called exactly 1x.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.step.call_count == 1 + + def test_step_not_called_mid_cycle(self, simple_lightning_module, sample_batch): + """3 of 4 micro-steps done: opt.step() never called.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(3): + model.training_step((x, y), i) + + mock_opt.step.assert_not_called() + + def test_loss_scaled_for_backward(self, simple_lightning_module, sample_batch): + """manual_backward receives loss / accumulation_steps.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + output = model.training_step((x, y), 0) + + backward_arg = model.manual_backward.call_args[0][0] + expected = output / 4 + assert torch.allclose(backward_arg, expected) + + def test_unscaled_loss_returned(self, simple_lightning_module, sample_batch): + """training_step returns the original unscaled loss.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + output_accum = model.training_step((x, y), 0) + assert isinstance(output_accum, torch.Tensor) + + def test_two_full_cycles(self, simple_lightning_module, sample_batch): + """4 steps, accum=2: zero_grad 2x, opt.step() 2x.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=8, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.zero_grad.call_count == 2 + assert mock_opt.step.call_count == 2 + + # --- C. Backwards compatibility --- + + def test_no_accumulation_backwards_compatible( + self, simple_lightning_module, sample_batch + ): + """No accum params: every call does zero_grad + step.""" + model = analyzer(simple_lightning_module, disable_analyzer=True) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.zero_grad.call_count == 4 + assert mock_opt.step.call_count == 4 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_single_step_path_unchanged( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """Without accumulation, _analyze_single_step produces same results.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.5), + "batch_grad_norms_loss": torch.tensor(2.5), + } + mock_probe.return_value = { + "self": (1.0, 0.0, -1.0), + "cross": None, + } + + model = analyzer(simple_lightning_module, log_metrics=True) + model.log = Mock() + x, y = sample_batch + + model._before_training_step((x, y), 0) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + assert torch.allclose(logged["chi_net"], torch.tensor(1.5)) + assert torch.allclose(logged["chi_loss"], torch.tensor(2.5)) + assert logged["loss"] == 1.0 + assert logged["grad_norm_squared"] == 1.0 + assert logged["batch_size"] == 4 + + # --- D. Effective step --- + + def test_effective_step_with_accumulation(self, simple_lightning_module): + """_optimizer_step_count=2 → effective_step=2.""" + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=16, + ) + model._optimizer_step_count = 2 + assert model.effective_step == 2 + + model._optimizer_step_count = 0 + assert model.effective_step == 0 + + def test_effective_step_without_accumulation(self, simple_lightning_module): + """_optimizer_step_count=42 → effective_step=42.""" + model = analyzer(simple_lightning_module) + model._optimizer_step_count = 42 + assert model.effective_step == 42 + + # --- E. Analysis scheduling with accumulation --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_analysis_uses_effective_step_for_scheduling( + self, mock_compute, simple_lightning_module, sample_batch + ): + """_should_analyze uses effective_step, activates on first micro-batch.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + analyze_every=2, + micro_batch_size=4, + effective_batch_size=16, + ) + model.log = Mock() + x, y = sample_batch + + # effective_step=0, analyze_every=2 → 0 % 2 == 0 → analyze + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + assert model._analysis_active is True + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_analysis_skipped_when_schedule_says_no( + self, mock_compute, simple_lightning_module, sample_batch + ): + """When _should_analyze returns False, no accumulation or logging happens.""" + model = analyzer( + simple_lightning_module, + analyze_every=10, + micro_batch_size=4, + effective_batch_size=8, + ) + model.log = Mock() + x, y = sample_batch + + # effective_step=1, analyze_every=10 → 1 % 10 != 0 → skip + model._optimizer_step_count = 1 + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + + assert model._analysis_active is False + mock_compute.assert_not_called() + model.log.assert_not_called() + + # Second micro-batch also skipped (flag persists) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + mock_compute.assert_not_called() + model.log.assert_not_called() + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_analysis_step_logs_effective_step( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Logged analysis_step equals effective_step, not global_step.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # _optimizer_step_count=3 → effective_step=3 + model._optimizer_step_count = 3 + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + assert logged["analysis_step"] == 3 + + # --- F. Sample-wise metric accumulation --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_accumulated_chi_net_is_mean( + self, mock_compute, simple_lightning_module, sample_batch + ): + """chi_net_eff = mean of per-micro-batch chi_net values.""" + mock_compute.side_effect = [ + { + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(3.0), + }, + { + "batch_grad_norms_network": torch.tensor(4.0), + "batch_grad_norms_loss": torch.tensor(5.0), + }, + ] + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + + # chi_net_eff = mean([2.0, 4.0]) = 3.0 + assert torch.allclose(logged["chi_net"], torch.tensor(3.0)) + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_accumulated_chi_loss_is_mean( + self, mock_compute, simple_lightning_module, sample_batch + ): + """chi_loss_eff = mean of per-micro-batch chi_loss values. + + chi_loss is computed with normalize=True against a mean-reduced loss, + so the correct aggregation for an effective batch is the mean across + micro-batches (same as chi_net), not K * sum. + """ + mock_compute.side_effect = [ + { + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(3.0), + }, + { + "batch_grad_norms_network": torch.tensor(4.0), + "batch_grad_norms_loss": torch.tensor(5.0), + }, + ] + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + + # chi_loss_eff = mean([3.0, 5.0]) = 4.0 + assert torch.allclose(logged["chi_loss"], torch.tensor(4.0)) + + # --- G. Linearizer gradient accumulation --- + + def test_linearizer_accumulated_grad_norm( + self, simple_lightning_module, sample_batch + ): + """grad_norm_squared = ||Σ∇L_k||² / K² from accumulated grads.""" + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # Run a full accumulation cycle (K=2) + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ): + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + + # grad_norm_squared should be a positive float + assert "grad_norm_squared" in logged + assert logged["grad_norm_squared"] > 0 + + # Verify manually: compute the expected value + # Do two forward+backward passes, sum grads, compute ||sum||²/K² + model.model.zero_grad() + loss0 = model.criterion(model.model(x), y) + loss0.backward() + grads_0 = [ + p.grad.clone() for p in model.model.parameters() if p.grad is not None + ] + + model.model.zero_grad() + loss1 = model.criterion(model.model(x), y) + loss1.backward() + grads_1 = [ + p.grad.clone() for p in model.model.parameters() if p.grad is not None + ] + model.model.zero_grad() + + expected_norm_sq = ( + sum(((g0 + g1) ** 2).sum().item() for g0, g1 in zip(grads_0, grads_1)) / 4 + ) # K² = 2² = 4 + + assert abs(logged["grad_norm_squared"] - expected_norm_sq) < 1e-4 + + # --- H. Coupling with accumulated values --- + + def test_coupling_from_accumulated_values( + self, simple_lightning_module, sample_batch + ): + """coupling = grad_norm_sq / (chi_loss_eff * chi_net_eff).""" + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(3.0), + }, + ): + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + + # chi_net_eff = mean([2.0, 2.0]) = 2.0 + # chi_loss_eff = mean([3.0, 3.0]) = 3.0 + # coupling = grad_norm_sq / (3.0 * 2.0) + assert "chi_coup" in logged + expected_coupling = logged["grad_norm_squared"] / ( + logged["chi_loss"] * logged["chi_net"] + ) + assert abs(logged["chi_coup"] - expected_coupling) < 1e-5 + + # --- I. Logging behavior --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_logging_only_on_last_microbatch( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Metrics logged once per cycle on the last micro-batch only.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # First micro-batch — should NOT log yet + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + assert model.log.call_count == 0 + + # Second micro-batch (last) — should log + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + assert model.log.call_count > 0 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_effective_batch_size_logged( + self, mock_compute, simple_lightning_module, sample_batch + ): + """effective_batch_size is logged when accumulation is active.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + + assert "effective_batch_size" in logged + # micro_batch_size=4, accumulation_steps=2 → 4*2=8 + assert logged["effective_batch_size"] == 8 + + # --- J. Buffer cleanup --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_buffers_cleared_after_cycle( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Accumulation buffers are reset after a full cycle.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # Run one full cycle + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + # Buffers should be cleared + assert len(model._accum_chi_net) == 0 + assert len(model._accum_chi_loss) == 0 + assert model._accum_grad_train is None + assert model._accum_grad_measure is None + assert model._accum_train_loss == 0.0 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_buffers_dont_leak_between_cycles( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Second cycle doesn't contain data from the first cycle.""" + call_count = [0] + + def side_effect(*args, **kwargs): + call_count[0] += 1 + # Cycle 1: return 10.0, Cycle 2: return 20.0 + val = 10.0 if call_count[0] <= 2 else 20.0 + return { + "batch_grad_norms_network": torch.tensor(val), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + mock_compute.side_effect = side_effect + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # Cycle 1 + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + # Cycle 2 + model._accumulation_count = 0 + model._before_training_step((x, y), 2) + model._accumulation_count = 1 + model._before_training_step((x, y), 3) + + # Get chi_net from cycle 2 (the last logged value) + chi_net_calls = [ + call[0][1] for call in model.log.call_args_list if call[0][0] == "chi_net" + ] + # Cycle 1: mean([10, 10]) = 10, Cycle 2: mean([20, 20]) = 20 + assert len(chi_net_calls) == 2 + assert torch.allclose(chi_net_calls[0], torch.tensor(10.0)) + assert torch.allclose(chi_net_calls[1], torch.tensor(20.0)) + + def test_analysis_does_not_corrupt_training_gradients( + self, simple_lightning_module, sample_batch + ): + """Analysis backward must not affect the gradients seen by the optimizer. + + With accumulation_steps=2, the gradient accumulated into p.grad after + two training_step calls (with analysis enabled) must match the gradient + from two training_step calls with analysis disabled. + """ + torch.manual_seed(0) + x, y = sample_batch + + def run_two_steps(with_analysis): + torch.manual_seed(0) + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + disable_analyzer=not with_analysis, + log_metrics=False, + ) + model.log = Mock() + # Use a real optimizer so p.grad is populated + opt = torch.optim.SGD(model.parameters(), lr=0.0) + model.optimizers = Mock(return_value=opt) + model.manual_backward = lambda loss: loss.backward() + model._trainer = None + + opt.zero_grad() + model._accumulation_count = 0 + model.training_step((x, y), 0) + model.training_step((x, y), 1) + + return [ + p.grad.clone() if p.grad is not None else None + for p in model.model.parameters() + ] + + grads_with = run_two_steps(with_analysis=True) + grads_without = run_two_steps(with_analysis=False) + + for g_with, g_without in zip(grads_with, grads_without): + assert g_with is not None and g_without is not None + assert torch.allclose(g_with, g_without, atol=1e-6), ( + f"Analysis pass corrupted training gradients: " + f"max diff {(g_with - g_without).abs().max().item()}" + ) + + +class TestIndependentMeasureResponse: + """Test the independent measure_dataloader / measure_batch_size path.""" + + # --- A. Parameter validation --- + + def test_infers_measure_micro_from_dataloader(self, simple_lightning_module): + """measure micro-batch size is inferred from the loader's batch_size.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer(simple_lightning_module, measure_dataloader=measure_loader) + assert model._measure_micro_batch_size == 4 + assert model._measure_batch_sizes == [4] + + def test_measure_batch_size_int_normalized_to_list(self, simple_lightning_module): + """A scalar measure_batch_size is normalized to a length-1 list.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=8, + ) + assert model._measure_batch_sizes == [8] + + def test_measure_batch_size_not_divisible_raises(self, simple_lightning_module): + """measure_batch_size above micro, not an exact multiple, raises.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + with pytest.raises(ValueError, match="exact multiple"): + analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=6, + ) + + def test_measure_batch_size_below_micro_allowed(self, simple_lightning_module): + """measure_batch_size below the measure micro size is valid: it runs + as a single direct pass (no accumulation), not an error.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=2, + ) + assert model._measure_batch_sizes == [2] + + def test_measure_dataloader_batch_size_none_raises(self, simple_lightning_module): + """A measure_dataloader with batch_size=None (manual batching) raises.""" + ds = TensorDataset(torch.randn(4, 10), torch.randint(0, 2, (4,))) + loader = DataLoader(ds, batch_size=None) + with pytest.raises(ValueError, match="batch_size"): + analyzer(simple_lightning_module, measure_dataloader=loader) + + def test_measure_params_without_dataloader_raises(self, simple_lightning_module): + """measure_batch_size/measure_subset_seed require measure_dataloader.""" + with pytest.raises( + ValueError, match="only valid together with measure_dataloader" + ): + analyzer(simple_lightning_module, measure_batch_size=8) + + def test_sweep_without_schedule_warns(self, simple_lightning_module): + """A multi-size sweep without an analysis_schedule warns.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + with pytest.warns(UserWarning, match="batch-size sweep"): + analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + ) + + def test_sweep_with_schedule_does_not_warn(self, simple_lightning_module): + """A multi-size sweep with a logarithmic analysis_schedule doesn't warn.""" + from perspic.logger import logarithmic_windows + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + schedule = logarithmic_windows(max_steps=100) + + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + analysis_schedule=schedule, + ) + assert not any("batch-size sweep" in str(w.message) for w in record) + + # --- B. Single-measure logging behavior --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_logs_cross_keys_single_measure( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """A single measure_batch_size logs the existing unsuffixed cross_* keys.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + model._before_training_step((x, y), 0) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + for key in ( + "cross_chi_net", + "cross_chi_loss", + "cross_chi_coup", + "cross_loss", + "cross_grad_dot_product", + "cross_batch_size", + ): + assert key in logged + assert logged["cross_batch_size"] == 8 + assert "cross_chi_net@bs8" not in logged + + @patch.object(SamplewiseCalculatorOpacus, "compute_cross_metrics") + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_measure_chi_is_mean_over_kmeasure( + self, + mock_compute, + mock_compute_cross, + simple_lightning_module, + sample_batch, + ): + """Measure-side chi is averaged over K_measure = S // measure_micro, + not the (different) train accumulation_steps.""" + # 3 train micro-batches (accumulation_steps=3), then 2 measure chunks + # (measure_batch_size=8, measure_micro=4 -> K_measure=2). + values = [ + { + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(4.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(6.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(10.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(20.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ] + mock_compute.side_effect = values + mock_compute_cross.return_value = { + "batch_grad_norms_network": torch.tensor(0.0), + "batch_grad_norms_loss": torch.tensor(0.0), + } + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=5) + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=12, + measure_dataloader=measure_loader, + measure_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + model._accumulation_count = 2 + model._before_training_step((x, y), 2) + + assert mock_compute_cross.call_count == 1 + _, kwargs = mock_compute_cross.call_args + measure_agg = kwargs["sample_wise_metrics_cross"] + # mean([10.0, 20.0]) = 15.0, NOT sum/3 (train K) == 10.0 + assert torch.allclose( + measure_agg["batch_grad_norms_network"], torch.tensor(15.0) + ) + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_measure_grad_dot_reference( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """cross_grad_dot_product matches a hand-computed .""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=4, micro=4, seed=99) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._before_training_step((x, y), 0) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + assert "cross_grad_dot_product" in logged + + # Hand-compute: grad_train = ∇L(x, y); grad_measure = ∇L(x_m, y_m), + # where x_m, y_m is the same (only) batch measure_loader yields + # (shuffle=False), reused via a fresh iterator over the same loader. + x_m, y_m = next(iter(measure_loader)) + + model.model.zero_grad() + loss_t = model.criterion(model.model(x), y) + loss_t.backward() + grad_train = [p.grad.clone() for p in model.model.parameters()] + + model.model.zero_grad() + loss_m = model.criterion(model.model(x_m), y_m) + loss_m.backward() + grad_measure = [p.grad.clone() for p in model.model.parameters()] + model.model.zero_grad() + + expected_dot = sum( + (g1 * g2).sum().item() for g1, g2 in zip(grad_train, grad_measure) + ) + assert abs(logged["cross_grad_dot_product"] - expected_dot) < 1e-4 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_direct_pass_below_micro_reference( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """A measure_batch_size below the measure micro size runs as a + single direct pass (K_measure=1, no accumulation/padding) on exactly + S samples taken from the front of the gathered pool.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=8, micro=8, seed=99) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=4, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._before_training_step((x, y), 0) + + # 1 call for the train self chi + exactly 1 measure-side call + # (K_measure=1: a single direct pass, not padded up to micro=8). + assert mock_compute.call_count == 2 + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + assert logged["cross_batch_size"] == 4 + + # Hand-compute the reference: the pool is built from ceil(4/8)=1 + # micro-batch pull (the loader's only 8-sample batch), sliced to the + # first S_max=4 samples. + x_full, y_full = next(iter(measure_loader)) + x_m, y_m = x_full[:4], y_full[:4] + + model.model.zero_grad() + loss_t = model.criterion(model.model(x), y) + loss_t.backward() + grad_train = [p.grad.clone() for p in model.model.parameters()] + + model.model.zero_grad() + loss_m = model.criterion(model.model(x_m), y_m) + loss_m.backward() + grad_measure = [p.grad.clone() for p in model.model.parameters()] + model.model.zero_grad() + + expected_dot = sum( + (g1 * g2).sum().item() for g1, g2 in zip(grad_train, grad_measure) + ) + assert abs(logged["cross_grad_dot_product"] - expected_dot) < 1e-4 + + def test_measure_backward_does_not_corrupt_training_grad( + self, simple_lightning_module, sample_batch + ): + """Measure-side backward passes inside _measure_response must not + corrupt the (possibly partial) accumulated training gradient.""" + x, y = sample_batch + + def run_two_steps(with_measure): + torch.manual_seed(0) + kwargs = {} + if with_measure: + measure_loader = _make_measure_dataloader(n_samples=4, micro=4, seed=3) + kwargs = {"measure_dataloader": measure_loader} + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=False, + **kwargs, + ) + model.log = Mock() + opt = torch.optim.SGD(model.parameters(), lr=0.0) + model.optimizers = Mock(return_value=opt) + model.manual_backward = lambda loss: loss.backward() + model._trainer = None + + opt.zero_grad() + model._accumulation_count = 0 + model.training_step((x, y), 0) + model.training_step((x, y), 1) + + return [ + p.grad.clone() if p.grad is not None else None + for p in model.model.parameters() + ] + + grads_with = run_two_steps(with_measure=True) + grads_without = run_two_steps(with_measure=False) + + for g_with, g_without in zip(grads_with, grads_without): + assert g_with is not None and g_without is not None + assert torch.allclose(g_with, g_without, atol=1e-6), ( + f"Measure-response backward corrupted training gradients: " + f"max diff {(g_with - g_without).abs().max().item()}" + ) + + # --- C. Subset sampling --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_measure_subset_deterministic_with_seed( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """The same measure_subset_seed yields the same subset (and hence the + same swept cross metric); a different seed yields a different one.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + x, y = sample_batch + + def run(seed): + # Fix the model init seed too: cross_grad_dot_product is a real + # gradient dot product, so it depends on model weights as well + # as on which subset is drawn. + torch.manual_seed(0) + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=11) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[8, 4], + measure_subset_seed=seed, + log_metrics=True, + ) + model.log = Mock() + model._before_training_step((x, y), 0) + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + return logged["cross_grad_dot_product@bs4"] + + val_a = run(seed=123) + val_b = run(seed=123) + val_c = run(seed=456) + + assert val_a == val_b + assert val_a != val_c + + @patch.object(Linearizer, "compute") + def test_measure_subset_size_matches_S( + self, mock_probe, simple_lightning_module, sample_batch + ): + """The number of measure micro-batch chunks processed for each swept + size equals S // measure_micro_batch_size.""" + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=16, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[16, 8], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ) as mock_compute: + model._before_training_step((x, y), 0) + # 1 call for train self chi + (16 // 4 = 4) + (8 // 4 = 2) measure + # chunks. + assert mock_compute.call_count == 1 + 4 + 2 + + @patch.object(Linearizer, "compute") + def test_pool_gather_when_S_max_below_micro( + self, mock_probe, simple_lightning_module, sample_batch + ): + """When every swept size is below the measure micro size, the pool + gather still pulls at least one micro-batch (ceil(S_max/micro)=1) + and slices it down to S_max samples, rather than pulling zero.""" + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=32, micro=16, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[2, 4, 8], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + model, + "_next_measure_micro_batch", + wraps=model._next_measure_micro_batch, + ) as mock_pull, patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ): + model._before_training_step((x, y), 0) + + # S_max=8 < micro=16 -> ceil(8/16)=1 pull, independent of how many + # sizes are swept below it. + assert mock_pull.call_count == 1 + + @patch.object(Linearizer, "compute") + def test_mixed_regime_sweep_chunk_counts( + self, mock_probe, simple_lightning_module, sample_batch + ): + """A sweep spanning sizes below, at, and above the measure micro + size produces the correct chunk count for each: K_measure=1 for + sizes <= micro (a single direct pass), K_measure=S // micro for + sizes above it (accumulated).""" + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=32, micro=8, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[2, 8, 32], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ) as mock_compute: + model._before_training_step((x, y), 0) + # 1 train self chi call + (2 // 2 = 1) + (8 // 8 = 1) + # + (32 // 8 = 4) measure chunks. + assert mock_compute.call_count == 1 + 1 + 1 + 4 + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + for S in (2, 8, 32): + assert logged[f"cross_batch_size@bs{S}"] == S + + # --- D. Sweep logging --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_sweep_logs_suffixed_keys( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """Sweeping multiple measure_batch_size values logs @bs{S}-suffixed + keys instead of the unsuffixed cross_* keys.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + model._before_training_step((x, y), 0) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + for S in (4, 8): + for key in ( + "cross_chi_net", + "cross_chi_loss", + "cross_chi_coup", + "cross_grad_dot_product", + "cross_loss", + "cross_batch_size", + ): + assert f"{key}@bs{S}" in logged + assert "cross_chi_net" not in logged + + # --- E. Persistent iterator --- + + def test_measure_iterator_refills_on_exhaustion( + self, simple_lightning_module, sample_batch + ): + """A measure_dataloader smaller than what's pulled across several + analysis steps cycles instead of raising StopIteration.""" + measure_loader = _make_measure_dataloader(n_samples=4, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ), patch.object( + Linearizer, + "compute", + return_value={"self": (1.0, 0.0, -1.0), "cross": None}, + ): + for i in range(5): + model._before_training_step((x, y), i) + + def test_measure_partial_last_batch_raises( + self, simple_lightning_module, sample_batch + ): + """drop_last=False measure loader with a non-divisible dataset raises + a clear ValueError instead of silently building a short pool.""" + measure_loader = _make_measure_dataloader( + n_samples=6, micro=4, seed=2, drop_last=False + ) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ), patch.object( + Linearizer, + "compute", + return_value={"self": (1.0, 0.0, -1.0), "cross": None}, + ): + with pytest.raises(ValueError, match="drop_last"): + model._before_training_step((x, y), 0)