From 7b9c1128d852e40ea9293322b199a9c33cc85da9 Mon Sep 17 00:00:00 2001 From: jscheunemann Date: Tue, 14 Apr 2026 13:27:58 +0200 Subject: [PATCH 1/4] add gradient accumulation support with micro and effective batch sizes - Introduced `micro_batch_size` and `effective_batch_size` parameters to the analyzer function. - Implemented gradient accumulation logic, allowing for optimization steps to occur after a specified number of micro-batches. - Added validation for batch size parameters to ensure compatibility. - Updated tests to cover new functionality, including parameter validation, optimizer behavior, and logging behavior during accumulation cycles. --- .gitignore | 6 +- examples/batch_accumulation.ipynb | 628 ++++++++++++++++++++++++++ perspic/analyzer.py | 428 +++++++++++++++--- tests/unit/test_analyzer.py | 714 +++++++++++++++++++++++++++++- 4 files changed, 1715 insertions(+), 61 deletions(-) create mode 100644 examples/batch_accumulation.ipynb diff --git a/.gitignore b/.gitignore index fda3e64..c615a79 100644 --- a/.gitignore +++ b/.gitignore @@ -215,4 +215,8 @@ __marimo__/ .vscode/settings.json # Ignore logs -logs/ \ No newline at end of file +logs/ +*/logs/ +*ckpt +.nfs** +examples/cifar-10-batches-py/* diff --git a/examples/batch_accumulation.ipynb b/examples/batch_accumulation.ipynb new file mode 100644 index 0000000..b7881d8 --- /dev/null +++ b/examples/batch_accumulation.ipynb @@ -0,0 +1,628 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "630912d7", + "metadata": {}, + "source": [ + "# Batch Accumulation with Perspic\n", + "\n", + "This notebook demonstrates how to use **gradient accumulation** with the `perspic` analyzer.\n", + "Gradient accumulation lets you simulate a large effective batch size while only fitting a small micro-batch in GPU memory.\n", + "\n", + "We train a small Vision Transformer on CIFAR-10 and compare training **with** and **without** batch accumulation." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e2d7ffb7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 7\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "import pytorch_lightning as pl\n", + "import torch\n", + "import torchvision\n", + "from pytorch_lightning.callbacks import LearningRateMonitor\n", + "from pytorch_lightning.loggers import CSVLogger\n", + "from torch.utils.data import DataLoader, random_split\n", + "from torchvision.datasets import CIFAR10\n", + "from torchvision.models import VisionTransformer\n", + "\n", + "from perspic.analyzer import analyzer\n", + "from examples.models import ClassificationModule\n", + "\n", + "pl.seed_everything(7)\n", + "\n", + "PATH_DATASETS = os.environ.get(\"PATH_DATASETS\", \".\")\n", + "MICRO_BATCH_SIZE = 64\n", + "EFFECTIVE_BATCH_SIZE = 256 # 4x accumulation\n", + "NUM_WORKERS = int(os.cpu_count() / 2)" + ] + }, + { + "cell_type": "markdown", + "id": "d0818a3e", + "metadata": {}, + "source": [ + "## Data Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7ef5f919", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 170M/170M [00:07<00:00, 23.0MB/s] \n" + ] + } + ], + "source": [ + "stats = ((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n", + "train_transform = torchvision.transforms.Compose([\n", + " torchvision.transforms.RandomCrop(32, padding=4),\n", + " torchvision.transforms.RandomHorizontalFlip(),\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize(*stats),\n", + "])\n", + "test_transform = torchvision.transforms.Compose([\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize(*stats),\n", + "])\n", + "\n", + "train_dataset_full = CIFAR10(PATH_DATASETS, train=True, download=True, transform=train_transform)\n", + "val_dataset_full = CIFAR10(PATH_DATASETS, train=True, download=True, transform=test_transform)\n", + "test_set = CIFAR10(PATH_DATASETS, train=False, download=True, transform=test_transform)\n", + "\n", + "generator = torch.Generator().manual_seed(42)\n", + "train_set, _ = random_split(train_dataset_full, [45000, 5000], generator=generator)\n", + "_, val_set = random_split(val_dataset_full, [45000, 5000], generator=generator)\n", + "\n", + "# Use micro-batch size for the DataLoader — accumulation handles the rest\n", + "train_dataloader = DataLoader(train_set, batch_size=MICRO_BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS, drop_last=True)\n", + "val_dataloader = DataLoader(val_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)\n", + "test_dataloader = DataLoader(test_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)" + ] + }, + { + "cell_type": "markdown", + "id": "321b9a72", + "metadata": {}, + "source": [ + "## Model Definition\n", + "\n", + "A small Vision Transformer suitable for CIFAR-10." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "06369a8f", + "metadata": {}, + "outputs": [], + "source": [ + "model_vit = VisionTransformer(\n", + " image_size=32,\n", + " patch_size=8,\n", + " num_layers=2,\n", + " num_heads=4,\n", + " hidden_dim=128,\n", + " mlp_dim=256,\n", + " num_classes=10,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "563f9db6", + "metadata": {}, + "source": [ + "## Training with Batch Accumulation\n", + "\n", + "The key parameters are `micro_batch_size` and `effective_batch_size`. The analyzer will:\n", + "- Zero gradients only at the start of each accumulation cycle\n", + "- Scale the loss by `1 / accumulation_steps` during backward\n", + "- Step the optimizer only after `accumulation_steps = effective_batch_size // micro_batch_size` micro-batches\n", + "- Accumulate analysis metrics across micro-batches and log once per effective step" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "998cb095", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accumulation steps: 4\n", + "Micro-batch size: 64\n", + "Effective batch: 256\n" + ] + } + ], + "source": [ + "vit_accum = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=MICRO_BATCH_SIZE,\n", + " effective_batch_size=EFFECTIVE_BATCH_SIZE,\n", + " model=model_vit,\n", + " lr=0.005,\n", + ")\n", + "\n", + "print(f\"Accumulation steps: {vit_accum.accumulation_steps}\")\n", + "print(f\"Micro-batch size: {MICRO_BATCH_SIZE}\")\n", + "print(f\"Effective batch: {EFFECTIVE_BATCH_SIZE}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "930b6ba5", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "You are using a CUDA device ('NVIDIA GeForce RTX 4090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params | Mode \n", + "----------------------------------------------------\n", + "0 | model | VisionTransformer | 293 K | train\n", + "----------------------------------------------------\n", + "293 K Trainable params\n", + "0 Non-trainable params\n", + "293 K Total params\n", + "1.174 Total estimated model params size (MB)\n", + "32 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c212575e0e8c49fe97c41ce521e73e22", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Sanity Checking: | | 0/? [00:00:0: UserWarning: Full backward hook is firing when gradients are computed with respect to module outputs since no inputs require gradients. See https://docs.pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.register_full_backward_hook for more details.\n", + "/tikhome/jscheunemann/usr/miniconda3/envs/mast/lib/python3.13/site-packages/torch/autograd/graph.py:829: UserWarning: There is a performance drop because we have not yet implemented the batching rule for aten::_scaled_dot_product_efficient_attention_backward. Please file us an issue on GitHub so that we can prioritize its implementation. (Triggered internally at /pytorch/aten/src/ATen/functorch/BatchedFallback.cpp:81.)\n", + " return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "887a7dcd3bdd43b9a9ba1c78500c4c7a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Validation: | | 0/? [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n", + "\n", + "fig, axes = plt.subplots(2, 2, figsize=(12, 8))\n", + "fig.suptitle(f\"ViT with Batch Accumulation ({MICRO_BATCH_SIZE} → {EFFECTIVE_BATCH_SIZE})\", fontsize=14)\n", + "\n", + "# Loss\n", + "ax = axes[0, 0]\n", + "subset = metrics[[\"train_loss\", \"step\"]].dropna()\n", + "ax.plot(subset[\"step\"], subset[\"train_loss\"], alpha=0.3, label=\"Raw\")\n", + "ax.plot(subset[\"step\"], subset[\"train_loss\"].ewm(span=50).mean(), label=\"EMA\")\n", + "ax.set_title(\"Train Loss\")\n", + "ax.set_xlabel(\"Step\")\n", + "ax.legend()\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# chi_net\n", + "ax = axes[0, 1]\n", + "step_col = \"analysis_step\" if \"analysis_step\" in metrics.columns else \"step\"\n", + "subset = metrics[[\"chi_net\", step_col]].dropna()\n", + "if len(subset) > 0:\n", + " ax.plot(subset[step_col], subset[\"chi_net\"], alpha=0.3, label=\"Raw\")\n", + " ax.plot(subset[step_col], subset[\"chi_net\"].ewm(span=50).mean(), label=\"EMA\")\n", + "ax.set_title(\"χ_net\")\n", + "ax.set_xlabel(\"Step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.legend()\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# chi_loss\n", + "ax = axes[1, 0]\n", + "subset = metrics[[\"chi_loss\", step_col]].dropna()\n", + "if len(subset) > 0:\n", + " ax.plot(subset[step_col], subset[\"chi_loss\"], alpha=0.3, label=\"Raw\")\n", + " ax.plot(subset[step_col], subset[\"chi_loss\"].ewm(span=50).mean(), label=\"EMA\")\n", + "ax.set_title(\"χ_loss\")\n", + "ax.set_xlabel(\"Step\")\n", + "ax.legend()\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# Coupling\n", + "ax = axes[1, 1]\n", + "subset = metrics[[\"chi_coup\", step_col]].dropna()\n", + "if len(subset) > 0:\n", + " ax.plot(subset[step_col], subset[\"chi_coup\"], alpha=0.3, label=\"Raw\")\n", + " ax.plot(subset[step_col], subset[\"chi_coup\"].ewm(span=50).mean(), label=\"EMA\")\n", + "ax.set_title(\"Coupling Coefficient (χ_coup)\")\n", + "ax.set_xlabel(\"Step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.legend()\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "0c915b2b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
chi_netchi_losschi_coupstep
0NaNNaNNaN0
1NaNNaNNaN0
2NaNNaNNaN1
3NaNNaNNaN1
4NaNNaNNaN2
...............
7031242.86471613.599990.0000413513
7032NaNNaNNaN3514
7033242.86471613.599990.0000413514
7034NaNNaNNaN3514
7035NaNNaNNaN3515
\n", + "

7036 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " chi_net chi_loss chi_coup step\n", + "0 NaN NaN NaN 0\n", + "1 NaN NaN NaN 0\n", + "2 NaN NaN NaN 1\n", + "3 NaN NaN NaN 1\n", + "4 NaN NaN NaN 2\n", + "... ... ... ... ...\n", + "7031 242.864716 13.59999 0.000041 3513\n", + "7032 NaN NaN NaN 3514\n", + "7033 242.864716 13.59999 0.000041 3514\n", + "7034 NaN NaN NaN 3514\n", + "7035 NaN NaN NaN 3515\n", + "\n", + "[7036 rows x 4 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metrics.keys()\n", + "metrics\n", + "metrics[[\"chi_net\", \"chi_loss\", \"chi_coup\", \"step\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65578b50", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "mast", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/perspic/analyzer.py b/perspic/analyzer.py index 1c842e9..0ffafc4 100644 --- a/perspic/analyzer.py +++ b/perspic/analyzer.py @@ -22,6 +22,8 @@ def analyzer( analyze_every: Optional[int] = None, analysis_schedule: Optional[LogarithmicWindowSchedule] = None, cross_response: bool = False, + micro_batch_size: Optional[int] = None, + effective_batch_size: Optional[int] = None, **model_kwargs, ): """Factory function that wraps a LightningModule with analysis capabilities. @@ -52,11 +54,19 @@ def analyzer( analysis runs only at the scheduled steps. If both analyze_every and analysis_schedule are provided, analysis_schedule takes precedence. - cross_response: If True, enables cross-batch response analysis and assumes - the training batch is a dict with 'train' and 'measure' keys. - Defaults to False. - **model_kwargs: Additional keyword arguments passed to the - LightningModule constructor. + cross_response: If True, enables cross-batch response + analysis and assumes the training batch is a dict + with 'train' and 'measure' keys. Defaults to False. + micro_batch_size: The actual micro-batch size used by the + DataLoader. Required when effective_batch_size is + set. Can be provided alone (no accumulation). + effective_batch_size: The desired simulated batch size + achieved through gradient accumulation. Must be + divisible by micro_batch_size. When set, the optimizer + step is only performed every + (effective_batch_size // micro_batch_size) micro-batches. + **model_kwargs: Additional keyword arguments passed to + the LightningModule constructor. Returns: An initialized Analyzer instance that wraps the provided @@ -117,6 +127,8 @@ def __init__( analyze_every=analyze_every, analysis_schedule=analysis_schedule, cross_response=cross_response, + micro_batch_size=micro_batch_size, + effective_batch_size=effective_batch_size, **model_kwargs, ): super().__init__(**model_kwargs) @@ -178,6 +190,67 @@ def __init__( self.delegate_optimization = False self.automatic_optimization = False # We handle optimization manually + # Gradient accumulation setup + self.micro_batch_size = micro_batch_size + self.effective_batch_size = effective_batch_size + + if ( + effective_batch_size is not None + and micro_batch_size is None + ): + raise ValueError( + "micro_batch_size must be specified when " + "effective_batch_size is set." + ) + + if ( + micro_batch_size is not None + and effective_batch_size is not None + ): + if effective_batch_size < micro_batch_size: + raise ValueError( + f"effective_batch_size " + f"({effective_batch_size}) must be " + f">= micro_batch_size ({micro_batch_size})." + ) + if effective_batch_size % micro_batch_size != 0: + raise ValueError( + f"effective_batch_size " + f"({effective_batch_size}) must be " + f"divisible by micro_batch_size " + f"({micro_batch_size})." + ) + self.accumulation_steps = ( + effective_batch_size // micro_batch_size + ) + else: + self.accumulation_steps = 1 + + if ( + self.accumulation_steps > 1 + and self.delegate_optimization + ): + raise ValueError( + "Gradient accumulation is not supported " + "when the wrapped model uses manual " + "optimization (delegate_optimization=True)." + ) + + self._accumulation_count = 0 + self._optimizer_step_count = 0 + + # Analysis accumulation buffers + self._accum_chi_net = [] + self._accum_chi_loss = [] + self._accum_cross_chi_net = [] + self._accum_cross_chi_loss = [] + self._accum_grad_train = None + self._accum_grad_measure = None + self._accum_train_loss = 0.0 + self._accum_measure_loss = 0.0 + # Track whether analysis is active for this cycle + self._analysis_active = False + # Check if model has criterion attribute if not hasattr(self, "criterion"): raise AttributeError( @@ -223,29 +296,53 @@ def training_step(self, batch, batch_idx): # Initializing manual optimization opt = self.optimizers() - opt.zero_grad() + + # Zero gradients only at start of accumulation cycle + if self._accumulation_count == 0: + opt.zero_grad() # BEFORE logic if not self.disable_analyzer: - self._before_training_step(batch, batch_idx, batch_measure) + self._before_training_step( + batch, batch_idx, batch_measure + ) # Original training step output = super().training_step(batch, batch_idx) if not self.delegate_optimization: - # Backward pass - self.manual_backward(output) - # Optimizer step - opt.step() + # Scale loss for gradient accumulation + scaled_output = ( + output / self.accumulation_steps + ) + self.manual_backward(scaled_output) + + self._accumulation_count += 1 - # Step schedulers with interval='step' - if self._trainer is not None and self.trainer.lr_scheduler_configs: - for config in self.trainer.lr_scheduler_configs: - if config.interval == "step": - config.scheduler.step() + # Step optimizer only at end of accumulation cycle + if ( + self._accumulation_count + >= self.accumulation_steps + ): + opt.step() + self._optimizer_step_count += 1 + self._accumulation_count = 0 + + # Step schedulers with interval='step' + if ( + self._trainer is not None + and self.trainer.lr_scheduler_configs + ): + for config in ( + self.trainer.lr_scheduler_configs + ): + if config.interval == "step": + config.scheduler.step() # AFTER logic if not self.disable_analyzer: - self._after_training_step(batch, batch_idx, output) + self._after_training_step( + batch, batch_idx, output + ) return output @@ -263,6 +360,13 @@ def on_train_epoch_end(self): super().on_train_epoch_end() + @property + def effective_step(self): + """Return the effective optimizer step count.""" + if self.delegate_optimization: + return self.global_step + return self._optimizer_step_count + def _should_analyze(self, step: int) -> bool: """Determine if analysis should run at the given step.""" # If schedule provided, use it @@ -278,8 +382,9 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): """Hook executed before the wrapped training step. Computes analysis metrics including sample-wise gradients and - linearization probes. Only runs if the scheduler determines - this step should be analyzed. + linearization probes. When gradient accumulation is active, + metrics are accumulated across micro-batches and only logged + after the full accumulation cycle. Args: batch: Training batch containing input data and labels. @@ -289,67 +394,64 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): Returns: None """ - # Check if we should run analysis at this step - if not self._should_analyze(self.global_step): + if self.accumulation_steps == 1: + return self._analyze_single_step( + batch, batch_idx, cross_response_batch + ) + else: + return self._analyze_accumulated_step( + batch, batch_idx, cross_response_batch + ) + + def _analyze_single_step(self, batch, batch_idx, cross_response_batch=None): + """Run analysis for a single step (no accumulation).""" + if not self._should_analyze(self.effective_step): return None x, y = batch - - # Get cross-response batch if available + # Get cross-response batch if applicable x2, y2 = None, None if self.cross_response: x2, y2 = cross_response_batch samples_results = {} with BatchStatSnapshot(self.model, x): - # Compute samplewise metrics for the training batch + # Compute sample-wise metrics and self response samples_results["self"] = self.sample_calc.compute( - self.model, - self.criterion, - x, - y, + self.model, self.criterion, x, y, ) - # Compute samplewise metrics for the cross batch if available + # Compute sample-wise metrics and cross response if applicable if x2 is not None and y2 is not None: - probe_results_cross_preliminary = self.sample_calc.compute( - self.model, - self.criterion, - x2, - y2, + cross_preliminary = self.sample_calc.compute( + self.model, self.criterion, x2, y2, ) - samples_results["cross"] = self.sample_calc.compute_cross_metrics( - sample_wise_metrics_self=samples_results["self"], - sample_wise_metrics_cross=probe_results_cross_preliminary, + samples_results["cross"] = ( + self.sample_calc.compute_cross_metrics( + sample_wise_metrics_self=samples_results["self"], + sample_wise_metrics_cross=cross_preliminary, + ) ) - - # Linearizer compute + # Linearizer probe probe_results = self.linearizer.compute( - model=self.model, - criterion=self.criterion, - x1=x, - y1=y, - x2=x2, - y2=y2, + model=self.model, criterion=self.criterion, + x1=x, y1=y, x2=x2, y2=y2, ) # Get "self" result for coupling calculation loss_self, _, delta_loss_self = probe_results["self"] - - # Compute coupling value (using self response) chi_coup = self.coupling_calc.calculate( delta_loss=delta_loss_self, chi_loss=samples_results["self"]["batch_grad_norms_loss"], chi_net=samples_results["self"]["batch_grad_norms_network"], ) + chi_coup_cross = None if self.cross_response and "cross" in probe_results: - # Optionally, compute coupling for cross response as well - loss_cross, _, delta_loss_cross = probe_results["cross"] + _, _, delta_loss_cross = probe_results["cross"] chi_coup_cross = self.coupling_calc.calculate( delta_loss=delta_loss_cross, chi_loss=samples_results["cross"]["batch_grad_norms_loss"], chi_net=samples_results["cross"]["batch_grad_norms_network"], ) - # Log results with fixed metric names if self.log_metrics: self._log_analysis_results( @@ -359,7 +461,6 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): chi_coup=chi_coup, batch_size=x.shape[0], ) - # Log cross response if available if "cross" in samples_results and samples_results["cross"] is not None: self._log_analysis_results( @@ -369,11 +470,10 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): chi_coup=chi_coup_cross, batch_size=x2.shape[0] if x2 is not None else 0, ) - # Log window tracking info if using logarithmic schedule if self.analysis_schedule is not None: window_info = self.analysis_schedule.get_window_info( - self.global_step + self.effective_step ) if window_info is not None: self.log("window_id", window_info["window_id"]) @@ -382,6 +482,219 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): return None + def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None): + """Run analysis with gradient accumulation across micro-batches. + + On each micro-batch: accumulate sample-wise metrics and linearizer + gradients. On the last micro-batch of the cycle: finalize, log, clear. + """ + # On first micro-batch of cycle, decide whether to analyze + if self._accumulation_count == 0: + self._analysis_active = self._should_analyze( + self.effective_step + ) + if self._analysis_active: + self._clear_accumulation_buffers() + + if not self._analysis_active: + return None + + x, y = batch + x2, y2 = None, None + if self.cross_response: + x2, y2 = cross_response_batch + + with BatchStatSnapshot(self.model, x): + # Accumulate sample-wise metrics + self_metrics = self.sample_calc.compute( + self.model, self.criterion, x, y, + ) + self._accum_chi_net.append( + self_metrics["batch_grad_norms_network"] + ) + self._accum_chi_loss.append( + self_metrics["batch_grad_norms_loss"] + ) + + if x2 is not None and y2 is not None: + cross_preliminary = self.sample_calc.compute( + self.model, self.criterion, x2, y2, + ) + cross_metrics = self.sample_calc.compute_cross_metrics( + sample_wise_metrics_self=self_metrics, + sample_wise_metrics_cross=cross_preliminary, + ) + self._accum_cross_chi_net.append( + cross_metrics["batch_grad_norms_network"] + ) + self._accum_cross_chi_loss.append( + cross_metrics["batch_grad_norms_loss"] + ) + + # Accumulate linearizer gradients (train side) + self._accumulate_linearizer_grads( + x, y, is_train=True + ) + # Accumulate linearizer gradients (measure side) + # TODO: How would a measure batchsize different to the effective batch size work here? + # !!! We would need to accumulate separately and then combine at the end. + if x2 is not None and y2 is not None: + self._accumulate_linearizer_grads( + x2, y2, is_train=False + ) + + # On last micro-batch: finalize and log + is_last = ( + self._accumulation_count == self.accumulation_steps - 1 + ) + if is_last: + self._finalize_accumulated_analysis(x, x2) + + return None + + def _accumulate_linearizer_grads(self, x, y, is_train=True): + """Forward+backward on a micro-batch and add grads to accumulator.""" + self.model.zero_grad() + loss = self.criterion(self.model(x), y) + loss.backward() + + loss_val = loss.detach().item() + if is_train: + self._accum_train_loss += loss_val + if self._accum_grad_train is None: + self._accum_grad_train = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + else: + for acc, p in zip( + self._accum_grad_train, self.model.parameters() + ): + if acc is not None and p.grad is not None: + acc.add_(p.grad) + else: + self._accum_measure_loss += loss_val + if self._accum_grad_measure is None: + self._accum_grad_measure = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + else: + for acc, p in zip( + self._accum_grad_measure, self.model.parameters() + ): + if acc is not None and p.grad is not None: + acc.add_(p.grad) + + self.model.zero_grad() + + def _finalize_accumulated_analysis(self, x, x2): + """Combine accumulated metrics and log results.""" + K = self.accumulation_steps + B = x.shape[0] + + # Combine sample-wise metrics + chi_net_eff = sum(self._accum_chi_net) / K + chi_loss_eff = K * sum(self._accum_chi_loss) + + samples_result_self = { + "batch_grad_norms_network": chi_net_eff, + "batch_grad_norms_loss": chi_loss_eff, + } + + # Compute self linearizer result from accumulated grads + grad_norm_sq = sum( + (g ** 2).sum().item() + for g in self._accum_grad_train if g is not None + ) / (K ** 2) + + avg_train_loss = self._accum_train_loss / K + delta_loss_self = -grad_norm_sq + probe_result_self = ( + avg_train_loss, + avg_train_loss + delta_loss_self, + delta_loss_self, + ) + + chi_coup = self.coupling_calc.calculate( + delta_loss=delta_loss_self, + chi_loss=chi_loss_eff, + chi_net=chi_net_eff, + ) + + # Cross response + samples_result_cross = None + probe_result_cross = None + chi_coup_cross = None + if self._accum_grad_measure is not None: + chi_net_cross_eff = sum(self._accum_cross_chi_net) / K + chi_loss_cross_eff = K * sum(self._accum_cross_chi_loss) + samples_result_cross = { + "batch_grad_norms_network": chi_net_cross_eff, + "batch_grad_norms_loss": chi_loss_cross_eff, + } + + cross_dot = sum( + (g1 * g2).sum().item() + for g1, g2 in zip( + self._accum_grad_train, + self._accum_grad_measure, + ) + if g1 is not None and g2 is not None + ) / (K ** 2) + + avg_measure_loss = self._accum_measure_loss / K + delta_loss_cross = -cross_dot + probe_result_cross = ( + avg_measure_loss, + avg_measure_loss + delta_loss_cross, + delta_loss_cross, + ) + chi_coup_cross = self.coupling_calc.calculate( + delta_loss=delta_loss_cross, + chi_loss=chi_loss_cross_eff, + chi_net=chi_net_cross_eff, + ) + + # Log results + if self.log_metrics: + self._log_analysis_results( + prefix="", + samples_result=samples_result_self, + probe_result=probe_result_self, + chi_coup=chi_coup, + batch_size=B, + ) + if samples_result_cross is not None: + self._log_analysis_results( + prefix="cross_", + samples_result=samples_result_cross, + probe_result=probe_result_cross, + chi_coup=chi_coup_cross, + batch_size=x2.shape[0] if x2 is not None else 0, + ) + if self.analysis_schedule is not None: + window_info = self.analysis_schedule.get_window_info( + self.effective_step + ) + if window_info is not None: + self.log("window_id", window_info["window_id"]) + self.log("window_center", window_info["window_center"]) + self.log("window_width", window_info["window_width"]) + + self._clear_accumulation_buffers() + + def _clear_accumulation_buffers(self): + """Reset all accumulation buffers.""" + self._accum_chi_net.clear() + self._accum_chi_loss.clear() + self._accum_cross_chi_net.clear() + self._accum_cross_chi_loss.clear() + self._accum_grad_train = None + self._accum_grad_measure = None + self._accum_train_loss = 0.0 + self._accum_measure_loss = 0.0 + def _log_analysis_results( self, prefix: str, @@ -402,10 +715,15 @@ def _log_analysis_results( self.log(f"{prefix}chi_coup", chi_coup) self.log(f"{prefix}batch_size", batch_size) + if self.accumulation_steps > 1: + self.log( + f"{prefix}effective_batch_size", + batch_size * self.accumulation_steps, + ) # Only log analysis_step once (usually with empty prefix) if prefix == "": - self.log("analysis_step", self.global_step) + self.log("analysis_step", self.effective_step) # Log probe results (linearization) if probe_result is not None: @@ -443,5 +761,7 @@ def _after_training_step(self, batch, batch_idx, output): analyze_every=analyze_every, analysis_schedule=analysis_schedule, cross_response=cross_response, + micro_batch_size=micro_batch_size, + effective_batch_size=effective_batch_size, **model_kwargs, ) diff --git a/tests/unit/test_analyzer.py b/tests/unit/test_analyzer.py index 6046012..4dff146 100644 --- a/tests/unit/test_analyzer.py +++ b/tests/unit/test_analyzer.py @@ -717,10 +717,7 @@ def test_before_hook_skipped_when_not_scheduled( ): """Test _before_training_step skips analysis when not scheduled.""" model = analyzer(simple_lightning_module, analyze_every=10) - model._global_step = 5 # Not a multiple of 10 - - # Mock global_step property - type(model).global_step = property(lambda self: 5) + model._optimizer_step_count = 5 # Not a multiple of 10 model._before_training_step(sample_batch, 0) @@ -753,7 +750,6 @@ def test_logs_window_info_with_schedule( simple_lightning_module, analysis_schedule=schedule, log_metrics=True ) model.log = Mock() - type(model).global_step = property(lambda self: 0) model._before_training_step(sample_batch, 0) @@ -779,7 +775,6 @@ def test_no_window_info_without_schedule( model = analyzer(simple_lightning_module, log_metrics=True) model.log = Mock() - type(model).global_step = property(lambda self: 0) model._before_training_step(sample_batch, 0) @@ -787,3 +782,710 @@ def test_no_window_info_without_schedule( assert "window_id" not in logged_names assert "window_center" not in logged_names assert "window_width" not in logged_names + + +class TestGradientAccumulation: + """Test gradient accumulation functionality.""" + # The tests are categorized into sections A-J for clarity. + + # --- A. Parameter validation --- + + def test_accumulation_steps_default( + self, simple_lightning_module + ): + """No params → accumulation_steps=1.""" + model = analyzer(simple_lightning_module) + assert model.accumulation_steps == 1 + + def test_batch_size_only_no_accumulation( + self, simple_lightning_module + ): + """micro_batch_size alone → no accumulation, value stored.""" + model = analyzer( + simple_lightning_module, micro_batch_size=8 + ) + assert model.accumulation_steps == 1 + assert model.micro_batch_size == 8 + assert model.effective_batch_size is None + + def test_accumulation_steps_computed( + self, simple_lightning_module + ): + """micro=8, effective=32 → accumulation_steps=4.""" + model = analyzer( + simple_lightning_module, + micro_batch_size=8, + effective_batch_size=32, + ) + assert model.accumulation_steps == 4 + + def test_effective_without_micro_batch_raises( + self, simple_lightning_module + ): + """effective_batch_size alone → ValueError.""" + with pytest.raises( + ValueError, match="micro_batch_size must be specified" + ): + analyzer( + simple_lightning_module, + effective_batch_size=32, + ) + + def test_effective_less_than_micro_batch_raises( + self, simple_lightning_module + ): + """effective=8, micro=32 → ValueError.""" + with pytest.raises( + ValueError, match="must be >= micro_batch_size" + ): + analyzer( + simple_lightning_module, + micro_batch_size=32, + effective_batch_size=8, + ) + + def test_not_divisible_raises( + self, simple_lightning_module + ): + """effective=30, micro=8 → ValueError (not divisible).""" + with pytest.raises( + ValueError, match="must be divisible" + ): + analyzer( + simple_lightning_module, + micro_batch_size=8, + effective_batch_size=30, + ) + + def test_accumulation_with_delegate_raises( + self, manual_optimization_module + ): + """Accumulation + manual optimization module → ValueError.""" + with pytest.raises( + ValueError, + match="Gradient accumulation is not supported", + ): + with pytest.warns( + UserWarning, match="manual optimization" + ): + analyzer( + manual_optimization_module, + micro_batch_size=8, + effective_batch_size=32, + ) + + # --- B. Optimizer behavior --- + + def test_zero_grad_once_per_cycle( + self, simple_lightning_module, sample_batch + ): + """4 micro-steps, accum=4: zero_grad called exactly 1x.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.zero_grad.call_count == 1 + + def test_step_once_per_cycle( + self, simple_lightning_module, sample_batch + ): + """4 micro-steps, accum=4: opt.step() called exactly 1x.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.step.call_count == 1 + + def test_step_not_called_mid_cycle( + self, simple_lightning_module, sample_batch + ): + """3 of 4 micro-steps done: opt.step() never called.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(3): + model.training_step((x, y), i) + + mock_opt.step.assert_not_called() + + def test_loss_scaled_for_backward( + self, simple_lightning_module, sample_batch + ): + """manual_backward receives loss / accumulation_steps.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + output = model.training_step((x, y), 0) + + backward_arg = model.manual_backward.call_args[0][0] + expected = output / 4 + assert torch.allclose(backward_arg, expected) + + def test_unscaled_loss_returned( + self, simple_lightning_module, sample_batch + ): + """training_step returns the original unscaled loss.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=16, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + output_accum = model.training_step((x, y), 0) + assert isinstance(output_accum, torch.Tensor) + + def test_two_full_cycles( + self, simple_lightning_module, sample_batch + ): + """4 steps, accum=2: zero_grad 2x, opt.step() 2x.""" + model = analyzer( + simple_lightning_module, + disable_analyzer=True, + micro_batch_size=4, + effective_batch_size=8, + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.zero_grad.call_count == 2 + assert mock_opt.step.call_count == 2 + + # --- C. Backwards compatibility --- + + def test_no_accumulation_backwards_compatible( + self, simple_lightning_module, sample_batch + ): + """No accum params: every call does zero_grad + step.""" + model = analyzer( + simple_lightning_module, disable_analyzer=True + ) + mock_opt = Mock(zero_grad=Mock(), step=Mock()) + model.optimizers = Mock(return_value=mock_opt) + model.manual_backward = Mock() + + x, y = sample_batch + for i in range(4): + model.training_step((x, y), i) + + assert mock_opt.zero_grad.call_count == 4 + assert mock_opt.step.call_count == 4 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_single_step_path_unchanged( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """Without accumulation, _analyze_single_step produces same results.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.5), + "batch_grad_norms_loss": torch.tensor(2.5), + } + mock_probe.return_value = { + "self": (1.0, 0.0, -1.0), + "cross": None, + } + + model = analyzer(simple_lightning_module, log_metrics=True) + model.log = Mock() + x, y = sample_batch + + model._before_training_step((x, y), 0) + + logged = { + call[0][0]: call[0][1] + for call in model.log.call_args_list + } + assert torch.allclose(logged["chi_net"], torch.tensor(1.5)) + assert torch.allclose(logged["chi_loss"], torch.tensor(2.5)) + assert logged["loss"] == 1.0 + assert logged["grad_norm_squared"] == 1.0 + assert logged["batch_size"] == 4 + + # --- D. Effective step --- + + def test_effective_step_with_accumulation( + self, simple_lightning_module + ): + """_optimizer_step_count=2 → effective_step=2.""" + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=16, + ) + model._optimizer_step_count = 2 + assert model.effective_step == 2 + + model._optimizer_step_count = 0 + assert model.effective_step == 0 + + def test_effective_step_without_accumulation( + self, simple_lightning_module + ): + """_optimizer_step_count=42 → effective_step=42.""" + model = analyzer(simple_lightning_module) + model._optimizer_step_count = 42 + assert model.effective_step == 42 + + # --- E. Analysis scheduling with accumulation --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_analysis_uses_effective_step_for_scheduling( + self, mock_compute, simple_lightning_module, sample_batch + ): + """_should_analyze uses effective_step, activates on first micro-batch.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + analyze_every=2, + micro_batch_size=4, + effective_batch_size=16, + ) + model.log = Mock() + x, y = sample_batch + + # effective_step=0, analyze_every=2 → 0 % 2 == 0 → analyze + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + assert model._analysis_active is True + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_analysis_skipped_when_schedule_says_no( + self, mock_compute, simple_lightning_module, sample_batch + ): + """When _should_analyze returns False, no accumulation or logging happens.""" + model = analyzer( + simple_lightning_module, + analyze_every=10, + micro_batch_size=4, + effective_batch_size=8, + ) + model.log = Mock() + x, y = sample_batch + + # effective_step=1, analyze_every=10 → 1 % 10 != 0 → skip + model._optimizer_step_count = 1 + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + + assert model._analysis_active is False + mock_compute.assert_not_called() + model.log.assert_not_called() + + # Second micro-batch also skipped (flag persists) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + mock_compute.assert_not_called() + model.log.assert_not_called() + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_analysis_step_logs_effective_step( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Logged analysis_step equals effective_step, not global_step.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # _optimizer_step_count=3 → effective_step=3 + model._optimizer_step_count = 3 + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = { + call[0][0]: call[0][1] + for call in model.log.call_args_list + } + assert logged["analysis_step"] == 3 + + # --- F. Sample-wise metric accumulation --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_accumulated_chi_net_is_mean( + self, mock_compute, simple_lightning_module, sample_batch + ): + """chi_net_eff = mean of per-micro-batch chi_net values.""" + mock_compute.side_effect = [ + { + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(3.0), + }, + { + "batch_grad_norms_network": torch.tensor(4.0), + "batch_grad_norms_loss": torch.tensor(5.0), + }, + ] + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = { + call[0][0]: call[0][1] + for call in model.log.call_args_list + } + + # chi_net_eff = mean([2.0, 4.0]) = 3.0 + assert torch.allclose(logged["chi_net"], torch.tensor(3.0)) + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_accumulated_chi_loss_is_k_times_sum( + self, mock_compute, simple_lightning_module, sample_batch + ): + """chi_loss_eff = K * sum of per-micro-batch chi_loss values.""" + mock_compute.side_effect = [ + { + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(3.0), + }, + { + "batch_grad_norms_network": torch.tensor(4.0), + "batch_grad_norms_loss": torch.tensor(5.0), + }, + ] + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = { + call[0][0]: call[0][1] + for call in model.log.call_args_list + } + + # chi_loss_eff = K * sum([3.0, 5.0]) = 2 * 8.0 = 16.0 + assert torch.allclose(logged["chi_loss"], torch.tensor(16.0)) + + # --- G. Linearizer gradient accumulation --- + + def test_linearizer_accumulated_grad_norm( + self, simple_lightning_module, sample_batch + ): + """grad_norm_squared = ||Σ∇L_k||² / K² from accumulated grads.""" + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # Run a full accumulation cycle (K=2) + with patch.object( + SamplewiseCalculatorOpacus, "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ): + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = { + call[0][0]: call[0][1] + for call in model.log.call_args_list + } + + # grad_norm_squared should be a positive float + assert "grad_norm_squared" in logged + assert logged["grad_norm_squared"] > 0 + + # Verify manually: compute the expected value + # Do two forward+backward passes, sum grads, compute ||sum||²/K² + model.model.zero_grad() + loss0 = model.criterion(model.model(x), y) + loss0.backward() + grads_0 = [ + p.grad.clone() for p in model.model.parameters() + if p.grad is not None + ] + + model.model.zero_grad() + loss1 = model.criterion(model.model(x), y) + loss1.backward() + grads_1 = [ + p.grad.clone() for p in model.model.parameters() + if p.grad is not None + ] + model.model.zero_grad() + + expected_norm_sq = sum( + ((g0 + g1) ** 2).sum().item() + for g0, g1 in zip(grads_0, grads_1) + ) / 4 # K² = 2² = 4 + + assert abs(logged["grad_norm_squared"] - expected_norm_sq) < 1e-4 + + # --- H. Coupling with accumulated values --- + + def test_coupling_from_accumulated_values( + self, simple_lightning_module, sample_batch + ): + """coupling = grad_norm_sq / (chi_loss_eff * chi_net_eff).""" + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(3.0), + }, + ): + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = { + call[0][0]: call[0][1] + for call in model.log.call_args_list + } + + # chi_net_eff = mean([2.0, 2.0]) = 2.0 + # chi_loss_eff = 2 * sum([3.0, 3.0]) = 12.0 + # coupling = grad_norm_sq / (12.0 * 2.0) + assert "chi_coup" in logged + expected_coupling = ( + logged["grad_norm_squared"] / (logged["chi_loss"] * logged["chi_net"]) + ) + assert abs(logged["chi_coup"] - expected_coupling) < 1e-5 + + # --- I. Logging behavior --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_logging_only_on_last_microbatch( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Metrics logged once per cycle on the last micro-batch only.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # First micro-batch — should NOT log yet + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + assert model.log.call_count == 0 + + # Second micro-batch (last) — should log + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + assert model.log.call_count > 0 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_effective_batch_size_logged( + self, mock_compute, simple_lightning_module, sample_batch + ): + """effective_batch_size is logged when accumulation is active.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + logged = { + call[0][0]: call[0][1] + for call in model.log.call_args_list + } + + assert "effective_batch_size" in logged + # micro_batch_size=4, accumulation_steps=2 → 4*2=8 + assert logged["effective_batch_size"] == 8 + + # --- J. Buffer cleanup --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_buffers_cleared_after_cycle( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Accumulation buffers are reset after a full cycle.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # Run one full cycle + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + # Buffers should be cleared + assert len(model._accum_chi_net) == 0 + assert len(model._accum_chi_loss) == 0 + assert model._accum_grad_train is None + assert model._accum_grad_measure is None + assert model._accum_train_loss == 0.0 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_buffers_dont_leak_between_cycles( + self, mock_compute, simple_lightning_module, sample_batch + ): + """Second cycle doesn't contain data from the first cycle.""" + call_count = [0] + + def side_effect(*args, **kwargs): + call_count[0] += 1 + # Cycle 1: return 10.0, Cycle 2: return 20.0 + val = 10.0 if call_count[0] <= 2 else 20.0 + return { + "batch_grad_norms_network": torch.tensor(val), + "batch_grad_norms_loss": torch.tensor(1.0), + } + + mock_compute.side_effect = side_effect + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + # Cycle 1 + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + + # Cycle 2 + model._accumulation_count = 0 + model._before_training_step((x, y), 2) + model._accumulation_count = 1 + model._before_training_step((x, y), 3) + + # Get chi_net from cycle 2 (the last logged value) + chi_net_calls = [ + call[0][1] + for call in model.log.call_args_list + if call[0][0] == "chi_net" + ] + # Cycle 1: mean([10, 10]) = 10, Cycle 2: mean([20, 20]) = 20 + assert len(chi_net_calls) == 2 + assert torch.allclose(chi_net_calls[0], torch.tensor(10.0)) + assert torch.allclose(chi_net_calls[1], torch.tensor(20.0)) From 64442a999a8661e72396f5fea63d3aeb28443a63 Mon Sep 17 00:00:00 2001 From: jscheunemann Date: Tue, 14 Jul 2026 16:58:11 +0200 Subject: [PATCH 2/4] Enhance gradient accumulation: track micro-batch losses and ensure analysis does not corrupt training gradients --- perspic/analyzer.py | 51 ++++++++++++++++++++++++---- tests/unit/test_analyzer.py | 67 +++++++++++++++++++++++++++++++++---- 2 files changed, 105 insertions(+), 13 deletions(-) diff --git a/perspic/analyzer.py b/perspic/analyzer.py index 0ffafc4..bbc76d7 100644 --- a/perspic/analyzer.py +++ b/perspic/analyzer.py @@ -248,6 +248,7 @@ def __init__( self._accum_grad_measure = None self._accum_train_loss = 0.0 self._accum_measure_loss = 0.0 + self._accum_step_losses = [] # micro-batch losses for per-opt-step logging # Track whether analysis is active for this cycle self._analysis_active = False @@ -318,6 +319,20 @@ def training_step(self, batch, batch_idx): self._accumulation_count += 1 + if self.accumulation_steps > 1: + self._accum_step_losses.append(output.detach()) + # Tag every micro-batch with its cycle's effective_step so + # groupby(effective_step).mean() in the plot averages exactly + # the K micro-batches of that cycle (no Lightning forward-fill + # ambiguity). opt.step() has not fired yet, so +1 gives the + # current cycle number. + self.log( + "effective_step", + float(self._optimizer_step_count + 1), + on_step=True, + on_epoch=False, + ) + # Step optimizer only at end of accumulation cycle if ( self._accumulation_count @@ -327,6 +342,9 @@ def training_step(self, batch, batch_idx): self._optimizer_step_count += 1 self._accumulation_count = 0 + if self.accumulation_steps > 1 and self._accum_step_losses: + self._accum_step_losses.clear() + # Step schedulers with interval='step' if ( self._trainer is not None @@ -504,6 +522,15 @@ def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None) if self.cross_response: x2, y2 = cross_response_batch + # Save training grads before any analysis backward/zero_grad calls. + # sample_calc.compute and _accumulate_linearizer_grads both call + # model.zero_grad() internally; restoring here ensures the training + # accumulation loop sees unmodified gradients after this hook. + saved_grads = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + with BatchStatSnapshot(self.model, x): # Accumulate sample-wise metrics self_metrics = self.sample_calc.compute( @@ -543,6 +570,10 @@ def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None) x2, y2, is_train=False ) + # Restore training grads clobbered by analysis backward passes + for p, s in zip(self.model.parameters(), saved_grads): + p.grad = s + # On last micro-batch: finalize and log is_last = ( self._accumulation_count == self.accumulation_steps - 1 @@ -553,7 +584,12 @@ def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None) return None def _accumulate_linearizer_grads(self, x, y, is_train=True): - """Forward+backward on a micro-batch and add grads to accumulator.""" + """Forward+backward on a micro-batch and add grads to accumulator. + + Must be called inside a BatchStatSnapshot context (caller's + responsibility). Training grads are saved/restored by the caller + (_analyze_accumulated_step) around the full analysis block. + """ self.model.zero_grad() loss = self.criterion(self.model(x), y) loss.backward() @@ -586,16 +622,18 @@ def _accumulate_linearizer_grads(self, x, y, is_train=True): if acc is not None and p.grad is not None: acc.add_(p.grad) - self.model.zero_grad() - def _finalize_accumulated_analysis(self, x, x2): """Combine accumulated metrics and log results.""" K = self.accumulation_steps B = x.shape[0] - # Combine sample-wise metrics + # Combine sample-wise metrics. + # Both chi_net and chi_loss are computed with normalize=True, which + # makes them extensive in the batch size via a 1/B or *B factor + # derived from mean-reduced loss. For an effective batch N=K*B, the + # correct aggregate for both quantities is the mean across micro-batches. chi_net_eff = sum(self._accum_chi_net) / K - chi_loss_eff = K * sum(self._accum_chi_loss) + chi_loss_eff = sum(self._accum_chi_loss) / K samples_result_self = { "batch_grad_norms_network": chi_net_eff, @@ -628,7 +666,7 @@ def _finalize_accumulated_analysis(self, x, x2): chi_coup_cross = None if self._accum_grad_measure is not None: chi_net_cross_eff = sum(self._accum_cross_chi_net) / K - chi_loss_cross_eff = K * sum(self._accum_cross_chi_loss) + chi_loss_cross_eff = sum(self._accum_cross_chi_loss) / K samples_result_cross = { "batch_grad_norms_network": chi_net_cross_eff, "batch_grad_norms_loss": chi_loss_cross_eff, @@ -694,6 +732,7 @@ def _clear_accumulation_buffers(self): self._accum_grad_measure = None self._accum_train_loss = 0.0 self._accum_measure_loss = 0.0 + self._accum_step_losses.clear() def _log_analysis_results( self, diff --git a/tests/unit/test_analyzer.py b/tests/unit/test_analyzer.py index 4dff146..3266b43 100644 --- a/tests/unit/test_analyzer.py +++ b/tests/unit/test_analyzer.py @@ -1201,10 +1201,15 @@ def test_accumulated_chi_net_is_mean( assert torch.allclose(logged["chi_net"], torch.tensor(3.0)) @patch.object(SamplewiseCalculatorOpacus, "compute") - def test_accumulated_chi_loss_is_k_times_sum( + def test_accumulated_chi_loss_is_mean( self, mock_compute, simple_lightning_module, sample_batch ): - """chi_loss_eff = K * sum of per-micro-batch chi_loss values.""" + """chi_loss_eff = mean of per-micro-batch chi_loss values. + + chi_loss is computed with normalize=True against a mean-reduced loss, + so the correct aggregation for an effective batch is the mean across + micro-batches (same as chi_net), not K * sum. + """ mock_compute.side_effect = [ { "batch_grad_norms_network": torch.tensor(2.0), @@ -1235,8 +1240,8 @@ def test_accumulated_chi_loss_is_k_times_sum( for call in model.log.call_args_list } - # chi_loss_eff = K * sum([3.0, 5.0]) = 2 * 8.0 = 16.0 - assert torch.allclose(logged["chi_loss"], torch.tensor(16.0)) + # chi_loss_eff = mean([3.0, 5.0]) = 4.0 + assert torch.allclose(logged["chi_loss"], torch.tensor(4.0)) # --- G. Linearizer gradient accumulation --- @@ -1333,9 +1338,9 @@ def test_coupling_from_accumulated_values( for call in model.log.call_args_list } - # chi_net_eff = mean([2.0, 2.0]) = 2.0 - # chi_loss_eff = 2 * sum([3.0, 3.0]) = 12.0 - # coupling = grad_norm_sq / (12.0 * 2.0) + # chi_net_eff = mean([2.0, 2.0]) = 2.0 + # chi_loss_eff = mean([3.0, 3.0]) = 3.0 + # coupling = grad_norm_sq / (3.0 * 2.0) assert "chi_coup" in logged expected_coupling = ( logged["grad_norm_squared"] / (logged["chi_loss"] * logged["chi_net"]) @@ -1489,3 +1494,51 @@ def side_effect(*args, **kwargs): assert len(chi_net_calls) == 2 assert torch.allclose(chi_net_calls[0], torch.tensor(10.0)) assert torch.allclose(chi_net_calls[1], torch.tensor(20.0)) + + def test_analysis_does_not_corrupt_training_gradients( + self, simple_lightning_module, sample_batch + ): + """Analysis backward must not affect the gradients seen by the optimizer. + + With accumulation_steps=2, the gradient accumulated into p.grad after + two training_step calls (with analysis enabled) must match the gradient + from two training_step calls with analysis disabled. + """ + torch.manual_seed(0) + x, y = sample_batch + + def run_two_steps(with_analysis): + torch.manual_seed(0) + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + disable_analyzer=not with_analysis, + log_metrics=False, + ) + model.log = Mock() + # Use a real optimizer so p.grad is populated + opt = torch.optim.SGD(model.parameters(), lr=0.0) + model.optimizers = Mock(return_value=opt) + model.manual_backward = lambda loss: loss.backward() + model._trainer = None + + opt.zero_grad() + model._accumulation_count = 0 + model.training_step((x, y), 0) + model.training_step((x, y), 1) + + return [ + p.grad.clone() if p.grad is not None else None + for p in model.model.parameters() + ] + + grads_with = run_two_steps(with_analysis=True) + grads_without = run_two_steps(with_analysis=False) + + for g_with, g_without in zip(grads_with, grads_without): + assert g_with is not None and g_without is not None + assert torch.allclose(g_with, g_without, atol=1e-6), ( + f"Analysis pass corrupted training gradients: " + f"max diff {(g_with - g_without).abs().max().item()}" + ) From 3c3bd289a170f189268134f469da8f4e457cfc3d Mon Sep 17 00:00:00 2001 From: jscheunemann Date: Wed, 15 Jul 2026 18:14:53 +0200 Subject: [PATCH 3/4] Add implementation notes for batch accumulation, fix chi_loss aggregation error, and update notebook metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Created `batch_accumulation_notes.md` detailing the gradient accumulation process, issues encountered, and their resolutions. - Corrected the aggregation formula for `chi_loss` to use mean instead of sum, addressing a K² error in previous calculations. - Added a new image file `first_wrong_test_batch_accumulation.png` to illustrate the initial error in batch accumulation. --- CLAUDE.md | 74 + examples/batch_accumulation.ipynb | 1564 ++++++++++++++++---- examples/batch_accumulation_notes.md | 243 +++ examples/batch_size_scaling_analysis.ipynb | 6 +- first_wrong_test_batch_accumulation.png | Bin 0 -> 180740 bytes 5 files changed, 1594 insertions(+), 293 deletions(-) create mode 100644 CLAUDE.md create mode 100644 examples/batch_accumulation_notes.md create mode 100644 first_wrong_test_batch_accumulation.png diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..75f0876 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,74 @@ +# perspic + +A tool to study neural network training dynamics. Package `perspic`, v0.0.1, authors Konstantin Nikolaou and Jonas Scheunemann, requires Python >=3.11. Built on PyTorch and PyTorch Lightning. + +## Overview + +perspic wraps a user's `pytorch_lightning.LightningModule` with an `analyzer()` factory, transparently instrumenting training to compute per-sample gradient-norm and linear-response metrics — quantities used to study training dynamics (gradient coupling, sensitivity, collective variables) — without requiring the user to modify their model code. The wrapped module trains normally under a standard `pytorch_lightning.Trainer`; the analysis runs and logs metrics (`chi_net`, `chi_loss`, `chi_coup`, `grad_norm_squared`, `loss`, `batch_size`, ...) alongside it. + +## Architecture + +`perspic/analyzer.py` — `analyzer(lightning_module, sample_wise_engine="opacus", disable_analyzer=False, log_metrics=True, opacus_strict=False, opacus_approximate_with_n=None, analyze_every=None, analysis_schedule=None, cross_response=False, micro_batch_size=None, effective_batch_size=None, measure_dataloader=None, measure_batch_size=None, measure_subset_seed=None, **model_kwargs)`. A factory function that dynamically subclasses the given `LightningModule` into an `Analyzer` class. `training_step` is overridden to: run pre-step analysis (`_before_training_step`) → delegate to the wrapped module's own `training_step` → manual backward and optimizer step → step any `interval='step'` LR schedulers → log results. `on_train_epoch_end` steps `interval='epoch'` LR schedulers. The wrapped module must define a `criterion` attribute. Supports `cross_response=True`, which expects a dict batch `{"train": ..., "measure": ...}` (via `CombinedLoader`) to additionally measure the model's linear response against a held-out batch, and sparse analysis scheduling via `analyze_every` or `analysis_schedule`. + +`micro_batch_size`/`effective_batch_size` enable train-side gradient accumulation: the DataLoader yields `micro_batch_size` micro-batches, and the optimizer steps only every `effective_batch_size // micro_batch_size` of them, with per-sample gradient-norm metrics accumulated across the cycle and combined (via the correct extensive/intensive scaling — see `examples/batch_accumulation_notes.md`) before logging. + +`measure_dataloader`/`measure_batch_size`/`measure_subset_seed` give the cross-response measurement side an **independent** batch size, decoupled from the train/`CombinedLoader` path above: the analyzer holds a persistent iterator over `measure_dataloader` and, on each analyzed step, gathers a pool of `max(measure_batch_size)` samples. `measure_batch_size` accepts an int or a `list[int]` to sweep multiple sizes per step (largest processed first; smaller sizes are seed-fixable random subsets of the same pool via `measure_subset_seed`). `measure_dataloader.batch_size` is the maximum single-pass batch size; each swept size `S <= measure_dataloader.batch_size` is measured with a single direct pass (no accumulation), while each `S >` it must be an exact multiple and is measured via gradient accumulation (`S // measure_dataloader.batch_size` passes combined into one measurement) — logging `cross_*` metrics with a `@bs{S}` suffix per swept size. See `examples/measurement_batch_sweep.md` for the full design and usage. + +`perspic/calculator/` — the analysis engines used by `analyzer()`: +- `coupling.py` — `CouplingCalculator`: computes the coupling value `chi_coup = ||grad_L||^2 / (chi_loss * chi_net)`. +- `linearizer.py` — `Linearizer`: computes the exact first-order linear response of the loss via gradient dot products. `compute(model, criterion, x1, y1, x2=None, y2=None)` returns `(loss, perturbed_loss, delta_loss)`, with an optional cross term against a second batch. +- `samplewise.py` — `SamplewiseCalculator`: abstract base class defining the interface for per-sample gradient-norm calculators (`compute`, network- and loss-level norm helpers), plus shared helpers. +- `samplewise_functorch.py` — `SamplewiseCalculatorFunctorch`: per-sample gradient norms via `torch.func` (`vmap` + `jacrev`). +- `samplewise_opacus.py` — `SamplewiseCalculatorOpacus`: per-sample gradient norms via Opacus ghost clipping (`GradSampleModuleFastGradientClipping`), with custom BatchNorm grad/norm samplers for eval-mode frozen stats and an optional Hutchinson trace-estimator approximation (`approximate_with_n`) for the network-level norm. + +`perspic/logger.py` — `LogarithmicWindowSchedule` (dataclass) and `logarithmic_windows(max_steps, points_per_decade=10, base_window=5, adaptive_scale=0.0)`, for building logarithmically-spaced step schedules so analysis can run sparsely over long training runs (pass as `analyzer(..., analysis_schedule=...)`). + +`perspic/utils.py` — `BatchStatSnapshot`, a context manager that freezes BatchNorm running stats (with Bessel's correction) so per-sample `vmap` gradients match a train-mode forward pass; `MultiEpochsDataLoader` and `RepeatSampler`, a `DataLoader` subclass that reuses its workers/iterator across epochs. + +Class design relies on: ABCs for the calculator hierarchy, a dataclass for the logging schedule, and a factory-returns-dynamic-subclass pattern for `analyzer()` (it builds `class Analyzer(lightning_module): ...` at call time rather than requiring users to subclass anything themselves). + +## Public API + +`from perspic import ...`: +- `analyzer` — the primary entry point; wraps and instantiates an `Analyzer` from a `LightningModule`. +- `Linearizer` — standalone linear-response calculator. +- `SamplewiseCalculatorFunctorch`, `SamplewiseCalculatorOpacus` — standalone per-sample gradient-norm calculators, also selectable inside `analyzer()` via `sample_wise_engine`. +- `LogarithmicWindowSchedule`, `logarithmic_windows` — sparse analysis scheduling. +- `MultiEpochsDataLoader` — performance-oriented DataLoader. + +Also available via `perspic.calculator`: `CouplingCalculator`, `SamplewiseCalculator` (base class). + +## Design notes + +`analyzer()` sets `automatic_optimization = False` because its analysis metrics require extra forward/backward passes separate from the training pass: the `Linearizer` does its own `zero_grad -> forward -> backward` to get `||grad_L||^2`, and `SamplewiseCalculatorOpacus` runs `output_dim` extra forward+backward passes through the Opacus ghost-clipping hooks for `chi_net` (plus one extra forward for `chi_loss`). Manual optimization keeps these extra passes from corrupting Lightning's own gradient/optimizer state. If the wrapped module already uses manual optimization itself, `analyzer()` delegates to it instead of double-handling the optimizer step. + +## Development + +Install: `pip install -r requirements.txt -r dev-requirements.txt` + +Test: `pytest` +- `tests/unit/` mirrors each `perspic` module 1:1 (`test_analyzer.py`, `test_linearizer.py`, `test_logger.py`, `test_samplewise.py`, `test_samplewise_functorch.py`, `test_samplewise_opacus.py`, `test_utils.py`), using `unittest.mock` and small synthetic Lightning modules. +- `tests/integration/` covers end-to-end/deployment scenarios: `test_analyzer_deployment.py`, `test_hutchinson_approximation.py`, `test_linearizer_deployment.py`, `test_lna_ntk_ground_truth.py`, `test_normalization_scaling.py`, `test_training_deployment.py`. + +Lint/format: `pre-commit run --all-files` (runs `black`, `isort`, `flake8` in that order, `fail_fast: true`), or individually `black .`, `isort .`, `flake8`. + +## Conventions + +- Line length 88 (`black`, preview mode; `flake8` matches). +- `isort` uses the `black` profile. +- `flake8` ignores `W503, E203, E402, C901`; `examples/` is excluded from flake8 checks. +- Docstrings are predominantly Google-style (`Args:`, `Returns:`, `Raises:`). +- Type hints are used throughout function/method signatures. + +## Examples + +`examples/`: +- `cifar10.ipynb` — basic `analyzer()` usage: wrap a `LightningModule`, train on CIFAR-10, plot logged metrics. +- `cifar10_CrossReseponse.ipynb` — `cross_response=True` demo, measuring linear response against a held-out batch via a `CombinedLoader`. +- `batch_accumulation.ipynb` / `batch_accumulation_notes.md` — `micro_batch_size`/`effective_batch_size` gradient-accumulation demo and the design notes explaining the χ_loss aggregation fix and `effective_step`-based logging. +- `batch_size_scaling_analysis.ipynb` — synthetic sweep over batch size (repeated identical samples) verifying `chi_net`/`chi_loss` batch-size invariance directly against `SamplewiseCalculatorFunctorch`, independent of `analyzer()`. +- `measurement_batch_sweep.md` — design notes for `measure_dataloader`/`measure_batch_size`/`measure_subset_seed`, the independent (and sweepable) cross-response measurement batch size. +- `logging_scheduler.ipynb` — `logarithmic_windows` / `LogarithmicWindowSchedule` demo combined with LR scheduling. +- `mup_integration.ipynb` — integrating Maximal Update Parametrization (mup) with perspic. +- `core/hutchinson_convergence.py` — convergence of the Opacus Hutchinson trace-estimator approximation toward the exact per-sample gradient norm. +- `models/` — shared model zoo used by the notebooks above (`cnns.py`, `mlps.py`, `lightning_modules.py`, `utils.py`). diff --git a/examples/batch_accumulation.ipynb b/examples/batch_accumulation.ipynb index b7881d8..7198c5d 100644 --- a/examples/batch_accumulation.ipynb +++ b/examples/batch_accumulation.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 45, "id": "e2d7ffb7", "metadata": {}, "outputs": [ @@ -32,11 +32,12 @@ "\n", "import pytorch_lightning as pl\n", "import torch\n", + "import torch.nn as nn\n", "import torchvision\n", "from pytorch_lightning.callbacks import LearningRateMonitor\n", "from pytorch_lightning.loggers import CSVLogger\n", "from torch.utils.data import DataLoader, random_split\n", - "from torchvision.datasets import CIFAR10\n", + "from torchvision.datasets import CIFAR10, MNIST\n", "from torchvision.models import VisionTransformer\n", "\n", "from perspic.analyzer import analyzer\n", @@ -45,9 +46,19 @@ "pl.seed_everything(7)\n", "\n", "PATH_DATASETS = os.environ.get(\"PATH_DATASETS\", \".\")\n", - "MICRO_BATCH_SIZE = 64\n", - "EFFECTIVE_BATCH_SIZE = 256 # 4x accumulation\n", - "NUM_WORKERS = int(os.cpu_count() / 2)" + "\n", + "# CIFAR-10 / ViT experiment\n", + "MICRO_BATCH_SIZE = 64\n", + "HALF_BATCH_SIZE = 128 # Run 3: direct batch, no accumulation\n", + "EFFECTIVE_BATCH_SIZE = 256 # 4x accumulation\n", + "\n", + "# MNIST / MLP experiment\n", + "MNIST_MICRO_BATCH = 64\n", + "MNIST_EFFECTIVE_BATCH = 256\n", + "\n", + "NUM_WORKERS = int(os.cpu_count() / 2)\n", + "\n", + "torch.set_float32_matmul_precision(\"highest\")" ] }, { @@ -60,23 +71,15 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 46, "id": "7ef5f919", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 170M/170M [00:07<00:00, 23.0MB/s] \n" - ] - } - ], + "outputs": [], "source": [ "stats = ((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n", "train_transform = torchvision.transforms.Compose([\n", - " torchvision.transforms.RandomCrop(32, padding=4),\n", - " torchvision.transforms.RandomHorizontalFlip(),\n", + " #torchvision.transforms.RandomCrop(32, padding=4),\n", + " #torchvision.transforms.RandomHorizontalFlip(),\n", " torchvision.transforms.ToTensor(),\n", " torchvision.transforms.Normalize(*stats),\n", "])\n", @@ -94,11 +97,33 @@ "_, val_set = random_split(val_dataset_full, [45000, 5000], generator=generator)\n", "\n", "# Use micro-batch size for the DataLoader — accumulation handles the rest\n", - "train_dataloader = DataLoader(train_set, batch_size=MICRO_BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS, drop_last=True)\n", + "train_dataloader = DataLoader(train_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)\n", "val_dataloader = DataLoader(val_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)\n", "test_dataloader = DataLoader(test_set, batch_size=MICRO_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)" ] }, + { + "cell_type": "code", + "execution_count": 31, + "id": "969579dc", + "metadata": {}, + "outputs": [], + "source": [ + "mnist_transform = torchvision.transforms.ToTensor()\n", + "\n", + "mnist_full = MNIST(PATH_DATASETS, train=True, download=True, transform=mnist_transform)\n", + "mnist_test = MNIST(PATH_DATASETS, train=False, download=True, transform=mnist_transform)\n", + "\n", + "generator_mnist = torch.Generator().manual_seed(42)\n", + "mnist_train, _ = random_split(mnist_full, [55000, 5000], generator=generator_mnist)\n", + "generator_mnist2 = torch.Generator().manual_seed(42)\n", + "_, mnist_val = random_split(mnist_full, [55000, 5000], generator=generator_mnist2)\n", + "\n", + "mnist_train_dl = DataLoader(mnist_train, batch_size=MNIST_MICRO_BATCH, shuffle=True, num_workers=NUM_WORKERS, drop_last=True)\n", + "mnist_train_dl_full = DataLoader(mnist_train, batch_size=MNIST_EFFECTIVE_BATCH, shuffle=True, num_workers=NUM_WORKERS, drop_last=True)\n", + "mnist_val_dl = DataLoader(mnist_val, batch_size=MNIST_EFFECTIVE_BATCH, shuffle=False, num_workers=NUM_WORKERS, drop_last=True)" + ] + }, { "cell_type": "markdown", "id": "321b9a72", @@ -111,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 47, "id": "06369a8f", "metadata": {}, "outputs": [], @@ -127,23 +152,51 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": 40, + "id": "4cb8a66b", + "metadata": {}, + "outputs": [], + "source": [ + "class MLP(nn.Module):\n", + " \"\"\"Three-layer MLP for MNIST (784 → 256 → 128 → 10).\"\"\"\n", + "\n", + " def __init__(self, input_dim=784, hidden_dim1=256, hidden_dim2=1028, hidden_dim3=128, output_dim=10):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(input_dim, hidden_dim1)\n", + " self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)\n", + " self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)\n", + " self.fc4 = nn.Linear(hidden_dim3, output_dim)\n", + "\n", + " def forward(self, x):\n", + " x = x.flatten(start_dim=1) # (B, 1, 28, 28) → (B, 784)\n", + " x = torch.relu(self.fc1(x))\n", + " x = torch.relu(self.fc2(x))\n", + " x = torch.relu(self.fc3(x))\n", + " return self.fc4(x)" + ] + }, { "cell_type": "markdown", "id": "563f9db6", "metadata": {}, "source": [ - "## Training with Batch Accumulation\n", + "## Training Setup\n", "\n", - "The key parameters are `micro_batch_size` and `effective_batch_size`. The analyzer will:\n", - "- Zero gradients only at the start of each accumulation cycle\n", - "- Scale the loss by `1 / accumulation_steps` during backward\n", - "- Step the optimizer only after `accumulation_steps = effective_batch_size // micro_batch_size` micro-batches\n", - "- Accumulate analysis metrics across micro-batches and log once per effective step" + "We run four experiments with the same model architecture:\n", + "\n", + "- **Accum (`micro=64, effective=256`)**: DataLoader B=64, gradient accumulation over 4 steps simulates B=256.\n", + "- **Full batch (`B=256`)**: DataLoader B=256 directly, no accumulation.\n", + "- **Half batch (`B=128`)**: DataLoader B=128, no accumulation. Intermediate reference.\n", + "- **Single batch (`B=64`)**: DataLoader B=64, no accumulation. Shows the raw noise level of the micro-batch size; makes 4× more optimizer steps per epoch than the B=256 runs.\n", + "\n", + "If accumulation is correct, Run 1 and Run 2 should overlay. Run 4 lets you see the noise floor a B=64 batch introduces — the accumulation run should match B=256 noise, not B=64 noise." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 48, "id": "998cb095", "metadata": {}, "outputs": [ @@ -151,30 +204,78 @@ "name": "stdout", "output_type": "stream", "text": [ - "Accumulation steps: 4\n", - "Micro-batch size: 64\n", - "Effective batch: 256\n" + "Run 1 — accumulation steps: 4 (micro=64, effective=256)\n", + "Run 2 — accumulation steps: 1 (full batch=256)\n", + "Run 3 — accumulation steps: 1 (half batch=128)\n", + "Run 4 — accumulation steps: 1 (single batch=64, no accum)\n" ] } ], "source": [ + "import copy\n", + "\n", + "# --- Run 1: gradient accumulation (micro_batch=64, effective=256) ---\n", + "model_vit_accum = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", "vit_accum = analyzer(\n", " lightning_module=ClassificationModule,\n", " sample_wise_engine=\"opacus\",\n", " micro_batch_size=MICRO_BATCH_SIZE,\n", " effective_batch_size=EFFECTIVE_BATCH_SIZE,\n", - " model=model_vit,\n", + " model=model_vit_accum,\n", + " lr=0.005,\n", + ")\n", + "\n", + "# --- Run 2: full batch (batch=256, no accumulation) ---\n", + "model_vit_full = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", + "vit_full = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=EFFECTIVE_BATCH_SIZE, # DataLoader batch IS the effective batch\n", + " model=model_vit_full,\n", + " lr=0.005,\n", + ")\n", + "\n", + "# --- Run 3: half batch (batch=128, no accumulation) ---\n", + "model_vit_half = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", + "vit_half = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=HALF_BATCH_SIZE,\n", + " model=model_vit_half,\n", " lr=0.005,\n", ")\n", "\n", - "print(f\"Accumulation steps: {vit_accum.accumulation_steps}\")\n", - "print(f\"Micro-batch size: {MICRO_BATCH_SIZE}\")\n", - "print(f\"Effective batch: {EFFECTIVE_BATCH_SIZE}\")" + "# --- Run 4: single micro-batch (batch=64, no accumulation) — noise baseline ---\n", + "model_vit_single = VisionTransformer(\n", + " image_size=32, patch_size=8, num_layers=2, num_heads=4,\n", + " hidden_dim=128, mlp_dim=256, num_classes=10,\n", + ")\n", + "vit_single = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " sample_wise_engine=\"opacus\",\n", + " micro_batch_size=MICRO_BATCH_SIZE, # no effective_batch_size → no accumulation\n", + " model=model_vit_single,\n", + " lr=0.005,\n", + ")\n", + "\n", + "print(f\"Run 1 — accumulation steps: {vit_accum.accumulation_steps} (micro={MICRO_BATCH_SIZE}, effective={EFFECTIVE_BATCH_SIZE})\")\n", + "print(f\"Run 2 — accumulation steps: {vit_full.accumulation_steps} (full batch={EFFECTIVE_BATCH_SIZE})\")\n", + "print(f\"Run 3 — accumulation steps: {vit_half.accumulation_steps} (half batch={HALF_BATCH_SIZE})\")\n", + "print(f\"Run 4 — accumulation steps: {vit_single.accumulation_steps} (single batch={MICRO_BATCH_SIZE}, no accum)\")" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 49, "id": "930b6ba5", "metadata": {}, "outputs": [ @@ -182,11 +283,11 @@ "name": "stderr", "output_type": "stream", "text": [ + "Seed set to 7\n", "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n", "GPU available: True (cuda), used: True\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", - "You are using a CUDA device ('NVIDIA GeForce RTX 4090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n", "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", "\n", " | Name | Type | Params | Mode \n", @@ -204,7 +305,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c212575e0e8c49fe97c41ce521e73e22", + "model_id": "1d5e87ec911344cebe5d3e5ff84ad3e7", "version_major": 2, "version_minor": 0 }, @@ -218,7 +319,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bf1fb678533349adb35faf28d6c90b3d", + "model_id": "9c446d7611a94a0595464a13cdffd688", "version_major": 2, "version_minor": 0 }, @@ -241,7 +342,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "887a7dcd3bdd43b9a9ba1c78500c4c7a", + "model_id": "6ca8838b582a4c6a8d4e78c8a5ea63d5", "version_major": 2, "version_minor": 0 }, @@ -255,7 +356,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "75752b136ed244b3b2a3a863cb99ea49", + "model_id": "825aeaaf66ef49529f9188d0f362b556", "version_major": 2, "version_minor": 0 }, @@ -269,7 +370,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e57d185ba4ba4c829c9536bcad68ed4b", + "model_id": "a9fbc016e415496cb00bc7702c7b16a2", "version_major": 2, "version_minor": 0 }, @@ -283,7 +384,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a0a276e639314e5280e9085ba2515507", + "model_id": "6737ee1a55f744adb12238f944019133", "version_major": 2, "version_minor": 0 }, @@ -297,7 +398,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "893cd1f76f354362814a5ae7b1976bbd", + "model_id": "05a257bc686f4b1c9c87ed02fdd00c90", "version_major": 2, "version_minor": 0 }, @@ -313,292 +414,1175 @@ "output_type": "stream", "text": [ "`Trainer.fit` stopped: `max_epochs=5` reached.\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n" + "Seed set to 7\n", + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params | Mode \n", + "----------------------------------------------------\n", + "0 | model | VisionTransformer | 293 K | train\n", + "----------------------------------------------------\n", + "293 K Trainable params\n", + "0 Non-trainable params\n", + "293 K Total params\n", + "1.174 Total estimated model params size (MB)\n", + "32 Modules in train mode\n", + "0 Modules in eval mode\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "96ee6acf5f894512845e48d3e2906d94", + "model_id": "cd13f990c6cc4a738710529f817f4a03", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Testing: | | 0/? [00:00" + "Training: | | 0/? [00:00 0:\n", - " ax.plot(subset[step_col], subset[\"chi_net\"], alpha=0.3, label=\"Raw\")\n", - " ax.plot(subset[step_col], subset[\"chi_net\"].ewm(span=50).mean(), label=\"EMA\")\n", - "ax.set_title(\"χ_net\")\n", - "ax.set_xlabel(\"Step\")\n", - "ax.set_yscale(\"log\")\n", - "ax.legend()\n", - "ax.grid(True, alpha=0.3)\n", - "\n", - "# chi_loss\n", - "ax = axes[1, 0]\n", - "subset = metrics[[\"chi_loss\", step_col]].dropna()\n", - "if len(subset) > 0:\n", - " ax.plot(subset[step_col], subset[\"chi_loss\"], alpha=0.3, label=\"Raw\")\n", - " ax.plot(subset[step_col], subset[\"chi_loss\"].ewm(span=50).mean(), label=\"EMA\")\n", - "ax.set_title(\"χ_loss\")\n", - "ax.set_xlabel(\"Step\")\n", - "ax.legend()\n", - "ax.grid(True, alpha=0.3)\n", - "\n", - "# Coupling\n", - "ax = axes[1, 1]\n", - "subset = metrics[[\"chi_coup\", step_col]].dropna()\n", - "if len(subset) > 0:\n", - " ax.plot(subset[step_col], subset[\"chi_coup\"], alpha=0.3, label=\"Raw\")\n", - " ax.plot(subset[step_col], subset[\"chi_coup\"].ewm(span=50).mean(), label=\"EMA\")\n", - "ax.set_title(\"Coupling Coefficient (χ_coup)\")\n", - "ax.set_xlabel(\"Step\")\n", - "ax.set_yscale(\"log\")\n", - "ax.legend()\n", - "ax.grid(True, alpha=0.3)\n", - "\n", - "plt.tight_layout()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "0c915b2b", - "metadata": {}, - "outputs": [ + }, { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
chi_netchi_losschi_coupstep
0NaNNaNNaN0
1NaNNaNNaN0
2NaNNaNNaN1
3NaNNaNNaN1
4NaNNaNNaN2
...............
7031242.86471613.599990.0000413513
7032NaNNaNNaN3514
7033242.86471613.599990.0000413514
7034NaNNaNNaN3514
7035NaNNaNNaN3515
\n", - "

7036 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " chi_net chi_loss chi_coup step\n", - "0 NaN NaN NaN 0\n", - "1 NaN NaN NaN 0\n", - "2 NaN NaN NaN 1\n", - "3 NaN NaN NaN 1\n", - "4 NaN NaN NaN 2\n", - "... ... ... ... ...\n", - "7031 242.864716 13.59999 0.000041 3513\n", - "7032 NaN NaN NaN 3514\n", - "7033 242.864716 13.59999 0.000041 3514\n", - "7034 NaN NaN NaN 3514\n", - "7035 NaN NaN NaN 3515\n", - "\n", - "[7036 rows x 4 columns]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "metrics.keys()\n", - "metrics\n", - "metrics[[\"chi_net\", \"chi_loss\", \"chi_coup\", \"step\"]]" + "application/vnd.jupyter.widget-view+json": { + "model_id": "69a3b5452f8b4c258afa8048759e6a7a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Validation: | | 0/? [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "metrics_accum = pd.read_csv(f\"{trainer_accum.logger.log_dir}/metrics.csv\")\n", + "metrics_full = pd.read_csv(f\"{trainer_full.logger.log_dir}/metrics.csv\")\n", + "metrics_half = pd.read_csv(f\"{trainer_half.logger.log_dir}/metrics.csv\")\n", + "metrics_single = pd.read_csv(f\"{trainer_single.logger.log_dir}/metrics.csv\")\n", + "\n", + "EMA_SPAN = 20\n", + "\n", + "COLORS = {\n", + " \"accum\": \"steelblue\",\n", + " \"full\": \"tomato\",\n", + " \"half\": \"seagreen\",\n", + " \"single\": \"olive\",\n", + "}\n", + "\n", + "ZORDER = {\n", + " \"accum\": 25,\n", + " \"full\": 30,\n", + " \"half\": 20,\n", + " \"single\": 10,\n", + "}\n", + "\n", + "def plot_metric(ax, metrics, col, step_col, label, color, alpha_raw=0.2, zorder=None):\n", + " subset = metrics[[col, step_col]].dropna()\n", + " if subset.empty:\n", + " return\n", + " ax.plot(subset[step_col], subset[col], alpha=alpha_raw, color=color, zorder=zorder or ZORDER.get(label.split()[-1], 0))\n", + " ax.plot(subset[step_col], subset[col].ewm(span=EMA_SPAN).mean(),\n", + " label=label, color=color, linewidth=1.8, zorder=zorder or ZORDER.get(label.split()[-1], 0))\n", + "\n", + "def loss_per_opt_step(metrics, loss_col=\"train_loss\"):\n", + " \"\"\"Return (x, y) for train_loss aligned to one point per optimizer step.\n", + "\n", + " For accumulation runs: effective_step is logged on every micro-batch, so\n", + " groupby(effective_step).mean() averages exactly the K micro-batches of each\n", + " cycle — the correct per-cycle mean loss.\n", + "\n", + " For non-accumulation runs: no effective_step column; use plain step.\n", + " \"\"\"\n", + " if \"effective_step\" in metrics.columns:\n", + " sub = metrics[[\"effective_step\", loss_col]].dropna()\n", + " sub = sub.groupby(\"effective_step\")[loss_col].mean().reset_index()\n", + " return sub[\"effective_step\"], sub[loss_col]\n", + " else:\n", + " sub = metrics[[\"step\", loss_col]].dropna()\n", + " return sub[\"step\"], sub[loss_col]\n", + "\n", + "fig, axes = plt.subplots(2, 2, figsize=(14, 8))\n", + "fig.suptitle(\n", + " \"ViT: \"\n", + " f\"Accum (micro={MICRO_BATCH_SIZE}→eff={EFFECTIVE_BATCH_SIZE}) vs \"\n", + " f\"Full B={EFFECTIVE_BATCH_SIZE} vs Half B={HALF_BATCH_SIZE} vs Single B={MICRO_BATCH_SIZE}\",\n", + " fontsize=12,\n", + ")\n", + "\n", + "step_accum = \"analysis_step\" if \"analysis_step\" in metrics_accum.columns else \"step\"\n", + "step_full = \"analysis_step\" if \"analysis_step\" in metrics_full.columns else \"step\"\n", + "step_half = \"analysis_step\" if \"analysis_step\" in metrics_half.columns else \"step\"\n", + "step_single = \"analysis_step\" if \"analysis_step\" in metrics_single.columns else \"step\"\n", + "\n", + "# --- Train Loss ---\n", + "ax = axes[0, 0]\n", + "x_a, y_a = loss_per_opt_step(metrics_accum)\n", + "x_f, y_f = loss_per_opt_step(metrics_full)\n", + "x_h, y_h = loss_per_opt_step(metrics_half)\n", + "x_s, y_s = loss_per_opt_step(metrics_single)\n", + "for x, y, col, lbl in [\n", + " (x_a, y_a, COLORS[\"accum\"], f\"Accum micro={MICRO_BATCH_SIZE}→eff={EFFECTIVE_BATCH_SIZE}\"),\n", + " (x_f, y_f, COLORS[\"full\"], f\"Full B={EFFECTIVE_BATCH_SIZE}\"),\n", + " (x_h, y_h, COLORS[\"half\"], f\"Half B={HALF_BATCH_SIZE}\"),\n", + " (x_s, y_s, COLORS[\"single\"], f\"Single B={MICRO_BATCH_SIZE} (no accum)\"),\n", + "]:\n", + " ax.plot(x, y, alpha=0.15, color=col, zorder=ZORDER.get(lbl.split()[-1], 0))\n", + " ax.plot(x, y.ewm(span=EMA_SPAN).mean(), color=col, linewidth=1.8, label=lbl, zorder=ZORDER.get(lbl.split()[-1], 0))\n", + "ax.set_title(\"Train Loss\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# --- χ_net ---\n", + "ax = axes[0, 1]\n", + "plot_metric(ax, metrics_accum, \"chi_net\", step_accum, f\"Accum micro={MICRO_BATCH_SIZE}\", COLORS[\"accum\"], zorder=ZORDER[\"accum\"])\n", + "plot_metric(ax, metrics_full, \"chi_net\", step_full, f\"Full B={EFFECTIVE_BATCH_SIZE}\", COLORS[\"full\"], zorder=ZORDER[\"full\"])\n", + "plot_metric(ax, metrics_half, \"chi_net\", step_half, f\"Half B={HALF_BATCH_SIZE}\", COLORS[\"half\"], zorder=ZORDER[\"half\"])\n", + "plot_metric(ax, metrics_single, \"chi_net\", step_single, f\"Single B={MICRO_BATCH_SIZE}\", COLORS[\"single\"], zorder=ZORDER[\"single\"])\n", + "ax.set_title(\"χ_net\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.set_ylim(1e2,5e3)\n", + "ax.set_xlim(1, 5e3)\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# --- χ_loss ---\n", + "ax = axes[1, 0]\n", + "plot_metric(ax, metrics_accum, \"chi_loss\", step_accum, f\"Accum micro={MICRO_BATCH_SIZE}\", COLORS[\"accum\"], zorder=ZORDER[\"accum\"])\n", + "plot_metric(ax, metrics_full, \"chi_loss\", step_full, f\"Full B={EFFECTIVE_BATCH_SIZE}\", COLORS[\"full\"], zorder=ZORDER[\"full\"])\n", + "plot_metric(ax, metrics_half, \"chi_loss\", step_half, f\"Half B={HALF_BATCH_SIZE}\", COLORS[\"half\"], zorder=ZORDER[\"half\"])\n", + "plot_metric(ax, metrics_single, \"chi_loss\", step_single, f\"Single B={MICRO_BATCH_SIZE}\", COLORS[\"single\"], zorder=ZORDER[\"single\"])\n", + "ax.set_title(\"χ_loss\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "# --- Coupling ---\n", + "ax = axes[1, 1]\n", + "plot_metric(ax, metrics_accum, \"chi_coup\", step_accum, f\"Accum micro={MICRO_BATCH_SIZE}\", COLORS[\"accum\"], zorder=ZORDER[\"accum\"])\n", + "plot_metric(ax, metrics_full, \"chi_coup\", step_full, f\"Full B={EFFECTIVE_BATCH_SIZE}\", COLORS[\"full\"], zorder=ZORDER[\"full\"])\n", + "plot_metric(ax, metrics_half, \"chi_coup\", step_half, f\"Half B={HALF_BATCH_SIZE}\", COLORS[\"half\"], zorder=ZORDER[\"half\"])\n", + "plot_metric(ax, metrics_single, \"chi_coup\", step_single, f\"Single B={MICRO_BATCH_SIZE}\", COLORS[\"single\"], zorder=ZORDER[\"single\"])\n", + "ax.set_title(\"Coupling (χ_coup)\")\n", + "ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\")\n", + "ax.set_xscale(\"log\")\n", + "ax.legend(fontsize=8)\n", + "ax.grid(True, alpha=0.3)\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0a62bfce", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## MLP — MNIST with Gradient Accumulation\n", + "\n", + "Same accumulation comparison on a simpler model and dataset: a three-layer MLP (784→256→128→10) trained on MNIST.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "a51fb353", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 7\n", + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params | Mode \n", + "---------------------------------------\n", + "0 | model | MLP | 598 K | train\n", + "---------------------------------------\n", + "598 K Trainable params\n", + "0 Non-trainable params\n", + "598 K Total params\n", + "2.393 Total estimated model params size (MB)\n", + "5 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MLP accum: 4 steps (micro=64 → eff=256)\n", + "MLP full: 1 step (batch=256)\n", + "MLP half: 1 step (batch=128)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "032453cc60a741a3a5852a28ace98b21", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Sanity Checking: | | 0/? [00:00:0: UserWarning: Full backward hook is firing when gradients are computed with respect to module outputs since no inputs require gradients. See https://docs.pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.register_full_backward_hook for more details.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1bc3a766a8214c54b5159733bf7aa512", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Validation: | | 0/? [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import glob\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "def _load_latest_csv(log_name):\n", + " matches = sorted(glob.glob(f\"logs/{log_name}/version_*/metrics.csv\"))\n", + " if not matches:\n", + " raise FileNotFoundError(\n", + " f\"No logs found under logs/{log_name}/. Run the MLP training cell first.\"\n", + " )\n", + " return pd.read_csv(matches[-1])\n", + "\n", + "metrics_mlp_accum = _load_latest_csv(\"mlp_accumulation\")\n", + "metrics_mlp_full = _load_latest_csv(\"mlp_full_batch\")\n", + "metrics_mlp_half = _load_latest_csv(\"mlp_half_batch\")\n", + "\n", + "_EMA = 10\n", + "\n", + "def _loss_per_opt_step(metrics, loss_col=\"train_loss\"):\n", + " if \"effective_step\" in metrics.columns:\n", + " sub = metrics[[\"effective_step\", loss_col]].dropna()\n", + " sub = sub.groupby(\"effective_step\")[loss_col].mean().reset_index()\n", + " return sub[\"effective_step\"], sub[loss_col]\n", + " sub = metrics[[\"step\", loss_col]].dropna()\n", + " return sub[\"step\"], sub[loss_col]\n", + "\n", + "def _plot_metric(ax, metrics, col, step_col, label, color):\n", + " subset = metrics[[col, step_col]].dropna()\n", + " if subset.empty:\n", + " return\n", + " ax.plot(subset[step_col], subset[col], alpha=0.2, color=color)\n", + " ax.plot(subset[step_col], subset[col].ewm(span=_EMA).mean(),\n", + " label=label, color=color, linewidth=1.8)\n", + "\n", + "fig, axes = plt.subplots(2, 2, figsize=(13, 8))\n", + "fig.suptitle(\n", + " f\"MLP/MNIST: Accum (micro={MNIST_MICRO_BATCH}→eff={MNIST_EFFECTIVE_BATCH}) \"\n", + " f\"vs Full batch ({MNIST_EFFECTIVE_BATCH})\"\n", + " f\" vs Half batch ({HALF_BATCH_SIZE})\",\n", + " fontsize=13,\n", + ")\n", + "\n", + "step_ma = \"analysis_step\" if \"analysis_step\" in metrics_mlp_accum.columns else \"step\"\n", + "step_mf = \"analysis_step\" if \"analysis_step\" in metrics_mlp_full.columns else \"step\"\n", + "step_mh = \"analysis_step\" if \"analysis_step\" in metrics_mlp_half.columns else \"step\"\n", + "\n", + "ax = axes[0, 0]\n", + "x_a, y_a = _loss_per_opt_step(metrics_mlp_accum)\n", + "x_f, y_f = _loss_per_opt_step(metrics_mlp_full)\n", + "x_h, y_h = _loss_per_opt_step(metrics_mlp_half)\n", + "ax.plot(x_a, y_a, alpha=0.2, color=\"steelblue\")\n", + "ax.plot(x_a, y_a.ewm(span=_EMA).mean(), color=\"steelblue\", linewidth=1.8,\n", + " label=f\"Accum micro={MNIST_MICRO_BATCH}\")\n", + "ax.plot(x_f, y_f, alpha=0.2, color=\"tomato\")\n", + "ax.plot(x_f, y_f.ewm(span=_EMA).mean(), color=\"tomato\", linewidth=1.8,\n", + " label=f\"Full batch={MNIST_EFFECTIVE_BATCH}\")\n", + "ax.plot(x_h, y_h, alpha=0.2, color=\"green\")\n", + "ax.plot(x_h, y_h.ewm(span=_EMA).mean(), color=\"green\", linewidth=1.8,\n", + " label=f\"Half batch={HALF_BATCH_SIZE}\")\n", + "ax.set_title(\"Train Loss\"); ax.set_xlabel(\"Effective optimizer step\")\n", + "ax.set_yscale(\"log\"); ax.set_xscale(\"log\")\n", + "ax.legend(); ax.grid(True, alpha=0.3)\n", + "\n", + "for ax, col, title in [\n", + " (axes[0, 1], \"chi_net\", \"χ_net\"),\n", + " (axes[1, 0], \"chi_loss\", \"χ_loss\"),\n", + " (axes[1, 1], \"chi_coup\", \"Coupling (χ_coup)\"),\n", + "]:\n", + " _plot_metric(ax, metrics_mlp_accum, col, step_ma, f\"Accum micro={MNIST_MICRO_BATCH}\", \"steelblue\")\n", + " _plot_metric(ax, metrics_mlp_full, col, step_mf, f\"Full batch={MNIST_EFFECTIVE_BATCH}\", \"tomato\")\n", + " _plot_metric(ax, metrics_mlp_half, col, step_mh, f\"Half batch={HALF_BATCH_SIZE}\", \"green\")\n", + " ax.set_title(title); ax.set_xlabel(\"Effective optimizer step\")\n", + " ax.set_yscale(\"log\"); ax.set_xscale(\"log\")\n", + " ax.legend(); ax.grid(True, alpha=0.3)\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" ] }, { "cell_type": "code", "execution_count": null, - "id": "65578b50", + "id": "40bef94b", "metadata": {}, "outputs": [], "source": [] @@ -606,7 +1590,7 @@ ], "metadata": { "kernelspec": { - "display_name": "mast", + "display_name": "Python 3", "language": "python", "name": "python3" }, diff --git a/examples/batch_accumulation_notes.md b/examples/batch_accumulation_notes.md new file mode 100644 index 0000000..74e5a19 --- /dev/null +++ b/examples/batch_accumulation_notes.md @@ -0,0 +1,243 @@ +# Batch Accumulation in Perspic — Implementation Notes + +## What gradient accumulation does + +Instead of feeding a large batch of size N=K·B through the model in one forward pass, we +feed K micro-batches of size B and accumulate the resulting gradients before calling +`opt.step()`. The loss of each micro-batch is divided by K before `backward()` so that +the gradient that lands in `p.grad` after K steps is identical to the gradient you would +have obtained from one forward pass on the full N-sample batch. + +``` +gradient update = Σ_k grad( L_k / K ) + = (1/K) Σ_k grad( L_k ) # linearity of differentiation + = grad( (1/K) Σ_k L_k ) # same as mean loss over all N samples + = grad( L_full ) # ✓ +``` + +This is verified numerically: with Adam and SGD the max parameter-gradient difference +between the accumulated run and the full-batch run is < 2 × 10⁻⁸. + +--- + +## Issue 1 — χ_loss aggregation was wrong (K² error, now fixed) + +### Background: how χ_loss is normalised + +`SamplewiseCalculatorOpacus.compute()` (and the functorch variant) return two metrics with +`normalize=True`: + +| metric | formula returned | meaning | +|---|---|---| +| `chi_net` | `Σᵢ ‖∇_θ f(xᵢ)‖² / B` | average per-sample squared network-gradient norm | +| `chi_loss` | `Σᵢ ‖∇_f L(f(xᵢ),yᵢ)‖² · B` | scaled sum of per-sample squared loss-gradient norms | + +The `· B` factor in `chi_loss` looks like it makes it "extensive" (growing with B), but +this is cancelled by the `1/B` that appears inside `∇_f L` when the loss is mean-reduced +(PyTorch `cross_entropy` default). + +#### Why chi_loss is effectively intensive + +With mean-reduced cross-entropy, the per-sample loss gradient w.r.t. the network output +scales as **1/B**: + +``` +∂L_mean/∂f(xᵢ) = (1/B) · ∂L_individual(xᵢ)/∂f(xᵢ) +``` + +Substituting into the formula: + +``` +chi_loss(B) = Σᵢ ‖ (1/B) · gᵢ ‖² · B + = Σᵢ ‖gᵢ‖² / B² · B + = Σᵢ ‖gᵢ‖² / B +``` + +where `gᵢ = ∂L_individual(xᵢ)/∂f(xᵢ)` is independent of B. So `chi_loss(B)` scales as +`1/B`, and the product `(sum) · B` that the calculator returns is **constant w.r.t. B** +(at fixed data distribution). + +This can be verified empirically: `chi_loss(B=64) ≈ chi_loss(B=256) ≈ 0.9` for the same +model and data distribution. + +### The original wrong formula + +The original accumulation code used: + +```python +chi_loss_eff = K * sum(chi_loss_k) # WRONG +``` + +The reasoning was that `chi_loss` scales with B, so for N=K·B it should be multiplied by +K. But as shown above, chi_loss does *not* grow with B for mean-reduced losses. The +correct aggregation for an intensive quantity is the **mean**: + +``` +chi_loss(N=K·B) = Σ_{all N} ‖gᵢ‖² / N + = (K · Σ_{i in micro} ‖gᵢ‖²) / (K·B) + = (Σ_{i in micro} ‖gᵢ‖²) / B + = chi_loss(B) + = mean_k( chi_loss_k ) +``` + +### The error it caused + +| quantity | correct formula | old formula | error factor | +|---|---|---|---| +| `chi_net_eff` | `mean(chi_net_k)` | `mean(chi_net_k)` | 1 (correct) | +| `chi_loss_eff` | `mean(chi_loss_k)` | `K · sum(chi_loss_k)` | K² too large | +| `chi_coup` | `‖∇L‖² / (chi_loss · chi_net)` | (derived) | K² too small | + +With K=4 this produced a **16× discrepancy** in `chi_loss` and `chi_coup` compared to the +full-batch run, clearly visible in the bottom row of the comparison plot. + +### The fix + +```python +# was: +chi_loss_eff = K * sum(self._accum_chi_loss) +# now: +chi_loss_eff = sum(self._accum_chi_loss) / K +``` + +Same correction applies to `chi_loss_cross_eff` for the cross-response path. + +### Generalisation note + +If you use a **sum-reduced** loss (e.g. `reduction='sum'` in `cross_entropy`), then +`chi_loss` *would* scale as B and the correct formula would be `K · sum(chi_loss_k)`. +The right formula depends on the loss normalisation convention. With the default +mean-reduced loss, `mean` is correct. + +--- + +## Issue 2 — The train_loss curve looks different between runs (not a bug) + +The bottom-left panel of the comparison plot uses `global_step` (Lightning's micro-batch +counter) on the x-axis. This causes a visual mismatch between the two runs: + +### X-axis misalignment + +| run | steps per opt.step() | opt steps after N global steps | +|---|---|---| +| full batch (B=256) | 1 | N | +| accumulation (K=4) | 4 | N/4 | + +After 1000 global steps, the full-batch run has taken 1000 optimizer steps, while the +accumulation run has only taken 250. Both runs make the same number of optimizer steps +eventually (at step 4000 for accumulation vs step 1000 for full batch), but the x-axis +stretches the accumulation curve by a factor of K=4. + +**This is a display artifact, not a computation error.** The analysis metrics (χ_net, +χ_loss, χ_coup) use `analysis_step = effective_step` (the optimizer step count) and are +therefore correctly aligned. Only `train_loss` uses raw `global_step`. + +### Higher noise in the loss curve — a logging artifact, not a training difference + +The accumulation run *looks* noisier because `ClassificationModule.training_step` calls +`self.log("train_loss", loss)` on **every** call, which fires once per micro-batch. So +for K=4 accumulation the CSV gets four separate loss values per optimizer step, each +computed on a fresh B=64 mini-batch drawn from the DataLoader. + +The full-batch run logs once per optimizer step, each value computed on a B=256 batch. + +This creates two compounding visual effects: + +1. **Sampling variance.** A B=64 loss estimate has variance ≈ σ²/64; a B=256 estimate + has variance ≈ σ²/256 — four times smaller. So every individual logged point in the + accumulation run is noisier, which is reflected in the raw (faint) trace. + +2. **Four times as many points.** The EMA smoother sees four data points per optimizer + step instead of one, so it gets reset by each new noisy value before it can settle — + making the smoothed curve appear noisier too. + +**Neither effect reflects a real difference in what the optimizer is doing.** The +gradient that `opt.step()` acts on is the mean over all four micro-batches, which is +mathematically identical to the gradient from a single B=256 pass (verified numerically: +max parameter-gradient difference < 2 × 10⁻⁸). The apparent noise is entirely a +consequence of *what is being logged*, not of what is being computed. + +**Why `train_loss` can't be re-logged as a mean — Lightning key de-duplication:** + +Lightning's `CSVLogger` writes one row per `global_step` (micro-batch counter). When +`self.log(...)` is called anywhere during a `training_step`, the value is accumulated +into the *current step's* metrics dict and flushed to the CSV at the end of that step. +The wrapped model's `training_step` logs `train_loss` (a single micro-batch loss) first; +any subsequent call to `self.log("train_loss", mean_loss)` at cycle-end loses because +Lightning does not let a later log overwrite an earlier one for the same key and step. +The cycle-end mean is silently dropped — the raw micro-batch loss always appears in the +CSV. + +**First plot fix attempted — `.last()` per `effective_step` group:** + +When `effective_step` is logged only at the cycle end, Lightning **forward-fills** its +value into the next `K-1` rows (the first three micro-batches of cycle N+1). So +`effective_step=N` appears in four CSV rows: + +- `global_step` = 4N (micro-batch 4 of cycle N — cycle end, `effective_step` logged here) +- `global_step` = 4N+1 (micro-batch 1 of cycle N+1 — **forward-filled**) +- `global_step` = 4N+2 (micro-batch 2 of cycle N+1 — **forward-filled**) +- `global_step` = 4N+3 (micro-batch 3 of cycle N+1 — **forward-filled**) + +This means `.groupby("effective_step").last()` picks `global_step = 4N+3` — a +micro-batch from the *next* cycle, not the mean. `.first()` picks `global_step = 4N`, +which is the 4th micro-batch loss of cycle N (not a mean either, since the mean log +was dropped). Neither `.first()` nor `.last()` gives the true per-cycle mean. + +**Second fix — `.mean()` per `effective_step` group:** + +`.groupby("effective_step").mean()` averages the four rows that share `effective_step=N`. +Even though three of them belong to cycle N+1 (due to forward-fill), this is a +sliding-window average over four consecutive micro-batches and reduces noise by roughly +4×. Empirically: accumulation EMA residual std = 0.052 vs full-batch 0.053 — essentially +equal. But the averaging window is shifted by one micro-batch relative to the optimizer +step boundary. + +**Correct fix — log `effective_step` on every micro-batch:** + +The cleanest solution is to tag each micro-batch with its cycle's `effective_step` at +the time it runs, eliminating any dependence on Lightning's forward-fill: + +```python +# Inside training_step, for every micro-batch during accumulation: +# opt.step() has not fired yet so +1 gives the current cycle number +self.log("effective_step", float(self._optimizer_step_count + 1), + on_step=True, on_epoch=False) +``` + +With this in place, all K micro-batches of cycle N carry `effective_step=N` directly. +`groupby("effective_step").mean()` then averages exactly those K micro-batches — +the true mean loss over the effective batch of size K·B. No forward-fill, no shifted +window, no dropped values. + +### Is the optimizer step counter affected? + +No. PyTorch optimizers (Adam, SGD, …) maintain an internal step counter `t` that +increments by 1 on each call to `opt.step()`. Adam uses `t` for bias correction: + +``` +m̂ = m / (1 - β₁ᵗ), v̂ = v / (1 - β₂ᵗ) +``` + +With gradient accumulation, `opt.step()` is called exactly as many times as with a +full-batch run (once per effective batch), so `t` is identical between the two runs at +any given optimizer step. Adam's bias correction is unaffected. The gradient accumulation +does not introduce any error via the optimizer's internal state. + +### Summary of the implemented fix + +`analyzer.py` now logs `effective_step` on **every** micro-batch (not just at +cycle-end), using `_optimizer_step_count + 1` so all K micro-batches of cycle N share +the same `effective_step = N` value without relying on Lightning's forward-fill. + +The notebook's `loss_per_opt_step()` helper then applies: + +```python +sub = metrics[["effective_step", "train_loss"]].dropna() +sub = sub.groupby("effective_step")["train_loss"].mean().reset_index() +``` + +This averages exactly the K micro-batch losses from the same optimizer step, giving one +data point per optimizer step with variance σ²/(K·B) — the same as a direct B·K batch. +The full-batch run needs no special treatment. All four plot panels use `effective_step` +on the x-axis, making the curves directly comparable. diff --git a/examples/batch_size_scaling_analysis.ipynb b/examples/batch_size_scaling_analysis.ipynb index 0395363..624f1be 100644 --- a/examples/batch_size_scaling_analysis.ipynb +++ b/examples/batch_size_scaling_analysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "297431a4", "metadata": {}, "outputs": [], @@ -495,7 +495,7 @@ ], "metadata": { "kernelspec": { - "display_name": "lightning", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -509,7 +509,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.11" + "version": "3.13.7" } }, "nbformat": 4, diff --git a/first_wrong_test_batch_accumulation.png b/first_wrong_test_batch_accumulation.png new file mode 100644 index 0000000000000000000000000000000000000000..dfdf780958e58698fd80001076d17494384c106a GIT binary patch literal 180740 zcmdSBbySw$w>=6fC?O&tAt9oGbb}xvAy^2)OG$%BcZVP$h=fQ=NJ>j1-3ZbmB`qLQ zf^>w3UcT1sqwL}u+HC=mwAkZ zg@cEMb(;JfE_`P|#EuL85^<1KcTl!Ac5v3Wdy4f?-@(Sh+QGuq;EK~zJ9|@WD?Uyh zZcZWgD<%#OHufT1T$cay8=Tg5&$u2GF{;43;MvG)*kfT`&_`d`SrVD1SlC!tcV%v? zxFr0ZaCT97y3?@!XDd!e-kUhcoB#F=R@(GHoW_?pILlQtR}F4!tA%5{3_fEbF*ky6 zEYyiLE@D@+$jXYGNoRf$*}nD8-OFpd>kWgL-*fR^zDK;ytK$bfM!fd(T|8ugM*`!o8+be``& zRggGxV828lk{D_^_KCK{b|`Z&S1oOLS>uz%ND*C!?X#JdS98ZhCC1%Snf*l$i{H%t zNVsj8IEYMyU;d&0{gdToc3zPDZ9zGV*oNq3gZaf%=8{bVYUbFDT-_vK7W=N`Vl z^^QW_KXMbPBpx}exH)hm@qTxxk!J;L&KflBk3PL&<@o_b3AE{K7LM)L?nj!o=}SUc8ZZt<@pKZC0YgxJEENOjXWQ*jgSgm@J>D zcDAn3;bMx&o-Fq)B_kuVnRJ~mGVN!ws+h)Y4WkbmTQQ!faV3Uv6j{Y zU)=Ml%b}NrLd8A*7V0&g;;dO~a&z@)>u7j!y7Kt%awmtx?znweqe*Cj9KC0xQraCJ zi|>VoVq{DY<62}Xv}HP@Inp{mS&p@SET6pB`(CVaF(cmOdm+P#dhvXG4T4eiwIA-n6^sR{K67A;IpC5ET`bmg6Ai z*8b0k@#hCClp!12k08G1ekw!4kO_I1B%b*E=&x=2^%!oeap6x^6PnW$4oY5ms>?-m zr{aaIjr&q%Zyo$Dkuy4QtF6050rn7iY%FHln^YQ~> zpI=jy=r`jY|2aA4GIrI}q|R0Sh~kB(v!HfIv**_4gVoyINqj!NUg6nb>Z7BKM@c)>E;gCKt&EBUzt}tfH+`%<2c?Z$52InmC=wP?aq5uw( zcQh60I~~=Xbyd}fwe4Zu`Mi>zCgYOuH*Xq}Z~N}8R{!bvNKttbCUGpEB<^V=Op~6T zZUc)jENs8xdqG?hgW;_B;bT`W`#yK-`9TQwY1|(UB6lA>nuk|1k>O8o?pMsecQ+Zx zx~Q5P1^XEjZ>2V}nUli`%S`<&$x~1vlN1hG01>Ur-gH3PvW!`}=k}o5A)J;B+1Hog zRwv#6h+&Oe30=6ghQ*;-+F(%hCOo_mE>-ublddJiPKfiQYrqq$N`Huc5~2X}t?{0C zp|D31N0CqkXjMNgK#~gXoXc4g#9&P{% zwV+x4VdXo%XQyOh)VAuCo_PM{Qv_%kXkjZCTLtb;b!}=`_ zeZ2hY)hl5dqBtSzp0Z(``#L&0i95Qan8E4XtYbaT_Rh}Pzsm)%t2hD8jjr>FZ4gzR zA9pu8x%f=JUZEB+^Bb{X;^T`5XOwFVzOg}xHR&=N5^GxdltZJ~kASP@O=Kilo+?{+ zywH3`yj6~k$O#0@;%FHgYy@O_2yY#!6vYs_G01gcje2@tlW+5MDnEYQfpZRf^poX% zNXcxTp1y*Zk=b<>x9Xgn!>N-a`e%ubs{E1*c9l7YNedIzLPP<+yHj5EadOw-0H`qE zmV%5e!0Y+fAwhvTM%#%hJ3Cv;p-;93k_<}_Nr}!CV!gE=jSphEU*Mjn<~IFy4aKL3 z^J*7lH8Smj!ooMJ6V;n7q>sMVdAg&8rE^xS~*`a{qnL4oIKm= z2|~za5Sy{bM+f~m7&i2RzgvsVues{J{A`c7VD1%w4~Gu&cB#e4TlYV;F(-%f5`;9u zZx;JA`^&9$PrbmxvK%darI17;?ja26#TY7RpQcUJJ8u0`<>Ph)OcAe`-@EugeTy$M zAO2RLD{+ZkUD4_YP=tM-EJa_Lx!Qa8)thiLEnAEIR1(iULg;!SkmMgcNXO)=l26vz z&nw^((a6UOTDtst_iSOLgt=F2mj+!&<>5xBMwuBoET9A_(*wQFLF9y&1@p$2Gp-J6 zJJXg=y0Jktp}c_ZVC&CR(gpFo z8qX3xL)qyaRMYK-J8m0!mek#oC~EjAJSjMr9zu>v*3W7r&r-Rb;hdmPkQ#h5z67 zcy1{L1qD$G+gyQ!dm|Z#*tqxoLo+j0KqPKkjxKN`JnlOdZre*Z#M;NbUdN@fX_ciLDdsc44I!k(Z-17JLM+q~a`Ng2w z2ekqnSTip2Rak=F>ty&CN|S1m*+2k>F)m6f6VA1u3I@@mzA|p_{p`RN8-c-_PiMY8kLY0P5tp-Hgx7GSlnT^ zV@8yt!u&=q+hNbZua&U8OptP{Y3@qmrX79>1k&)f`2qb zav?)%Wd4x(Rrdbky6lWRD;isY1W{aEjB?}p!4c0g)chA5Pd*H8U_s6)_vgS}2CIqK-3wLvX1jPt88DLR419N zk{I2i$^%&4W!7Yf#`2CfrAx${H3ho$I`9r=VRT|D^;7iPb>_pI0t^hdyL)<6rBf8U zjvz4p_CJ#NY_aP@z^NE=CvC~Dl6Bl#<6#`XQtz$JE_~}w_*Lv}DAunvo)2j}%N9vg zpcQe*%3-FX4=aIu~-N5B|p9w9mlL0u29n!pgbJ zt)T924u00|q`P(3#{TBKT)Flo#o0~(*r~ghg{%_rvrJ7*&EOahQ1KaOS=VkykCr@p z1S|M_Z`!VR%(8qU!`R43MmH16{$p$bF0O*q%*uO79viRB1mg#xQ)xLAllZ%khJVuL z)0jM3Y5%MY>IaWke230%co8k@#`y5h$9*SjDNZY+ADEb!av*c7oqi$NWtk=yrb_Rz zcH6Nh;pP%#>w-!9Ua_1(GtBJl?BJBwiJ0+3ON-2)Re7#cwKlDQS>)fwv-DZ8Iu9K7 z;{?n*Rt^9-W3ad1J$s++lt7N>FJ*S;&dX8AXR&iZsaa?#MI9E1wFLpp3UG2NoX7Qb z995IO`iGHRyGQ)scdR3;TY>A(a5)vp?)LU#Iy>9C@4hUfMuDfG1lkA;ii01G zzJ^0wH*VZ04}rzJhcnO~!Q93!vE|8Q&=yLYGh(4SR%)7diHt0=T5MzTqATZbC-<&d zWinYyNHUeZenEax448u9+~$A_1x^!z*~*zY-hT_cj@XR%cUHy*A<5^S?Du*NJQClR z@dWfY2pBrsTSTwHJ9c;L%QI_D&A7NXZ<56!TP~fY_ec(<6Vt>QsPjB%y)|`FXu=@^ zKMPtZ0lEU{CX~Ylu8~QdQ~DSt{x`6xsmX+iumOicOA??UpK+Jomp2T3bJq1og3tr- zC`&PJxpBPERw4t;Ck-J`u|4#JO}ilfP8i)oDJiL~zf{i>Zo@&ewY8lKYTErh5`pr2 zWMm{{vu8_McX9RMW0nskD)X9pRf%_7t5>6=h!|<L+E(map8ex>;$BJZ2fc}O z!SH6o=?g9F^=nN8iv!uD=z@Y$Wy&c`zbcW^iaG@!)3yH~ICIM5V5J=JAC?VV_qR`$ zPf!794Y@4$_%TVcW6&|YV<$u!?WV{_d0Ay?wr@}z>rRf{UlLL^0epJZZH$opRZh-O z{pOcZ@y4#}pDFlkXJiB|M^#K;nmD4XoSkRiI8meuYinMcZiYz8E)3N(>acGC=-cx( zMS2l{{cKxU7*#;$+*KMP`<1fc)(8530MQ-&-8O_?p({~T$k;u+UPGI2pAh%-u#P)N zulNBwbb<3*{g2L;W^Z{NZBzU^8xwwhXKvA5Nb)+q3C$gc5N5{)b*`d;9ZA=A{Kt#ginjG%$wU zdaU0ZCOstz1xGc%MjCZ0iyvZgjvjok_mT)o9jH*9zaVkgu(v+D_`Qe`O1fT`C7eP1 z^_kB+omN$gOjfnqsygl~0uXG?P(%o5Jn8`r8bUioNXd=m-N{)^2uK!MHuVY{{jPc5 zmbzBPdIHGx0dEmM6KIB7v(i$2Y9XtxU)>N7VCI%m1|623%G%l4VVR(_ zI(D@a8a`^$Yk3R%fgX4BvR;m+IEZLRZRcNgB<~EmRS+<#y0f3(0d3cpuC6=>&B+v2 ze22N8Kfgi;ydd6bHk?nJt5(1yA(6DT<>0*#voacVrh&6+AvKn33NRO#TRVYGG1YusM{uY4{}8Zhxyni4&CSb*T>4O!|JS2;(8Fr{|e%cDyf&E zTYP^5J5|#f5_b#gEXnXDQc^Jm*k;r2>jC5k57bjT&8gVdlYJZ^lR1!m~==z zIGg)Oz;it6?*03-z)LueK7U$1XjEUOM4uSUsO@2X_2$i+aqS}&wuV@|yu8pj;-$*^ z&_GxH6iz3i0(h`@<((826o@{5{@nLN_ThVGIOv}#bi0J6JemPx!(KXV)j^%DfxbR! zvb>7@TaHWZB=vALA0;7>SW!>t@Q9y7fA}Ctymlzj6)i8^@C_E0K-@^T{ z#WrmwFGKhAb$J_8C*(72xb*r%lVwIm#@ooq z7N|+8E5{KL5gJ7XvXyf{?|dAb%?bfR3cr4D&>;zWm{zm0Vx#uAu|F`f4=4PYovSI> zP4doqUwxDSH2qAl@5vr!9|^3qpSNp1FEn5>@%#60yF`0N>?W@I!?$X>o@K3LtmZ&n zQiim~-bjMCF@#uwlv|=j>;QnRuU=Lo+;QBI9 ze|UkMfA!vN1(`6ZeQ@MzhMrg)X2!_i;5wnV$9psR>0t_A6W2# zTSn~RxW{I<*7NePzR%=YL0G>OLN04HZJa;s@%v)ESHlf9O8yout7Mh>|;S{JW zx~n93!acx?LYB^L?TrAE7Nt~g;vYa^pa>D5V*D3iM=IAIM-(!}$cMBS&(cB5jq)Pw z{w%y{zJr9UtSqdpT_+?PdxRXIai-+++=P8Utp_{8vn+Qk9^Rm(A#t=tdM7O$(f>;~ zg$D$NwS4y7pE20=DM)zH`+R zJx^n_TY&(3CuFVD<*DO3NA7J`unvUU3M&UlFeQ=sAVvnSe($Li!byQC6;4?=*<}*PK@c1fmi}oe^1I3?x8v#}(0p zI6nj~2`D&?8oaT;uk6^p=;P}V_Jyz0UyB9X!svUUuswjBz(XCvu3MJ?N8|OtfC{KK zB=%GCwQXl|4!c&x8|V*C3i&wu`}?T?TR{JZ zEuIt>Z&j78byNv`Pk3yse1T5wmyV89AkBcug)mLU?qQq@6qx-ycPg^hd+~I#z#=er=6-h1!74c z19#kX3yDOf)`YYE(2iWu>*Q#cb=Z6`CknNSZnx+#7|h`Eb`GTASfD+h3>p+=;o;$# z#c*l^NX~2>FGx<71Ssr2c!@(Ziz(vnLj?s@Sh{RrF@%h3VZpRkz89sT9v@KWcHP0M z@^FD}!Ko!^1W2^4|4}AzLQ2LdOG-)xLEVXbatsh7%R4S2g0#eAC{M!-qz9FV_H@(+ zCEju^u5wwQu^TI}NJ50fXvV|K=#oH;S>h-kPhFj=*TortZtrv60YpeHY_HmiQ#F1T ztbN}YtENQpkwT(K_NCXdA+KL6pCzPBehzH&+)(cCW55_H*p6^Pb5^+!9MS`jkz=0d zjLi!$GiSDXmD>pC)dBmqglK|M3qfe;bAU)uWL8K}pKVim{5W+I*3Jy78M@ev4nhh>nJ z%(S!;%!-f;k)dJBc zTecq1(9KzS0e&0=p(5)%HNWCRrii%k@Dw5{UYR4PRm(NKUrq)gq;s^mC<$|5+Y8W- zHQ6t1ck%VApj*N}(Ph(Ajo%IVvdknTq(*zwER!4Bj*+7*? zo0X3Z0uS%l5_oY5mYdB(+iJW*iB{MqnVN!v!ZW`U(QTPTl-!abVPR?K{{#c8Apo=pSkaTZ^Qg4}}vIwxK4df>fLH zvUwVRH$EaFO$U@nGc&UsGv0Rt&?aen0-4FG<+ZiYho~QFLR|B$Sg1M)RSr3ty{@}( zPW*%y4@Q3MM9ecNMhN?U+s6!y!bTy95LF#0pyET#H=g~OtmOyD zh#)bjPjt)E=&gG0A}Jh{0J}kq+B&3$sCYrUeFMM@@;^&=vBLs$WZe|sdKgo|98W|q80o9_!SshZYcQpp-cZSPeQ`XKQ#T{ z-}(Rjk8LASy@Ogvx2*p^cR7b#2DTmjiw*)+?AldZ>K6oBp#|z&bnsYfJlp2R^6&O% zgx7jR|8PI;h%^)k5~My$46O0+=RKP^T}w_J_2(2Fy`BK%tTlX>$^* zu}|j{725cEZ#6@vzX#eofP9CQ(I|k5A0}Jt9G6uQRCpT^0rWWZ7dKJzpBr)qEN=>| zf>U*3diu30hs7q*KQ*Ma6LWSv5Lt18mO-Bl7{tU9Uz|Q$Y&X~8b~zVCP0P9!7(n0% z0rhZoXLaJTHXXjd@JeYvlA==-v+j(+!dgt$yaW401XNrQZ4op8sc-x*P{5Y}6tsd` znvl?iQPch$N-LULz6)6uszL(Pbh*wsc`*wH8B2L4;AIBB0*FGjTus}N9CBwmRB^1X zI01sp{&paPY()~0ci+5t39f_ui=a`8bifACzuds(O2*@hfh7jjz15mc1)xIePmV<>qqHm&HFZW;1x0N31lax`^PWH)zD$Lq-PTYvw%TdsBvi6FoxcpWY4YJvQU z+Mu6vbL#+izJV(Pnp{}}gR6)xy}`T84wpQ9|&EN3ZyPpXCJp|K0g;d zX7Op^4`y?_}4>m>35x z2ciZr{@92lIxG)`fOm=4xQh}Rz=PNWN{LP znRDIhlf(X~qX2L`7L;n#(t`VdKZ5zuFXXxbtR-ie_b)BCy#{X01~gG@P11@W1s1i; zD#P2dO5hbkR0MfGo8wu3;%!+})odw&hwOlivA>WO21>iLgR6o<4-d`YZ9sY;1E!2` zeBoaJHf4bq`~lEnQSum^c_HZy&Ynng-pu1=fOvH6^sPXcm*%DD%}!33&x)bf%;6(cV$LreDUjLQTJE&728 z-5l}T6z9&kz}<%pzA-@j3+itmuz9MwfFA&c_8T~E0J9L7B*u60#&?%mj$OiF@Nj)m zbFZ8UEe*>3&+AXVND!l<5jd-u{@#Az?w(kdW&kS2^0E=rF# zZ@xq6F(H9Q{BTWr%WXxg4-76}zI;iR3!~#V`RWZU(fYUhT(%&olTc8|>FYCq6YdwR zPbRz^Dn#AiB{raMDBgSSjHK@Lep;K)G?zKZ~cl@*@B)WH^&YYlSWWsSp1*mRs#*ID#INNK;L?KKF;zw7K zL=tiy1kkw!LQz91HZp=h{}$Fk2@S3C^akRrL52hk@+@kmL3a-9ou3Wn>-hn+^(+u# z!TASB&4>Mu2e=jb_ALqGNxv5vx-O&yU=@z7Aa9ok>}4O3duE~&HPCXrUMB}rVI8w5 zMEr!`qpOHwQ$qkv$8GUCTE63*@_O2E&k1MyXaj17lI)G72OD+r3Ki7KpVv; zAOIs79()54O-Qw*42e?Py?6iqYrzImh(G@`h<*9@;y`LmpcoDP1`B?*^B*QoPNGBw=e&S_^+w4sCuhd45;^(YY7-># zgK7TY!7FEEF#WS)cqDWN$A|lMg~QIUbtEb8yrf) z82qNe@?roY9rEIwr}i6L$=?!(S_eJX>4^3gFzz;j44k70foPR-vgEtR4)8?0ekUkXlQgL+~li*3TjNApUZ*|*O>zR2v`}<`Zi)U zfJV76RWAW)Xdal0+_gICmxTIIu%bebfIFK-4x^Wst*{XJieNUfz5;qmf|s=#5v>B( zgIq#YV*fjqaRam?3bV4#d4nWzZ+IVYEY&R+Zo~p1iz+Y|DB+CTb#lxN0q=sOZ0HRT zx5x)RArKhR^}v?o@=K~Gf}RSugHox%jsS3z&w(|^LNEZJ=Jzsl3J6(LI1iU~PjVdM z5z7SBKNJ~}kh@UbUNO)Rsl`rnn%z+%^J%;yp7dzxn1<1~MB}~tzYA9S(2P&Q8q`!_ zJG;=Ab`}sxfkQ1KhCA2_xSt=o&;e@=q5p&(d5-;?H6R)FAoBp0 z7vhH|vBQy%kIxkHNLMkyq_yb)dR4b1XisdHKbG6U25kb3j9?E!{X>ow=@}UY+>{*x z;t==r04NjOz^Z9iexRa3fj*R#JeIe<%DHB!_t&A6E)?_oN?$bW~PrSU-d`1#V#rG1m z$B5*ts1Sypkw6l6wm|9K>KA1Hd)}W)cg(EZrsoZ>bP$i6xBSP+PI%T5$oCnZ6Xz5 z9@y*@s51}V@#2(^(XKclPK4$j?^H{>q)ywv?35}}MnrNaE?{eAOvrd2PCJ#D8$Wno zpl7@O>l36gUhBzQ0Qs@O5G(^&FP8TyKG4~gl`G5JZtq=Y@SyP^SJtYAwBrF+4bm${ z4Q3*)^{1CrFqp9ID+&(btF55@KwYN~_hbo@va;fY9KNt! z_WnAjOl|#3t;s9y1Y+Mg?oeMBUDEfPadBd;JG<{84(Y;VD33}tcN^v`{G2KT@J_$L zf?jqGkU=_Jjw4+goa`Iy>_4IT%HI?WRhv(+CjwTF3%scs{^e@hZsOE!Jj|zWPLie? zY*djmUv-@bs$yBk8BzX$o&40eNoitty3W|#jfU~$JV26lTq&$G(j7c>uZUB5K`&s$ z^?d;*vo-Y#bA@o^Lg1*0ikvIc$um|5SodgX0X_d6Jh+8~!SRvxDF_)&j5-Tf!bZ9j+m5KaZeE}2r*U)ivu7S4Kqj{ zB?yNDNzG%}N(d(vs^JV!y3900n*+*^!o$OFySa&gPtheR%nuM81m}?xiEW}4u=jn?6artrW^*0`8KMb!3P8vP5=RSo;c@r` zm@XkeZZ1S^fqS#!o2A5B!)Y|?foiD%3}GM##$4Px1o08%MppyA_D$$0)}WkiK$%$s zB*5pgriX+!2|3@1+9sD zr_Pq=4tk=A4B#WN(f9+Rjz)@~LbjvuM$K<-epL|g7W{5WK*^mdgpzXzCu%_p^=V6Fz4kqO z0rv;6CO@IkHHN~`$6kYExo^~daEgG{d4UQ6WJheUkPsklreqQj24d0DhLgbdi~2NR z3vq~PPr%|A3y2d7%nMm|X2spcua6PG2`E7nLKF%BYAa!R(7Xnp{jUdD=qKpLmj-j$ zCYU7QP>&QDo()Qszxbc^Q!+=L{XiU-jR{i<&`O!19}j}|Jr76D*WFelN#_tX|e1V$9WRXV7EgC|H7;NA)m9;Tz&UupC>`CVzLje|;Kd zS)Fm4ACV5*9muH$HkX2}9hfANy&P<0Z2S%A1ru-LWM~vXUk4o!fIDh5A%zONx&P#$ z3b_NpBndsZ{gZ`;LzS%t7^y0>ILED=`2$raRK8bj(ex&6 zP`0yZvkQ7Pz*rhb*mwHx8wa6FGZj2AnuAM7smLjLPV)^SPv*e4nrS^%hj!QlrR=4F zSHL>{YkjML)yGG62*l=~5mPh~V^w|`T05gjw`C4AqJbv2AQayT+s<5cy#=HdLIF(Y z)O3%x3%qRMC!4^Og2U%B07a#zo-*E`67oWGwOV&zg{IgLmkK>1nDL$_wn7=#C^fl= zWDm>1MW!SRHYx8Ku^*i0iKq#`R#3?&#HNoIc9{%>fi@gamyrW^QF7P2N2{vp$jHE8>h0RnFK zzQeuzKmiCYe0+Qt0NHFcwK>C^eZrwPslKf1u*(kmc;I|^Dz`;P(VBWB-0e1F$@Jjv zjW@A`3wl$WYX=QRKEdo{wRF~3{D$dUXD`fLh)N3A_&DoG zURg!J?TWR)W$uNt(@x~)*vz)7fW^v01{bJ|;65bnuT`gs1iArQ65y3uZd}RBB6B?? zt2*sRPOi-o_<Y!zG?QtMMAbcU;V>IZXC^1e1avH*d5e^Aq$xLTt(dtHp-Om-(KZleOlzxcg8B_ z5`W|s{o0(YyOKsZMYjuH4>1lbJFvh)=jnaD|3ZtQC7j1?Yr#;=Z*~-U-SPsiDA+X? zN~MfF3K#XpPtH-l_4(S%abX%?}Cyv3-1M!e@S48etb2aXgYB($_gscZ+u1 zwS1h2mz-V^fQK7RfRfJ*Z}|4fe(YED0dh)wZj=HRtDKdADVRJ-DW1` zi0=*SbT2e4m&JSYo2^M-%$RG%^Wrj_B)`-E=Vivva+HYRVGkQywL2n1zo=AC4G{q%|4o%e_J3X zw*%I3OCLzba0TX;+84Jcw;2l(KjSkj|1wMq@1C0RQ&~AdUF(CtI;}17jUiJFEvGHbmCi&M0D{NOS{zNOT|Y zUCbPmdCTctmM$r(LGUBkH=Tc_#nQ6nx*40wT2htF9KKlO2YAN3@U*nj(m)iWeng|K=;l?It&3 z_M^wd*#hc6@$>}D&(7X6<=ZMQ^Sau7??3K?gjh0Ydr6bO(q2*j>zaRhX2qD~qg&VK zvkls#{5?kBim@$2tNw7`AeXwm@}0w3$cR6G(}K8GA)wYF&h}oykosqebJH7U@BCdq zi(}!Dp-U0Ca$CxFCeY~OC{5AXx8c4j^^ZiWEhA&Lr_a4Cv$!3A$z8NhZh2&tBIIV{ zX(HF?`_2zeHSC{oxF5N{RQV;E;DmEnzk=*{RVuxr3!Y%oevb+h)bQO#8z&cg%(_tOR>d zKAtUGzxSU@phBDCn|Y>5pW}*2Nlg93>Q^5>y7Du47nF_S!rPPOGW)~Bjb;mKNTh;x z1e(l(JjdSrlMz~8HBt~NtIvc9?qL;DZeJ7OXwi7MfE7e!LP7p@`3et4^!Agm#%h){ zB?7|LvFxbGn{NnioL_RJAYnl|h}kOlJpMV`NSO?3HK#Xb&Snyxbc#)K)>_OdfV7p| z-MJUGjhN{3L{&H=3LQGaQ_UQ`;rP^nJ_Eu@#lkjf{K1YvJ>~VE=UD*Px9n*#h%qbYtRt3byyhyudp7toCp%sbEs6Vb&+!|}=NHWe-xTs!r!oi665^Iw zeKw`~nw&D^x@S@-TDsVUg|k0O_|}c-1q!27H;8PSlY93{&aK#DgsvBfmX-9<2?@`H z_Iy|1eJV;#M@?87$I8WI#lL{(N9LoCQ?kDHI+TEcXbPI+Ug*{hqd&kCGSd1IDpd2- z5!Zlx9Ke$!AL6*%@<&QSTo`Z-jdc_P4DSIvj0R9qeNzPZ_=qG1C4Lq#$rTQc5P<*T zk&zAqbgzX4WJJg*BKOjmkFO>MMI=PWMtE0f+gy3Xws?w|qQH?ZKLGb?*`=g&joyOp z*Sxs+O+3^Eh)>Gh&emH`g>_rB=<}F$KVZK@CVM?E=X=*3YNnBP`Qq8a*uuE;3t>jB z=_uQsdUGX3xbDVu$K_}mR+4L5Rktx=zXxI?RIbUpn0Ob8+^Kz!g|EM8!`@=s{BK5H z$!h=qkYAykV8^FHiX4RBw#ZAQK7d5*4H_pJV?uTfKx0@90R0fPx(6~D3kyp!h{3n^ ze)wv^On#p&=aWytpdHBa?IPX{$PuLV!{G7{P>ye7Flf#KbOu1;vJO))VMT}ruRxJT z&G-%F3_adKakiz_%hth_lVIaRRK7TXV!PW-FabnDO3F6g12hKcRzY~ z;4wuXX;;VEsFyvRP`J-5Hccs^1}C`XBFds3Y__JKSG#^KUC`oCwG4V7oHxi+dNW5y zH`3ljtSOE++DVw`-X;ly3>8B6!0erZ<`?t_Iz5PGAe=5nwEOkWrvY_~X4t{uAP!b_ z5YR<5OHFQJVh6?s+_7 z!({eni2Rwrd*OYFrOQNXxvP~#m;0}ldaxT*#-$FXTyKOdE?{b}zPV&Q*IjgUeG}?I z%UwAizkpZ?FLA(ELcpZ~ltrDSp;)?de6HewhV~_Wzljv`+MSB)^6F<(Z1&y!1oR_` z+GbREpHUx{%`L2!+$E@3`lLDW6to23xfZt3gfu)MXL$EZ&K9_2xxvAsf_Vz88Nc~Z zFp%rEXz~!OCcq@UU#L3SYm?Xmg|-dm+~r{L<>loYECj)4X2^RL?2z|>423!QSH`%X z%m!JF>%mT511t<2aP(9lc*2U*B`zJB^Iw1*ykR_5whrb2#BaimZY~sqnHN&_9pEw6 z4vuOdhXJ`1gM2jLjUw>=e^rSbsb341?Vn=6AIHAe;lwIPC2yy0qZzV5bY2hVuuwuR zcau=B%xuJ$X;X>D@%+MhRPRJfCG{u8<$Hs1d3^N&6Z=6Ky?k!^yx)4%@j!mScV(MG z@$K3ehty~BgdUCDR=VfPC9-f%d*0r;*0zC1Hc;SCAsP#?DBF&jYO&%<;g`>M5`XAtf9#HO+ELVw?xvNAFKMYMa!)g3nCf z6rOqfEmv2!CD7WF615awT zwF`ZrX1P^2D0En$lOxAi4ICrn375R$J@p4BZPY5ep+|-pyaLutqH^7<;VRAr++CvcfBijlW?tLloss&PeaUJ1YxMAjIt0&@MBVreH zA%7T7f@us&$oVfMlRFMZECz?MMftt6`WF>D9CQmh9cE$t+YX+Bge1BR7*oNBz65Dt z9eO3%f@i>S!Q&xf9YOYO26|Y1<&&QxhCD`%fCyLWtZt{RAn9Eqg6NL8*RB(qhHt-| zR5at=HN^<|zcRNk&*@6L;OgLUX(Qc)(>v zy{KV@xj?jT-zZdHBETd;S0HqNJ&vW=!06o@aA#8CQKvhY@mhuM1x(Fl&H`=tU6h-+?T)((S z0=O%`E4r~)Nmd%vef-Ku+OOp~K9uq_swy%f2~w8i$ULI62EUtniSfg(VhHl#>})wl zV8NL|xjSVt;VEP_j-CR7^)VYBnWWW(xNDAnDZEniE z%~*5Mi9H~u(!EZOgQ*3AfY`3Nvx^=q^zPlezGdK|^Z+@+Ga*0*0UsE3m~?_$Q_xU$ z#aG&~Y~enZsEFiQ96xa*FX(3W}khk5>jzCAzTxf?;Mc0bPG=d{c+n#CtgOwqUK`gNJ>f2cZCuxx~LBPjmaa?C8aq zLHu7qnVu>gzXp8#-jm2jX3q$as}fz>&UbX4{kBbxiMk-e5NeMsKVXRbg~rIxgWkZa zWdKS#N-NuS1)zYVQD4A|FtL3F)ULh=XGlap;UP#r;D}tgapM*8S3zO^dahVwL}K>|iz32KbnbcIwn3o>>k0Cm!jPJbIX+S{0IBjJ9uW+3_667#>+Ja8lxv-^Z@v?8Sy`D7cHX7&U@>H0@$!ebiHQ>}njM$ct4 zG}{@BLBkIqdqhA-&<9`|o&z)^ zM4>`8@t?4oM}|Le%csHwO5ZlLU>zV)BHK(XjA}vb7!vOzt`c+KA&mZSQD<{@`OeKpA26$olaja2Im zK3tX`&UkQ8^MN+3B=k$b7F$foJXzMI5(syPE8nC^T zYJ5uk<7Ib&Ir&iLv-e6xGF@fGX%-QgcL``${ZvgD>ET1?2o&FBI6P7LLS_RjVD5JX zyU`*%?IBUbF~^~59n$h5JZ=g-F$^XXnZSRCTtlFS!f;zDb;$k9{?8ADh|Zh>PG1}v zme?C#U@RU@#vz{uc--Jbs#iIfmfE(1Q$f*m_AR)qcpaBinx?s-F-iw`y|=eV9a8X# zOqM}NCp-#%$k0K=GOB=_0B|1;i7q}v77xy}7A?d>VxzEc0#c6@*5)dLJhDuwutAr7tD zXC zC%C&?aCh&`e{YY`FWvprFX!axjb6PnEm9=7Wuv?+BVh~y`L4+Z{B6ZY`}h0o8C|{9FdG8jaP#oE7xxf>45IB zR%6$E=XGasd)MJguHT$-XHLZ{3+){(Pdj_xAvofLc}tf-0Wbdk$hW#+EgO4}e4Npi zPaOr!Ed*K-6&&JBfNJ>2Oai7snR8YhV4D5~JO){SjQ$Ob<$C}Uw?LZuyG+W!^6XkrxnVhwYKI=a9c&uUk}3eg=kG?jt^l~DRzVFT=~8b0IQjI>X~?4 zyS)y>tb1oI^y35Ab?U*^6; z7>EM~e(VN7?>tBQr2sfMJ{)@>`8Nr)y?}Y^qcWb2M8N$Ea7_s>U&EvzAz(^|W{VAn z@=lO#hNCA+2*`2dGEowZDKwTDYMZj6$q?g_2P6I?NHg1TDS5(Tt6PxWbb&D>G`j<8 zx~NKU!dqa}jI&jEh|4E%`@Aa#to02ql2oW`#8t$8%oG!j63KZT6dE^kl$r6mx^ z*B1|uerJ*0HNWvjj*VDw;7ol0TOd4js5qI#NcZe@C|a`r-4O0Q^ATykaij|blL1E0 zjS(r>4^sYitV-|6bnJp1?OCV1BOxa_854Orp2*@(F9&9l9~1%qRHv>g)h39qW-N2> zKq9g>$0gwooPJEcZ$E+N5D%a&bPhd%y`C6&&46Btu7w^TKR!ra0KVS3e=cx=8TkmS zek4!3MaI zMwmWauD^i72p|bI;FKXQL`Ae(fXHL&uF-l@*8YbQoc4aKKQ(5pqpNUQg`S1n4su+{ zW&c6GT&pd!h|hjA&UwoD%vq{RBNF{f_Pc^{;^(V~HgvB*4PNN<**u*D)ciIr7&ukc zy^o;lN8Ji2H$WR)F?>4#EHB6YNju<(`{-l=pg_UUkLiEm-GCOl9)KR6?oJ~C5Bww0 zc^U`Clri?f_X3kI#jQTDM}Ru{ka~_9WECgSMUVf&+&a~mxtR+JRci+Xv7wHUSL`_N zNR;&Fe#BE?NsjcXA)!1r8K$;)p__G>F75$5jSC6)llzDQmgb zZa1Isip3KyFn^MGsylzfbrTNsfKH$%+K)B81y>HK!bg&o&BWcq4MB2K%1r>xHrm;_ z9xi-otPXzgDWgAZ&8W|)(ZHVb+d%&eJzcaRP=vbdv>f#Ew4vP#sZ?4UT*NJC)v6y8 zpP_`r{tJs~^CP-DrLFv$;a(w5Cie}X<^Hc=--lEF2>>F7=jfdOn}eyMF{tqW0OpJN zCLiv~^n=-d%;#aHo~N;llRvg;Es-3GucCCyPsdwPHA%rDrM-8I9Pxvx2XuM;DpDKE z%6vcMFzKMaU&db6J2d_qJ|RgPR$x7wBw6Col=FQYJ9N15Nk_YNcQl){^K!7Q5xIpD z&+UbrxGELS0@-MlVF8}_8Ih9VlNGatW+rrEa*-vgZ zoiGPd%vdJ3rt7ri?+#VBEyqk@4}%{LP4Lb&AtL0H z_wqS^wX2s&Y$aMeS##ZWy``=<)4>8`VOiL zKQs@y*<0D?bS$ai!;lcOX>&{*Pds{*Rg>0WcmW4GA&W}QBa~Z~qiPrkZ2^3MQPMFG zSZx?yfkDRC*t!A&p7q1ew}4Q(e7Oe9++uEja7X9*KAo54Qt_1?1`?q$ur`rcLSuJ+ zd>X3-0$?0&)p9wsVrycQqrswzqNUMQMHT0`S`Uboj*;Sg1lPf-?uG9y<9y;Uq*d_Ii$4O-hSnM=v;&oK2%6qjCw9Xj*1x2 z3fVp(`}bII->u-@V=_sOieF2mopA2xd*gDKr8ongeLdG?#@K<2=jj7()}6-i%h_G% zgLS`a6evIx$xu-^b&koDEl3i{SOFu78e0XU{BwHI;v`)JCO&egq7cb11z0xpzRl}g zf!#(yrd$)hc~*oQ1@_QnQH-3@SES6#+NG$0X?k(hxw)Ezw2f|vRZ{uf^>usFTiD}tQCNe?$?NAH^b>cD78Yh(#Yp0*I% z#=^4|QTxZ{Qo7E;I%DNOE0IyIJ`K$VX^mW+ns}dGaSRAllK(6a*S?twd|JNDoGaE< zV#$?sTC5jFpyb>pK8-650)b3uG0(oXKTWHfljMF_7kxU+HBEsY}` zx-Q-WZ1GV8zeN}(33EzKIM{8WHOCqk;vhfA%I6ta`*jiqm_gYx7|-Kz4jo}f>6_ffYINULZ{s)4bi~ja1_}QKZPdqgm z8qk{G6acwvyU>6=<0C@_09N)I$dt!Yis*Oe8}fiKue?kVB`Fgb}FQi|Gt87?@HD!;zeG~S^7_eYh%))lO zybwB?Im{|4zKE*7`D#`N{wPUmaHlJ~RGZ$m`9ZnFbsi;l`-W|lIvST<29>zVSh*^* zR3VaeW^kQ2pvtPN(o=LxMCRZ7TI=8U_gt(1v(1GQPLlvqhW*8~6|+GZBT5Z#KFEDE zI^DdRMm*DXd)cV`E~pU%`t=}$-tzh|%xVmDa^%a6QLdt@is)?IUc%2=se6W^ttW1e zGPlB%T|jX4G~rPS-#{R~ zrj4;7bUY9;;!3X%Zcs#RvMEFQEUIab#vCd##rGyzL-S*i(;uDXH`}fj-GHIe(;eu4 zb70_{%VF+2Sf@W(K4RXm+-P7payR*e$XQwG?8}Iu_RBNl54z$mT<*eQSiP##HW_|M z){0RIO(>Q-4bQQ{9w+j4W)~Qx!!|HEf%8tMe;Kc1sUl0TQl2}@wYaITXfBY;KQO3j z`b-x%ullozNksckQMM{J{i>UUid-PaLcgtPFz^v01!I5c zl=3AnqQnElO<-7nlfshadutUneoO+uOa>;g>)me0_o9h6Di~2xVL^Tf+7e)M{(wyY z02&A(H1M=QAu5RY5)~fnHW(kb8b6wh%$#`A@HcEfb4F_R1+ngn#BUX&FsHC&`n$tbh4sGg_>9 z8#}nq87?-jpoZj})SlDJ$`mw6n*SpB_2TNgc;x$4ztr_++Ul?mV8ha$e1(fk0O9)% zQUc=2;AGjY994{beA&QlmMFHNnJ^X~u z?N|T@otk69Y(O>Q#% z6@U0VFpI0knC@f|T$`h7yt3#PNw~%&%Qc)SJ=Z~*!8~;ioEcY8t2p_zT)?BQflQhF zmYX2k?@n(UDd?C%0-NzmQF#-uqNsnJRDc2?&KP$%T(e_r5LR2+A*U(F}Z)iI{Qlpty$wX0h(t0 zJGSq|MakWKlSB9l(e7Kb4<%LeOG+dl?3Qo1UEUJ4ayGj_tGn8bA4Vr2TmAfSw;afn z5YAy?Apv&p#KO$|0Du^@14A^!TMEp9Q7>%E_C zZ;HZ<%Sv7laGJ@KpwxI+M96WnRt5zx`|m2{Db_A;lHK|fA4cMW56e%|*Dlw%o zuu8zyvvfkLAKC~5Nx~v%9*ayy9 z7q;h&3|5mkL_FTV`=IhZv05Q9EcY>P+`v@4BRR60gZ1Ukv&Jc)HrY%wev_~Htb}I{ z*4rS%5a;emRcSa3hY1NZd7W2MK-pTMV#b<_RY4xvz&H#ISI!{seBkUD1_WtaE7Ozp zm1t9|@E-}Ce{y*Q26fl;0B^2b`C%C4um7mFB0V%qIku1eAi111a`Lm=Ej z&OG;GmY<+diaO7ChvfM)CY&2ida>0vRMG5Dhj1?MbpRH4kanD&S&1Bi@%H*Q3;-sY zp7_xT1cd|IlFq4*7RHRnZW#Hc7q3t4FEu`2?{%i*!OQ@X4I8lO!Vy zP=W!672Btsib9G-iRk+2F56tqH_GYeioT{Zb?0nh@IY?921^vPTR)c80(t&#W?Cw= zY!Fv>=F!KbY$t7@&RXo)4a>olEUEM`zSS0i*wuJf_ZBmnNYD%lKb%DD*l}(O`>keN z5Z0Diw+#+eM%qa2WMI2&*%J;}7tFOBbep03T{dwhV4Qz#@S6_RxdOD|rxw^<+qCD8c+$m|7m9w3!;jj=2 zDN(81+b5Py-$DVZq3q~02J{qq(bc&v2;{wA$2DB}g?0wJba~VGcRuk}_j_g?5`mRe zC9!ZdKTBb61a4xU_CC|4WTqt0KIv(@JU-72=szy+>#2e2<@RvpOg`3x9JLBJR4N(A z9B0ozY9zE4V+75dPXUG$%ieV3t{2vJ0L8gNl)w0en4-#5S&Vko=Mbhru)6%N0;ri} z(%yN>9k0MROH?5UJMx`IJ0@(G{5r3sH9BZg?B>r%u+`k}W;PtOWHV<#f>{H=kW-<| z6JCFXKc5UU$5g7$axf8jAG}z<4DS02_yl8YbQ5)&BdrWGV>kNsYx3pJ{Lu`G!pUPcG zL^GopvdH^OkPA4>6{QE0Zyl0y((ST)6>y<*L2E_;6|nC&5-WghI_See8JNwP5zd)W z)^iCdWOxt> z|0p5fupo_EXgyhMJ?}{DEV0w{`bXYKtW<%7`#M5%tGK^MNwJcWYq2~6^cge%hhPfYMWjN+Q2m$8?vKOAw z&QC;qjXy4+NLJcX2^gJgg79X_6+o%q^Hx5cFg7D3zeARHgtE~!7-s@%i}`{k0-`Ig zHV+|p5;meise$M3{9n+Bfy)guex0oD*t6VtkS{P0QlGh{AcCI)oBR8Pm9(Nwdx2^o zArljg;Ga3tzVX~HFeoJcqyf#2MZBaIrWYPyxbdkkjoB^1giCG>x|o_-pQlXOXJ1m4 zs~3)}=D)YC;S&hk#$y%eVa-H8X6F(7>5!+p!|)hlxQWv~iFDOnZ>dhrlV^;A=Efsm zei)2H25P^BVgam^g*Q{~uC!bVL=k30gsX=4#Pnp;$>YXzLJJI;VRiJyMx^=5>0)}j zhaH1S`*XR|I_C~ozD6)sn@U*^+j%Xk7$sTpJ#!D@!0ej*n(E2Oe9$iNXQlXF-fi5{ zJd6DmtOxWp6yZX8G+c(e(+q6Y`-iaNWu$YOS7pD=L+* zi&9p#vEK2#vv`iJZ(X;sJjqv7rTE&K!Ma)Of@?|JjP;H%iWvSaqbdrHNBNl{+Ve7h-s1g#>bAKxL4Z_&y4cTb0{#{q$%2|Ntdbk5e8=PbPdn#gM&_U@hsbZ36>3XL?pHp9u zOtOK@PQr92mbqrs6mcF}h<7WBEsTP~l2cb8{|{0Y3U;z7JlMGhUr>H}5C%Q%SL{jL z*;zN@ugn;ozlqt8>rPUUJ&>)A`aoecF;s(tRKGZH-KCJ{?aRhAPuqD6?jmcp?gK}_ zeHI@=UBk^&E1?j9Cc8nVhD$myh=NKcjtLch=PO^Lx?&vDx~*xabUB^!LSW!u=bd3@ zIKT~I=~(Y-%W?Wtc{jDA?pDr9jgxlt8P;-zuhj(N3}-pMgkLY33Uxc@oPPgtYR%%; zcIm|%P*DbP_Pyz7Jn~h%$RV)erz7t~h05)l;{@7_SGdFHmS)Sj)zl8d$i; zA{Sl8uip_oiYYtyf)06@_zheQ)_rwLd17P@=H&Fm;)sf zzX_!l0^hVJidP<106)E&#`0C#g}d0~5n$O1AtdLL&xWx6K*IG-fu;dU-ABKpfYyA5 zdg2J#>~@wp_6KM=-h?$vVbAhtICtR)jB>HjFAsS8PZ11R4~}@Pir4`?7zfDZpM51k zRa$J^kW~W}^z0ZU3Qqi~G%jJ$U*yWDeqY3>Tsggh(g2S4C=s5H@7O^1xh{0t8VZ`J z9sKoO?cx(AMc);X&ragrih>I_E^rWZT8&pdq;jAK^!%*Ea||k%ptIc%3;pHH`-Tbv z4alSoSWg8!*o@CsmD8f{_BKTq3HC+<0(@+8e6vrZYZ?oZwx{uQdQ-h#wK6+sN$*`ybUhJOKU*BZd(lCH4=)C6Z^h3jWSuY zziU#UNw6>rvbA9v_ilqifvq|D9Cs)l_k{rG`!Jj3r!)hn7f}&3{T$nMZ_d)h!ZO`xfY$ zXZexP6h$Mi5Jyl4$T&$sJ%>0+q<8|(PhIRzesNVbfM0oW`s3t{;zw}9ExLh^4vZ;f zc4SHJjqu=XjOppkO(Vt(O{xi}Yku;$uhz;m&9Kdgydpoc zNI(#V$!9Tg?grvKc0O{QW0p1y6SNSFBoyD}VLWMeRN7Z2fp$gGd)?`>WyJ5whi|?h7Q_B%0>C+x)*h@!=;_8KukUO7_(b;6x?Y|voOfxyFI|Kvr1GCdv%(P? z06^FGt?VnhBKFCE0JGakIUmA;bl2@!K*yGJ)`*d^XL9Z22D*yHczq#2UV8Qm6{Cr< z#5?h4$8cL*k_GAmu~@?$8%lh&FND6-o^6@uS6g?%OgOh~?7mQN6wkGEV7Q_1izYHZ z^z%~i^!0`x3Gfp4W}RGTMC#D!wc`Gs^KpvLeJ*(=@?6sbOrIbuzE<$%TCd(8UVBuR zF5Hy}j*jVf0!Yj3?5|+uK0$K=-TpeCWd86-j;_ItV1aGQ4I4yHNC_MB3Mw;m6{9Cx z+h5hLOO>cayWbqyvU$}>N5K8tM{}i%n};o#3*ta6?!$_LLj~`nS)Sc|jR?oWsxZ9q zW75TA4`0(OjQkB8^>Z)$3^RX#tO2$ihjBBuhs%@}{|im1S{htbYbgW+Guki=E=xTu z7P`k}^}pJm+uH z8KLJ_h+63sF3=#YqRA zi;-#*Qp><%F=yRDZ+vQhX1G4`SGzV7x>dJifd3-RuxI<-*6KT&`O?{MiNian#${se zQfA>vYgPjeJfJ%OILE^Ng|_Sn5NHZz2>?(Q@iuEw)UH<8E3I<$N_Lj@N*r9OXDpr7 zrgIkzXOCfNX#L{?#(rM#FWg)7`=rFaSFR8JYwzd1+mfP;J_x9VmQ@DYA|Ps0vA+pB z`>P%tMa@~*^b3P=jg$OeO9vhN0$7Ja93)Qw`B?nZ2?=aG9YSBzBvq-5f$km|h?~qr zMSBt;p`A{BnI506==e^JkAw>@)A5H0@~6AoOTWJnO7+(f!<=u(t+)oSU(1&&Xn&S3 ze0zLysu7bL(5}$>RA#L^vMto}?8J4k!01eaq$q28hv7LS$C7Myqy7lNK{(3Z_HH@wVH;7UsBFF}b3Ci#I_mS) z))&^BIBYG~__a!i|Jenv-0^PEx?Zp?ScW4tO3l^Nh7&`8$0_m<6x68YoeOpU!f!Kg z^D`lt!b1Wg7f9OWF4Wr3>HmC4R2w(n$tJb+J>|)6DP|!+DssWwLCbFx?hTnf7mA1w zC>>LqKJHSaTq35#t3@+mf!lwpNlIAdy{n`#ymB$yiTb(Wh+);eyxnspC^QM)gE2`k zU~T-1xzPl;CxtFMUpEcKXLWcY!_gwu+U#Gdj|+YrXOd)G4dEDpKpX&t9jRA9h$;eC z$5HU{5xj=~oY$bKY|aDDK5T6$v+aUt$@MwhM8D)>pCK1k9y{y_Y?Kb`d(7XOwQM;y z9*nMMZuKtu--2x!bXTgo@=5U`|AN;MUs`h)CfWPj!juFf$Nd``Om9{`PAHv%=^!+bU4-w)ZI@pO^P~`$AgWaTwhT8a%;FNnDGEexnG^9(xHo}4fxLQAI-ytTZ1*a!TX53SyktQA zr;If_uKBN%+{sg_&c#p;RIzH7cn?oHEzCVMyZV}+GyGRNB#w79)Z7G6sec^q>@;lD zq|gc0Cd`M^#t?xo-?pYiJH@OprR%3Kgh$|O>vX4CK@4A{*Ge%@;rQDX6J48z^Z;&% zmW2u4%>!n;UnQ0ZHN4*VDq(&8I(T98)~D1Xec zX$L?9u58VvhJbkn`^Vr~Z}DZN8_&h_uPqqEig1CcLTaqx0^@d+n$RPU$L%w(D-Ig< zH}egDAdTpTXdzz?$tUZqdBS*eJrKI#7-GiLy>VYt>w;Oy-89vt3!ry|Da|k)&pB$$ z-PXZ(x(c3%h2({i(1*Zv$~pV{@7;Nn9k&o14Uxv?_iAwogt~;$?`7~E)5LjSQH|Ed zdk(leZ8qe`JB-rVN*|uxsVfwAxi~A`0Cb=vxFJEx$_+nv#UjahyNX(3pJpl`Skgkv z`NF<)urx`ptTBhfVyOgBqZZGv;?dk*QWnt$v4ALK!N}<}xFe-*HKn;XjI)F|UXQYp$#q0U?K9kkj8v%TAh{MRSS@ zavH;~e`c7sA9LJnp`I+l+jV>*k_pw@7v6{@u6*f;B?tIJRzNDU{bs~`P(ZMScEU!d zq1QyXRRv*7`19H94%3q64WL9FCni~F&#N?`BBxMKW8A0<1SbT(@z4r~QiT=EO zW>ng#Se<;o(K;9ZDD#ir9>M$Oe8bV?nfewFH+JSVf1kXJ>-gA4dTxZGQsK7tOZ@-J zOG_i}DtCkN!e4sq=63@KBp~ryH-pB4)W{r;jNlp>4;<|;-ArDqh6z~~4Hlph*#eOo z4NYxq)0bx}1v0_hY4=sEa@UJ~5W!xPg+lKS7?fOfRVL~aR>BH(0jr^qgL6tY5u?RO zZM*J*YQqAOtoV(J=*O|;RMSI;mz{t+4P45uH*a*?r6>-L*aTf38g7N4*4#L9eG*AS z6dXe=%0^aYS=9#I5|g9MWN(gV`NN6!5Kd%rj$+9bHR^J9qs!KB9=fdS9l@b|MKf`_ zxor;&tag=95PTHNHwO|4;(c}kc|Nz_f>UDrL&ktfQ@l54U%!=$;MQQP2AlYvnVMI- z!q};xk8UuKJKVdrE!n`b+roFLcb*q_5oDBc`u4lNIS7Wr=X4mIcYoSi4_Hipu%llS z2`w&L%>CJQK8bww=}*xfF<=VbFNZAMF0kc_ohH*26D9;>S8TSXqRzzwo?(kl5kY?@ zTj`DT%o$N`_t6Nj{ zx3R=?x_mTe(2~k>8@WbzD6Ih--s8Qh?jNgploS3NM<(&VtC3j`! zZv_5nWO}|~1e9eY6n%L>EE`=lntfKm|GPvu7mjYmrh%qufqw-$43N8y-~~f|Kz~U_ z$1P0%ra|8@q>$#G6b+2hxrFX%yYEe=ZN{3%`ODT9|CyGO<|NM|KW#(Tv&U1JHIk9! zgoN!xtONu&_wF~Q@JQ@3pa4xIRUA$kS1tF~JAN%1(j0Y!VG45NBWL8`jAnxoOMhVI zUPi^wyI#-vb9#{y+Cs`j>^Xja(=$FT=7E?>3^yvqq|dJL@y*D|mB1yJ2leIR!Ij4& zc3tKHg(}oE;MFj_9a*eCub*A@UN{Js>}v*!`b+oUtN8Hhhq^LG*y}vj)P9l#%X&~W zM3LVyWW_plTYuVC_Z2pD;MeBX{Tra_Yq7$F0~>;luKFQ%Jgnq4X+_`p5y9VExj>I4 zr}`~5AK&E)kPBaAYYI3H&QUJ^_+=pi0@W(})1lFy{`C2cE=uR`V*byF!4i$IrFaBu z_n^2XEb&82n%cRyu$$*Lf?uqA<4AoaT5OhXt(vh|^6C{2Y&Q+=-Z?b|+Av_cn-+97 zJQCW+lrLaLjtwr=RIvkE<{DyKCC%uN;fl{!I&g=Il`<22OvC-GpsLaEa~Zb@HAaWm z0i3tBWxVU79D`f=QXI{LCw;^48r9 zWNv)Gc1gR>7ppQ78nOD!1GnI*iM!&kGyWe9&{F_C20gEj3&&adEky9yC*{tEM3K;7 zJsalZHvJE2N}~1OWB+uiTv!49^mkEJRUjp4AVGpX&%MT7S*!&EGctYXTu8MD-~=ah zSKG+6C)DthOiN))=9`&nNFS29it65roaS!p&j&!H$YX*!`&3E`s5@ED(h|}&@k!gn zqL%|84uBw+G|~PwVO*T) zl1X;-z~`Z6H+CwgX2=t_e>?YnLy$7$=O)%Z-R<|@{9P!&;!h1DQbu1<3N}w6j03EcVn z%_pA581F`OGnTxWB|!e`-b2X8RC352wWOM|C`eorDjB7($&3LEgRvNhiskePQ0C8$ z6))TkF1Z3t;+(vyjl=VGKUJi>l~#y#kF-q~7Pu^L!lv$C;sOWVoOt*$5N>4|ni-Ef zW^EtV?y-RD4~?wq{y7_vSmz6RIpUFy2LJ$L7Bg4fK60GU%8oek52aU;p}} zRYH$cy`CL-zq8jw-{pCxj*6W^$kCjBSJO032pK|t^Hx85$ng@M(mr8g?=CdSFGvF(HV=W#xFCb^j-Nu5^zcOufM9@H?J6-?biGt zpn3Y|Mfmly20g38IhC0`30`*WAecBTD@<_DR*O%o4ZkpHU;O_0Tk7S9^+x}_@0n3_ z<7X#D?3re+tk$NP^k|V%+SI5?Z(=VN?Zme&6-0iN14&T;CW@xKa0)bH24i9=85taYUOil>3Qgd*;i%axIl&w^v^F$%MAe=4}1 z%*J@A-F`h=C5G1oFN#_7S;Jq6KEuk65Cc^}kct4gyw;`PRIF%r1tj^_o=mJgSZSyO zmii7t$PmNboVjNyGzU(46ZlNj(<_%==P1x{;q+IHr z{qTGx41p3D(BXg~pM<`!1j~}PUP!}WhRnjAz0CDxH(Cl4MsvkGV)K%8ERh4J0d4JC zW@`ZVR!d5XJ`L`cm+Q)7zHRUl2()(29UN23pcN!Ebz^?(h*uNy5D^$Xm&{y zFRw;diOiSCvrfMATe+dvtYr!PU00ttkxb4*g z(uufU52Ha(K(_%Cwmmpirc(mNNK?fR6AQ=e@jOrmQHiii`k~~s1pmEDV{z*fa0M1$ zL6!1^$NGKatDIP9|9L?f+xe{T^RAlfeXvHSm0ync%jmQ(QSf$-Vb`oJ8JLw;gdlW* zWM&IDMQGLTrf5V^!mF^k%I|9N&>&9puQHA|;rUFGuQxJb-nKB6E_&QBWFp%%z~^Y$ z^}k42y$lJB6@^*x=!yQN5p6Cnk8%wz{HPk+eOjN|cC6Frlt1S7e1n4`gi)g2+L||$ zYTQK6e4PmRW96si=2mv5lG1F_|5O|_q}fz4Fwwy36627(pdZKAj+d73M!h=!Ecb3F z)9N52r7!{vC{YXQzo)=&Ei*C0_Vkj}l(a-VtWeF$gFCaAEzXudYD63<#+c~7 z$jaS&lM=X;;f3K7Y_2O`l36c(-<>f#-G6+Uusx$pW__45)?Qr0+Ga^1jCN&G2NFu; zXhS6ZdsK}1me7u9;Hkxuc6R55PTcYjN0*$TrTUwkU~6+&F(4z zUx^`GM1)?PTSW9aMe-L7{~^Hc1P;QmAw94~dq_f*R59brp+-@JQy|I-m_Si6TJ7g_ z-gD@fMsSrWhPVa6YC}tkl9j=d96X_9j)Xh$lFGQbGr47g(odAlJhud@I8&APg{50J z-mbP8g~kwm-vmJ|HA(i0sJbMC1Ph@R)F!=aJ{mH*Z>hz^o_5dG1O4v#s8yzLp$%N^ zz@JP)IESk)<%XgVFeb&Cs@8a{3idb8L9$OT#dldhi_ZDeqlvL%tcjmi?4~ZWvUV^J0=FE|H>gQ^;-KmF%8k-J$vj}x_jrJh7#Zo62V`F(P}ma9U85$X715WEyc>Xf&UPQ5u>W0JpW<)XRa@s8 zo|56u{6UUQD`aCE9t_PUVSfRSJ>w1V0vErgD}nuD4!W!P{U?I98bVfeN(D++c}kVy zSw~u6w)|?ZIZ8su3sX#6Wp2|qK zz^2P6JJZ4E&x~!e%x6I}7bN|ceQA%YA(sjdZ!Ba)8PU8M#oo(Eygx@#6!bY6jG5@s zuq?sgvutpmnPmqg&2u-)%eM24mpyhHkpnp0x1+}^BvK`CPR@9?c{SGNgd8F-xd`Tg{#BE&O7@}km%WXaHi5cx)( zA>Z5A=w38$_ok{6;NnMO$F=v~rM4xup7r%l*T>V3#z+vPpYgY-Kgo*4;pRYy zbt$Q~&9P-FW0>@hV;$gC1`P7#bttAv${;979udAHxX1&1Dwb(?;{CV#KjPTGH;J(M z_f9>|kwb$e7CF!;EK+Zx2ZAItuthUz;u2i3wO|XRq!5^WZB1yX;m5+KL>{+U_o&s~ zm9FLmBZ6im2W_$GzV@RlNVLYb-l;SNu2_9RL4XwXO{%GWE}u3sj<%m4`=x!{h_&U$ z5#HC#RMtbD*nK>0(Fhlu6=$b+Q_4tFJiM&4py+_9T4qL;nM-X2|ruLn< z9QTRjYg@lcvKnrxilp|*zz4V{eO%RdQA=9#Yn3tuxsFH z(VRZqn%=>xX%*;m(uMOB%Rf26F)?&fZ6Y=rRhis@o@oCQJIEexRaaJ;`Lg6IED5<4 zJZR9Xb+8IWCj3bQnW{C$3-5h-E+)@>)qdoin#X*8ACOcr!_Rf*o0zr*=i&)PDW5`97 zx0o%3^-mI)RHX$iQ6lf`VY2-B;qRk(Ol!^$$G!yUGhrrj?eWx-!~%K9iBWXz8v00g z)&RPs?{Lu(`9$#wj|4`aFAvVEPCiAzzkTl@n_^^IUW>rN+Hr4sR0@{-#~6U}Mn})x zR_7r5xmvoUQV-T7jg+9$h5z%w&TWxs&qa_(j}Uq_(R=@S$eemtvHIpzXV39hf0J6~ z_3q&gWUIKomEMkhPA2ZPd|U=E5{C^FPC zU>`MfuwjXw%buo*S8*BjA`7hw-wsFSU?kpjMO9`XE%?TNd%XX!gS4M0kpW4+CP;5l zW1}xXd`7TQYnlBZ!{=f@)w)#?X;_nzmq&cL#$!KfK{eF?yQ4l~8v9tX4>T3Xg#0EB z5Nu;eI^K6mn=;!Q&!km0)jNw9XoDJZH8FyNKYFuC2sb9Xx#&bLQAYyU?C!QXjDG8K z(}6HT7AgxIPJ4fu{6IZ>QB`A=oNeB0yMW(f(y5g~|A&tgvan#l>xiYrDl^mGX{y@W zUvWDMv$@2;M{iw)+j-_NAgMu3S$vO9YK9-HjK>1g!KW?Clx{$tkjwq*#m;~*(_p$v zf2C$<=UZwmao7E`n$~P|W@wNfh2>Yl0JCg}mzRa5rO*)P8*~5nfL_vleKB6d=v@Jn zm}1|Uu*v0V7}RqZaj8yy_>z%-O7!-0w6wfG&8nEfd*M<#h!}sm`GBv#`VE%NsO=XR zGs)0f0H;N9<#_17O9*eHB*vt4rEc`^ItnS(Xt5oP?k{@o|ISh26bbOQbamygs`A+E z-*~nH=tKzPStWZ7?DWGVgwP)~R+WN{(>vN&ge=bOgF^fe-q8`tRisQ`FiEL~L?#CJ zT`z9S<#t2w+sBQo-3-92+-~Y!#RaZ$UvwOWe)g4tFE>hR@qizQ+ z76T!aqQC#xtrgf_66U^-ipp zjMK{Fuvd(k&raUw5Nl0Lnwj*wjdv9ZoG%Di|J5eI9)a)P#>kf;uc(fpR7@SPHL0?E z?48IMw|mct4L=Asn2^eVq%z0m=-M}iu<``<`!e>cQ$H|Z%xuh7ruT;fX#jj_@(_=_ zPTWSCVc0X72zC9Tb36#MA3X!a_p)11ay`-M4y~)o>1jx&p1mrSW3f4H<75(>GG5;K z{rPKWfKzT{qAUNH`VkDQ_;x1RTDVVV>PT{=e|;OjQO*mu;yXTXHQ%UocyQm!G3Q$w zS#A*ldxeSpIx{q5n^daH$$eClv+hu`2V&+uAuiz?1Q^S>m$?4Fl*W?!0yFg9hKT&g zNcJC&3Z}IBgQTTX_DvRb0qAni&^>)-=5}n?52!m#Ss*z{r zPtI!X?|i{x*bg*g*z*6-rhWCz>*h0`cgjissMXRb&R9 zL6kuyNM7U^?b*Kx$+S%_k<@iO`l}SUbnqeh+f$oQ*C_kj%9#6Zm~imISu??2XQUNx zOv$pdmY}<<9h969xKTGshtLoQ&a6_GKW%rHh5lRBc{{@Tg|8|41K*gt93b!_2Y|bq zLpdsNL)cLEn_4aG1$oi?{ya&|LWK(YN^nB>d%qegVIELr1v;!AMnNMf%x(X*qqyhN z!VE&z4|ba22Hn|gWffFkrJmcn2H|?tTVbbE<@FVq*TS;?q7h-{MpR`Z7ZbN%z0%d{ zCEtNI*cUCq$mDWU{UgC_MqX!`1SI{){7 z)7?EW$I;!*CZ?NVm}$nei6f?)={lIMO?NwLx~Jo4rfoWYxA*7!dpw+f{3BlX{kpE} zc|D{5?W{|m+WVb%4eGpfu9ED zG=OTEHmv^s7i|t;OQY_nfAJeZ{>#g^k`X~&2t(|O3gu!=Xn84yX zw9A7ZO`uLW0w{}km_ntUDJ}m0N!kx&hqVoD7L?V1nm0E*0F8$NICkx)HqZ~`$3e$P z)^5Zm5t)|A)*ytc4o;&irj&yPE$4cFrPnne$g_$u=(k`>8g?7@yiYL&Or#56yh{51`e*~ z+*SZj-9%y2V&_tjIhm=TL#<*sek}JsxR}V_vfjd;uAe0*G|GAQQT0@AA}{@I= zSgD;sl&|#cK1QLe!uCYh2(6Lk`W_L_9rg zU$yZn^M`I>KMpd6(8k20Usf80EajTM@b|QOnRCBgyU<4a3g#h()`K0kIqiL~vi8jp zp3@*%oIBjD0d*3W1IK50Ti4=G*C-Syk@fMYP=Gj|U%UueUTvL1tVKB@)GQf-A8!9v zGM{ejfpQx|;%e7?o+NjFY%*?n3>7tMVbl^|hzO>O#~&no0J(o#Dy^+*j_fZpa;geM zmv{&8^ub9FjBefha7N=cPENaQbu{wgr5H1)=t_?dPXi~&ySGmVR@j)W^m*ruiLfz_wh`{ zS&~b?8iC5BYh*crW6L2Pm-UeGlygmq25+Q0U?w7*wix@?<{qyGL)0oIe$M6vjD&<| zXJ>!Bua5*$r=YjDw@4CoK~b_s=*F0^5aHop=pocdre=@tz2&`2l_;;>(3Jj-ELP)8 z7npNP`|Pv6#S@G5qORO*y1D{co7xZ`H&M{sC!)_vUQHQn!l2l(ZXMrxvA65Rj>1VF zcNev9P9PsFA>+)gw}CzfbK}+DdAdsF(GC5cY@)Ya?Yx$%FN*%nzv~{~KXMff87Muf z9MPhOuRh{OBq`cuk>#-IMKh7BPDxPZ_U{%rx17d^b<1a03Npp5Q7(43wb?%vTHitrz&+n7h+BGz zg>g7+qg{3MI|;vWtq&_-`@$|8_S!?5eYe-uhP6yg$iv4LB2`Q~iI@^%NyMWp`j4NZ z)p%fQrI@2{*ys4sFnl<$625;@U^Pk~jAs&(5WXEerhx8ubF|!kQO<==PDAX4b#yqd z;v$DJt%ahgkbdyn(ZFibceWc=APaN!yU2XDSp))vDB=csN{h3HfSA$GxKj70wzg6R z&(TYdcb!j}fXn36>};R0$4VcN+VO7YxG2+M1Bhzu+&@0H+8#>2-;Ao}ctuCQGxeyDyfJLudyFhaJ*PS!w`F|>?I+e3q9 zN*_W($x!FP6~seSxuzU5p4{4{8V?GX_)s9L3%}-rH%*(a)HIJ-M#G4;f5gS=sbe&j`z-FJy8U2H)9)qUcy=ACmhF?DnAP6hCMYCvjmk2; z5t&H(Zl>7uJs5|ND^;1M0V~$hV46@Xr@6~eTLS~@Dyt=t+nV8!DwaT-p54J&g#})} zwccIxgJq{K>Azdor9^Af6#y!U)G1{*O6?|6n(6O#r->|{{_PKOyk}$* zsGp~^e{^3tbE5SNP*(GW1smdS2HzZHw?%c?jc-uxA;ytO-spPnM?hbv^;bje&UUL^ zvAX-1QXV9t1#}^`b_MsDgo+r9?x|NmqPxoq5Zd#c4h7g~C#oP#efs zz_l&Kg4dwV7cjg0`}c3sGT{7G3xtF}XO`7GtUl*K0ud=bH^U}YK#0%}qvnJt&X(}i z)zz9io#-VLhpQ2BBO3N%Ea?70ubSR1oER^R{%XSg?3Yix6-`-&AG7pk=ALg(vR_)t zwU&VD8N+A)0<8^Gy8i96nCblhd2QVTB!?YNV+CV#4o>54o(WZ9LSW1tsaSp00~hWG z6ZJ9RBu?7owY|2He|~7z`!jn(%qI!VARt`46*?Ev$4AEbj%wk`0d`sKJMcWNfEaY;W$55Hq+paYn`z8a1l#B7)Hx1q;tt0~Uw@RrXzJi;T0CG*F7^O!f| zwuN681xX~i44ecnwa8P%q|jWr#b@-3M<-ALXQE%@mMw?xp9r~?(I$*%k_K5;{e&cC z!zte1Q-(dbXaBDt?gpPfGZDvYXH7x1vjaJnmaz z(+PO0)&l8oJOhspC;mBcet+|yJq;1=53Qv(fncmnz0O;b93U*l_VedaSKKddZWhL& zn^ViUx_n9gQtGND>bJ5Z>k>PjQ_3TlyL*NVr0%dYpqEhiUmMQ_&ij;QdEAZN ziC}*%>}*r@%K)@u`i{r_M7(JdyTW5jMuSyhvi>O z;Cjmkr%j9T`i3|v8UM1Lu9rw*<|?CB0q!M3-{UHgLh5;X0s=y!*kYGlA--Q43QaHG z<|}5{S=~E%^(LYMkCqN3i(hxR%K5+c-HPqs3$kk2NKet_hSEiCXv5$pnH)zDJ<#W+ zb8khgJ4yfOVkW@EbOtWsWc&-VMy_4iALqh0CFju;vs(&wxRFc4B|Ev2Bpa*st<8tM zDEICw9c>N3P7HpZkCvvLC>EL?dwG$8@fZGhcNjkcH{H3cE)IrRSAFfE!O9~ zH~)-a*Y@1U`v+Y1o{y(ckIlo?pjJDwPmXA@0h|f{@}&P-fF!KKWQAL&SQVA{IGmH{ z={Kajjg438>hyGz0jt;?z)dI-Fu=?(bZk!sjICXspPU8D)MR3GHoAC)sxL?HJ5EEm zwy6IA#?iY#5&+sdP;@K+|6DgWHw|OsQ6xO^F8{|}f3*WoG!pGWDNp9yPZKkdwGi5W zp>JYH>Bwz>dFYjGTC^-7@Z|}moc%x$Nu^F{pvj$5T|l<(V$(#un1*PsNQ{y~%|v4; z6wbXRG{vTtf?*8l-Rv81y&FkKW6Hu|%E7|n(7qKeLJCVE#kg`U{Mj@bKl|Zi7g){3 zzInoHdhR+2CEr!3?=gF zzNnoMg4y3@C{XUu5fIuD3xCF$CM-H3+|^~)ptax4R-hm+3K=l)!{kjPWtInF05j%) zGeFd&PI7~{)I+`|v?axzbO(<2)wb)SiNlhGtTdzol4_`;WUfiqWGM@C>of813d73+ zMvAUYDxo9Ienrni7^>gZOeT5cP{)hvudVr=!ieXY9v!e*YYt0IR}$kMf{nxP}?HlftUN|}sXJa;}JDX~PIJkjhe6gzlBu;=x0#ro?CE?Nt_B}c^~ zcES&U`=dSJ_X`vXtHq`9Orf~vcN}nzi~~e$Tqo29D0A`#wyQoHQ1u@{cp_dDrhG=5Ev?TI*ko!?EFf zPtp9j2T7WJxm}f5a-mhme(ko!^Zt=B$y}}5`7g_a@RzhBw?uYc7RNAe4rBm|jeUpP|2Vz9~>u44Mp3#kLQVd}Xm zA@#fuYM5ROYbUZB8;%}r%W)Kq#%l^m%{W9O`+Qgm1uFmdrk=k8X(SS1WF_Kbv_BeI zZmRHqTjG%ZmTx-g4>GQvXNfEpt{d8^NjEwuvUKhq`XcDKfYX%{YsxMsOs+RlZuyG5 z*sf%r-He%X#alqWVVBes_^vrTQ>HB&A!Zr+kWRpq>c1h?=exK;|Gk0$mM=2*x=5om z58uFrfA$Z}Krkgsm*=$LbP33&Yb9GcT^CMa{#5k{Qu ztNVMi_7vvkzEGWx#LNNu!-*-E^&Vb^T5NxWPg8VfLG2r`AbrbEB`&okB3-Z|gs6=2 zG;KCkx>oAN0E~z7(y)B>K;>0Dlg9{8qifER3X&5hB4)eng2x@lT%XCS&kTGESK{M(CY?!&3LCMnVYEP;ykk@k7TZk^AUEV;+R> z6A5*dF#@##nN-7Fnj4-RIku|Fi)`4MVG(TAf?EZ?HL_KdL?sH=LcK#BA_H)=|L#^9 zPv4ElvIGb$;=0lr|JdNUf+4&1$ixC`ULszW7>W&9yuOy1y zY$$hWSKfW`M+!;CI;g$yCJE`Q$E8XMaM<3t47J5%+SX(Owi`@tHkGSC=9x+IeY$8ki@N%9)lhNXPq;N>IFA_Y^O<-FpDEg~AvzY-YQ2qbL zsF^%oMwq>SMrK<=L5ZBvqSGo$;5zoG=k(du8ArZI()3X zKVv&h2=Qzxm0c}(<28?JP8i5|^7jG(b@B<*^GXSh0{kze2+=&xss zQ*jS8-*j)0EgRyKn9*xBf}*BcSfqv?lB*v%S(2un-Bb{th-D$D zit@T2O|7W=ey5xpAc*2sxK|h5x;^_2t)tgHPf>I``vQTr+Sj)|f4O9%1pQJ_l9u0gk*5oey8! zjIDt*LUADf=s5x%7{HU=NJ6FQ0Pp$~-*zBi7%*p6H#VjQ9B7f&Lti?qXV_K1fgtBy zpn=+YZkd3jkA+VIgo*?>qaVQ`G3|{+?|0N3w*S2TZd=I6FYBULti1=iH25+|Nl-jc<_huxRK43j}Tzwj=P!!|Ku6eu(t41@nla z{$LxUt!uMw8E-|Rle$0pbY0roUgi8*S<)$?S}o)I-q+my3gGT#RR7Bp>o!E{hwiD^ zUW*~*x!wUXTKVu4s*wMNTZQD2b1YoDP2dG9A1KR_Gv zaE7-{f#1p(tb~LCiXgt~Y{*QqtfOw}8avv86{K**OExcTeW;)m1IvpbP}00Fhj4*Zo4z8M02Iu7dDnHobEoc4tDL zv3L;xPe(*{p^3h}zyyZiG%7IXG|bV6aJqWck=oHA>CkrewxzXIWGBts6ELWcicD7k z8xW&V8s`)VplC?QJTr~>%S5V!45Hc~-8HJMbI;mF|a z;DGY+50Oiy7v%$jM3D?PSB55A2eqj+2|2fD9wO$V@2;@QM1!i&+KMfjV|zg7<6u(2&vc-#ROS1-~R9zK(h4Qf2Y_ z+h}1n@AFe3<=3To|49CZyQL@`B0Jr?IYll&O!*mRsNrfY2V2=kFG`z6ss^W&5n(ee zGqCGgIE>euY`W zuL~iMi6&4r&s6G9eA|F;2A3FfBaA4HB!ve}f@zB1x%toY5%zJ*#a`|cV=yH1u*5rw z8*b@!oRf5%Cs*n$cMy{qIlUP?0&aZ!1>fhe>&RI5hktXfz(?IvC83c!16B1W3`P^z z{hEuProfB6-OqF2Zar`qzK;fni}l@rH%>=nHh4S;pspBDKT|)FYQ;P|Ohp+ZCyC7h zG(`%lOKB2yfVlrN9g*6vG3e{Sn>meOCvn{=Q&QFNhZS57i1r6Hunuv^Rdz!fWlrK# z`w75A92{*>vV6t)X|2Uh;^?0w_Vh-==C14Sn%{?ad~k4`Uaz1^d4Xr5wO+B?ylqj; zL!QxCy!Ct1Qp6Y4I>Y-ZCH<}#RxoME+dNA->A@|E81NoN#Kw&7ju0yDUng7<0F^<` zR}7VA#md;)W&3X%S*N{(pj)pUdpRVGVXhcN*_nu!)t+RSJ)hy*x%kJzCR#&RS99u0 zjRnXD&^3?&6nzE>~u0?2>0-yO(<_7Rai%b^Up!AV>9w8q8-K(owDl(!a(TNjv>;V54>hC62FG*I0nB#MYN!21~CHG!V^EHtq zJId>6O={qM-bJtQUvlA;Fr{<%gGUdKoW#_D69;K?(#fp-SV`UDWEa}@who;+ad*nd zJKNgh<~*YAw8AVWzOiPhw@w&oL_WkP6go*-yf=H}{$1tZ)!?JGmYQ_L7Dv<)8>5Wj*dLY7;>D(uejXQF&7&tn9bm2a#0Jgv0J02&tX z3_2aK?o!Y81EiM;F+xl20q*L8?xS%BCMsnuGN%i?Vh)3LpK!0S({Y>__%N&5HvB zo6iF#$0bn7uobm;nUZe2--6c)mRLJ$b6lI?&1(;;% zzBka<8^>YAXQ=1 zD*q})b>ksrM$Deu1EqEN!?}UJW}>=9cjasu3fm%|*(?DzmmPDB37gW%5uR!bV#EJR zhYXgSX33=}(65Ht!d-0mk(U>|O7oa?ys{SVeMe_$NCovKwKhlj5o9@%wL}g{GWR$3 zIeZQHx;`@Z?`S5DQqyDGT_S===_irn+J1MhAHTOu$e;-~F&E?nHW^2ar>3z=I>N!_{73ol=r(Bdbvdy0i431WgqHR**ym!LKa@2g-=B~PV$PB$UL3(0*^=< z-ulQx_NmQU>>gfxx^UGq-tpq)2UJ&kp03>OWuHw9_YmD^B1QUfxishK=FcG0xA#0M zd~CelSvP91kux?rSGsqzK1_M0?|-D}4LD=HJZ+^ma=#^vHJ<;r!bPlnzIgnm!)-~q zt=gk1&)kyvUjLA5I7W<%$uvT;_e(kRoDrk7xhXA$R_LIU@Yw33#)l;>^Otz73S#)t zlRWtS>`qE4e;h$1QN#6h>~y$Cdl;l4x^W9@FWUUZ`!NOX<`@54 zWk)M(#|Vja>L8?kfM(0GF`(a;ztPq7>0>F?k;`4&1tc3J2(4(a;5N3eMRW(a5!|`|GourqU^v%Zutp0d=I}sAF zCVv~0SP&skzts4%z=e_`zXMz%MS;@{Gy?zdqh2$kPvV0ULDuloS4|S@m57y&?^#h& zE!-65$Bi5`+dWH6#Kk+>+Q<{7=j7C2iDLoQw1L*Np^px9UpwxTJOO~3Ww*zcOxo2> zlPsewkJT(ZWY+Zvto8QYt20q|po6b2h$k6@HY}Uss>;1VXB~9QiTA8%im2G;TrA5& zHyx_`9%%Ar$FnT&dojvbbk%oe%$z10ii|RRMRKUOEEce(Tj?QKJ*0%k&xwW?s{n^Y zSs`N0l1e4j`4>6ws}ECd@~(He&N4f-1C$nq^Ko7_K}$drfcDc+J#@j@L!|OArkXMO zWc$=a!c4ZMvjldMJ^9;jg|ko$l;RaAdI>{+bcti22398o*8TTB5Do@(9Zl%ZLy2x# z)5noER&9?#1h-2>BM@b65|y4j$#pYBpMcEngO?8$u#JCw zs4-_1+YKUyta=RB6wNYlEC2f)`Mf6QrDl%A-x8OEmqi832|KW7KuR!mW-;G5iz+g> za>5q9oxEiKZOc-H-hQv150zv&poAw}MIka8nvey;7p zfC$~fNR^=^ZkeExOoovtIPe`%56i(u0QdC3;xtB1V}=yo2nHMg{l*#iCw$Sl$V+C? zIxk46Iw3`AuJ*IB#P@BLwl2eXL>h#ae@U*Ou)ilFOyEV8nFGgK=hgF(OFi7ZrO|Ip z_4iwHdB^$dsr_Y<9Dh7)cS@7EP}ohFYq?2(u=a}iedTyM%?2?%HVJ~1Z8Fvul**PMc#0M*|56?xA# zd28PH|2sK1t!i%la4k_9Mn0yjHS+a7p*wDk>){Kr$VGuU&*sy#x69S9;*Mgs+O4Ub z*?-eZCfTH=tbT^pobqR{@g(_n3K~lou#ors9wT^s9HQ9)%kN?JU!ORr{; z+XeqLrU$6s}S+%j_hpBjl3e52;LS>B5phs4^^^hQG?vW;} zM2r$g8X*(8zG&>o6Us(;9TU_NM?w+g9=g+zkSOOC4P9ZgLQkWA<5fE>bmR_e;MKKS zV{rh}nc{v@2z138FsdV`qy((~{tVwyZP*TED$gnHP>m3C&?N`Q8ha*U@|fh?Yz~re zoHa+v!3yF}eqjx0kr&Acr?8S@Ods*fv51INVG<%(wNPXB8O&&nlC8N@b^7`6Wd#v@ z`G<0|{FBS+^8B6Tzu5d!VR!anGY=E|o;sU~OO0*!od|b;MyvZY8olI4&G&PtS74Zx zlg5msiIhp1IR#9pKB>Q*?F7faGrjs7W>aydvF)rT>bR~yz(v-uEJ55|_uT_<9S50) zYOsqi#8;j2k>fJ$;T`7~6$87yuhviK&H&m5=yNX05>r7`%2rGkmzWC_Z}Tw-6%#|t zf@(plHsAnTgPCiiO`&Jy7$D8j_|A&?A`Yo9*IXYp6r-gmK_R_vE(*e@t-1elOm3x# zno>ZLX|DaVNdHHvU7Fec&DKV%HAnO&5eHkWAx)+(W%c!?>HxgM`7Y47bjLwU8oc5G%j?LSS#L?zAWetG2pLu^c@x3r6Z5LV+znNH^)9 zTLZ>PY7Jv1nr;H}LlT%VFNW$*Z9Eh;##d>aue36UW5R}U0w~F;NeA4<&EIyEiAxH* z)_b}g2kg=YWEskaY{xH0k_P#DqTQW#oa;P3W^h8oqEfpPEWx`}xMf(yGNYJimp`sM*CGq&q)&ku&T1NK>9szr)EZL@Hy}a-m%9xp@wN6{=z6tsH z4UEiV8pf<$3Hug74k-!QGS!T;s=72i=d7$le z5=PpZJ$fIOo=y=h_^QVoZ?%6$-KBE({E``&z(gqhcrQq(zHy}?>-(y4Z_!ctWt~(w zW3>I5;$VQxBPuxILGJr|;-|eogmf*&kYJ6yjKDifVPm`zP2AlsK^xE|kXW>>jGg{(9b((CShs#z(WNs!Czp$&dM~ z_S|*YbgI*_=4YQj6?&fLCmMn*)c~25Jd#M6(#U!0xff4ML6@S500S>0V6ZziQ2)<_ zJi#bdE4BybSEN+dYb{Kg9yx5eIx1Q2MmNd*vkeLgN%~W41m7;@Egg5mEC-tGPqnrN zULiJ&Jx;>@C|^X`DAe^#D9&0vz)E&2{4G@jxl_>6|E<7qK?jtYZfMtt!kdh}uO_*} zUJo;OI~}iKt z$wHfVVRxa^(}q6BZ=6N$;`e}%n^n%qN!NUAx6@z=!QdmHeI*=x>^%xd`LcIJykR^p zN#JdicRSkz-hd^egUC$j%2?0QFWoTZbFH6N4iPt?xQ3iGWF1j9{Ay1rLe62Rw!afV z$dK=3U+e={mbw&j`}1d{E}4F$(uUS@6}`J6(8~fJWN4D^yzA%H&8KdOG%L`6<&!^B z0(CE|bAHmQQUhCBXHc9=cYh{35&!j^Y?BLWdDns7-hFKpiRvtkr2kE|&;71CK}503 z?C_NR5%~^;Ry4$7Xlu!%ve=)$f9U4l^JFxs6B3{iUBs$>+|g4nW+N8jgIA0S&;UatcgB~DS?}WU8v&GDu6SNW zik;||)zBX2+bI!orc5&K3~q8Bvy$oLPo|+17+vW{}sCg8D!nz41rH> zQ2`&RhPQJq--y|-K|gl0$_p4k8^~e7R46)g zd@pmIbRm0uE%iX_w@wrze<62bV1GgoZ+%IjKk!|jxPE-|Y7TZGn?5GD?Q+LzgGA% z>+8e?E7+>NRcrcM6ymD9Q~fL4jG{69VW+c-IeZ14;!0TdT8)7i-oHIwUo{R4FHg>W z)%SY=IL#9Z#c%md0TpobG1>T5L~`XZ&H8ksc-UMaQA9C-a_qOjZ9oTN&LhYQRj%(e&kpDRKu)-`|h}Jm4q71mrD7L z*#FSP@>X`Mdn^hpHs)kmnez}TZiR%Y2>IrNk~$#Fumr`5Q_AVRBxOp*mNFe^qt(X* z5$eQHMgDlu`j=?0A4#TGPn=trP5(PYt&<`nf>>aIeOC>3LEb<@HIMOulZ`3AWL1 zG(;crBXOeC?im(GkZO?ra?*+0v(H%nnW$p5UHC2Pskk#<+qp`^;F)h_b)FYSru_6q z-q8m$$T)gL(kO~%;MqAEZWxiLk?67Wxw3k)Y}xz_UAjmC(4)>SUpvT+0d2XPn zZt&1?v%Xecng49}l8!RKf=YlgXlG|%I0z%Z*w_!?I{60CiAVTRMIr|IPKf5r#}7 zGK!U77xvTYl1&%hl=Q2RRa%^#;Z=53LByKhxsjkdcDzzLFg-3mOmRM5o*ySTy!}OD z1}y3~=HeXTzQr8X)SYg>DiDJ=Svvd%MBjoy?5&G`WapCXYgWh_E`iC}YXP_wi{b)4 zM#~Nr4E^~P+kPmiExOu!$*I?5kw+e?IP%WYv#c4752jT=7a}9&blkGgY-N-Uh}?HI zkaLtP^$W5N5&u@KexL9BbO;08nTHtuU!jWgrGfOh-HMfD2 z`ksf_t9XQI;*(~HG;0%;wGG|6hhszLl#pRYe5>mL`0fq#Ldwh5>^3``Q^nFSpNcNG>5VRrn()gp6Pz>hQm63Mf0Y|64cP}V zT)DyW!!$}U@^E0ckbl(`KM={iyQzj0#%Gu-%tQz%An588(3Le6D0GG0>DaAgVm`NM zvBXy;z20&`_xPBb$PH*~_sSxVbpVXi$Zf>v-N?_)(wMBax}SK(73c7a*mbfvdr-zN zlZDockgNjiC@j6mnN}d zo4NO>jtrJ+&BWUig}S`cm52}Tk5H>Lm#ld-wTk*y1Dq;MGOzX1>?#Rw-DC3Pq{Z2M>DwkI@kfTn+%CY+R5QIXEFT z`?ck6Z9DbEGOlf6(0)0{XP)P}u_Ixr(iu16cZjfwZ!G<$vFE6;1ij{`=zE(Cn3q+q zv*`~nFL99uEKQeF5YX^#y(TI_4U-9`w#LlxY|F!snC6gueW#?wj#ypNWTO+e2^v|) zdtHZ#8usz~OQVXjzP{3g@SfK>p4lp|Ww0XiCJr_CB9ImJxqmd3)^Bk0D!`V|(Um14 z5gqVaj$i+D1nWQ~1K6a38@wc~s= zX1OGbqpMP#2@y*Mg#lz;8((A-JY%2ZWgB00os`vrk0Do$U(PJf;|ibEVfaI3I8U{H zNHTC?HHWU))0>7ArDedd*nlnb(kZylFzj^%bsT z3tLeYxc>=+BhwuDd;cGcdB$jg;=o{3!e#Hf&BIgvi*)6<3g)o)+!^cgc#ce_`uF4Y z#Wi*Hj3*BfO?N+d%Twr45VI{w95Y$ds}{~mdLd~{l5g3_-l`6p@h6ZA^&$b(+^P*f zCYd4bZ7|p9*y@nfA3$ltOpJz-cSfu#z%YdLA3F;aPpt8&->8fmx|F+JKPni}*Zc6j zESCjS_X8R|lhM~!bF}xYu`b7jRR`Ewjz0!&4qQvRru5Y(0~^c?4)-u>s#vUPygr%x zf%UGOP*}XdiHw$M%u0{GyY{c2n-cr3qfss!Ld-}s^m-Va5SBg7f$4{U{ONVh>p%dy z{c+4`s(JEApB^}GvC$r4S!bMR{n+F_IU(HF3sX06 z4Hhtyn#Oa2Tn=tv7z;}*=iIkQF!$!a%=#^eg6)`rDrV%%frE zB`}Pnx`<#UklIi~q*am|;k(N|zza6IASW{=pP1f08IaZEixB^x|`h>k)Kx6yM zcP1>0_uG1>zl`yzx^FHjs}(%(jjOQ7glTInSt`b|Kuo5p^rbf!-Sp68igu0q(qIav z+yqIL=D+5+#-lq7{3=+gB+{w#>VxGO(0!@tIsHXl^W-qBcMYyjEfNc#rMy2PJx{Ow zzA_6BIjb``fO)|4+{U&$_e`e(gyWV2v`&9y^F%yWhnMd3d5beX@(*GtQ)&{U^wiA@ z;co=!T~3J+a%6A=OSpuq#hM@D&NYOr>9om&G)$~-Np~VM)mU6>(s6q>gU!{g>uvCJ z-kc~TvBn{-f!T=l?l!q?oQ59ZS2t{*lwWKc z1u0dq+P(cdgihHoL+|yBFMs(LTjRXOjR2q=eor%*3{sEdCVtrEq(UU0=LC|8$Yw8k zUp8w5253MtX?&2r8s;p%kN;lnyoN93?@4dXWhbXg!ft1Eo{+?fTAdF`LQ%bADc%v* z(bvEF$_K~}BN;II5=bg_`Uy*G`h-z~oEH!N*{gNUvt#QWQ%9(Y==l22fq*ry({Mz^ zf5#NYFPR#26XOTaYLL1;2kYJzDu^t9<(Pso<5O~AX~_BASrA(C%l@b)c{-+xjrOSZ zI(emxV63e`(Y!20N@^$;W>EVx?nq+$qH~-PHE^TD37jB|E(criN1}TML>dwa+S{hL zW&mnog`zM<7;5m>-G3Np<~Fih*Tc)yjiWi9$1n>-*gk%LG{XKrj5#xtRN)W09Fch% zo36n6fxxAnXr&gbF@=zmaMcK!FZw~-K#~JD)mnMSP2xt&Y@q! zTW8f)rfzn23922{_uv zFJml2Y8Axg#BD85-pCD-(b1m~lWrvB+I>>P(&G)3GY|!{GgwR&FjJl6R7^q@^!B8VwOz)A~>lU8HN?>kT3B4Jd8{X zASylsFZ6Ka)SNmeE!+|%qYqXX$^C&feGEm3L2?K`f@hFrJb)ig9Rxpn87m{q89Z3@ zsS&@^`W3jHYC3D!g_T*!(g7B)5E#npG1w;ie?vMoFggBIm($Y~uQ*RoS*4Qp_!{Kq zMzJBusoJ&q%1ZxwdB2071OaF=hFx?M`)8&ruH98M}LQGfD9nj<}Wk2c|~{Afw}U(3LxJQ_Sbti<3-{2)?Wq5Y^`W0NGgKozTAg<(IMxuhIM z&FoUU9r?yWatPR?w`!G?DXo^7GR9Y}ZQ>)Qjix0ptkv}P#78@yX}StK`{DQ>XrycZ z+1}XPalTQ+T3F>9I=@`}pWZgChdm=gJS>@;QZ<@FQj}<3D)GU2aZ!pnnO3ibPhEW-jAS$i9KiE%H`hhUq4m)P**8`i3L_2>Vr=qM z=N#k1a#rN@we!x;^+U9lX$TOhqr!Iwk@q7R@)lj0nct1^!uvCh{;&~wXL*hDhkV~V zw?I+MF|?lNrqFjHW^qJ%50=%nqc`>EiwGG`yXejNxJx>t@GT>i2DaRWDtRVHS0D;C z6UViVKTUC_vg$>s|NaK>CesjBMdpdfy2M1j_hQfHMqvvuv&% zf%}eg*_{z^CmT$Pm@8BlrUfwT5xmBRkZ+kOx|*7CzS_`W)fpeb*-cesf+?*<3Q>v# zOK5=%To^~U*8H4YnQ|d=j>tHlCMNQv(})f?R11=0O$ z_6b%iCZ~PlJYsL;5Ri!P0)p>-`xT`>$IMysb?xX`jFH0cRFD9mn+JGbr{t%$`>to^ zFgmWW$H-{=S|8kJ0Nmx$+I1kU6M2zzikn}yz?}fzqZgl6`-%gBV&UL@h?m?apE3K8 z6$SClA44LTIEL=#mCDx4Nu8balrUcGAM5DzveG?uHMt8w7b^38Sa+G^Bzg7N#iniZ z`bKU^LM_X`#fbu;=qKV|9xTcU0fCqO-b>R6ons0F5v6hu%rhPmU%lB!3msTEgs;V zh;eb$zv=`QRrP*C0KJ&iJ5B>%x`*7IgD)Y@h*#`et$M`P&*&lyx>4F+7{TAZkQ7D9 z1P_6$s40ddgnk;Ck}z|IYGE9^K|l5{q_OC1I0f)g`=t`#@{RQ!dRYL9kTeT^jzzFb zg?OCyxGswtgC+^e$~J5!#IP8`ExQ(9JmEY_M&Y*`R*mU6=HVE$OVi)FZ%Ik>Ky=}M zep0u3%K?R~Ay1EB@_vce&?R~SMuQ+NR;02lIek-=1G$c*2T{QY+q0F;% z#WM5asPbaW2GeUC@avQGgU5R>G^8tK%l%?Q`$%@pII z%AEH!Pega8LGR~Li2{mCMJ{^loix!gy$plt>O+2Y-e<)gc8F2WvVZv~U;5nyw`vPh zT8=g7Y4%WtYd7tGqRn&C9?A66ByPuf&1p61P6ae#+YGd>4OyuS%c_$1#iA4-zUhWP z!6*6xIsPp+UV^+Vw)!_%T@NexQSp?AnddB{f(0TxVQv5kAYtmd{8bu@h1pgZ=0KkG zi1ycpkdNv3aafvT@Lw2yMBVU_ZJKC#*BkUK!FcYV6*=@9cz=pzc`%e~>>5g$4;PNK z%@cDY4kKcqUZjeX%^i|4Y^Ob8(1T45blsBmHRD`%|DK~G?N*^E;UOs7U%JKY_*tp3 zSq^JElyMqcx(JbyFcBUx3O-VUsoXoh1=pQaB19HhUe&80^%24XObwV*BJzmQ1I$G* zelE+cvQDef9(GSrlk1vewGXh{tVf-Q##XLX<9-#o#ID&H{qPoqA?Y>-HxE@()9t0e z?i+A)`?_D(jP>(x9ld25iEdStiWV|8dVdW}OC*=eHf^D^zLcAo^N~8bKF^2yeykt- z;H<`o>{sJ|3_*kKe|wdlukk!F0bJ|Lw!O`tlC1k)A8!3hJ6lK)m{742k@%m^{I4x+ zi0d!0Zw(ZlY&b{N>h%t%-!XygREEs}(JTGpr}H?|3^bYe4<8RUa%>G8oAX?rZy<1? zb92ags){riY8-ar)qUUJVPmtgZ8Wy+#z|w_ZsRm+Y};&% z#*J;;w*M!6zQ4Qoy*M4Y=txHo_Fj9fIX{y$Xue>3R24hT4||3~#4p<2=gs6>3Vej9 z)ZOjI`VGqu^(^5Uy7~*MrE@>%44w(b7bwv~X8S}Kp|PKV4WplhCNrlMl?26hQ0nR6 z$P)bnND7hzb-nxK;b_D_ zw%Ot0<-UF>Q(N_+l0SlT?B1flYoURyX$zV=rjuzRyhjMUt-> zl+S;#PpVd!gDp%&29U&q;dfk>aR;uFRtJPVp6U2!p;Sk{ngaLplhL z5>1#5qPR3;ZQI0=P+YiM3A#CfwKc6eK8{HRlFSg!ZG;m}4lNgj7l%{g8FWJDkc343 zQiWN024gJ(w;?>&!k^snt)LVaoAaITFj|JJQEg(OqLSur!-k_xKphhgB$cg^b*`ufu{bAnL*yzb%>QPN=7^V?%8~d4 zSjFo&tJZcY>FJj4PWEg8?g&W=i&Q?mulxHaTQ5&s1sn9)1&SlGh+Y-my5_X8u8RmP z)|ASjj-ZIZpg(+?N#Do=4F`EI*|#a`70&vf9}t~8I+Y{h?dcPZIMcE3@hE#lX>Al- zq3bTM1>&J0z<6(GRLb=`GqLo&W9+=QapB?N@eT~6K6YhgoFwfXq8XdD%5J7y7OF#! zmlg8suvW3lIFqhb4;!t9?xlJd7*S0z@`^K51Y-C-Z-@8sv-2+)75PFq@cdO@!Af^2 zQ!irArCaQV=1{u8G^V&#B*;z`QfzhyFTTUd^n8#cX@|{7RADXg0F`a#k;G>E-erS6 znukXYg8|cc(TKfPzhI+cEIS+abXWe;#4H{%#uZLh?+Oi0RSd=Hq)m-(q7+HvzV>*7 zA0}H4M;;^-8gLFjbq}y5!jrYn^Ngtc4Wcp#?Er7#e!2PXB|?VBvftGP?)PMCV@GEG znISTS`#y`-ZZLd3D>NGKn%;BHei*)mz%!u?7Mg1 zB^anJ|2%3y{ZBI-@eK*1c6ut)%5b{XF!2#hO6&0Y51=o@ z>NPsH=)<{hVLTTrx3mwPcDX5!-Qs&V8m0D9zD+f;x>OMq@XD8>UoRm3usQJj_BY>t zfwNxolI{wsws(w~k50itv>}usR!^`0b?$v(Y}>SRkbBPpMhx9(OcyicWJ0d|X}x`4 zt>YntedTHA>z7m9TqmZHkKS9BKz$73%Ud7s59EzozJQm<=2P!OB&&s0w+&$#N*ZU(`575q<5f)xYG&j$%<_u4Luu{f zcoq_aA8<@{b~Fi26rQgemoUWiK6pvXVk!wxMWvx&^uN$BSgTGgQ&U0yn0K8;oCF3No-p78xajc==iEcY${syh&4xc7MGI{uL@ zL+a?_IP(tLZ@jyB%IxAb(+LJx$iNPQMH&ME2wTgbWCdZw<1m5V!ET)8#!QdY@uL*ZcWO|TgQcjxyg9@Lry!{Nm7yVqXf`{NbF4b64Bp!T{vq3@ z%}*UKyNq?VtK45Jbq2+-bVcp#Y;$r73xn15eg4_8mu(s+r>E;WgBkxll9Ak&D~uwgvd@RP zUK)6HQ(kG8Pfz0lm79$Uwi%B4br_#*>0%`N`;Vc1t3%tQ_>)g(gik~b!*$hbPcopE z(y%q|XJQ#*A!dv@=frX5qc($4AX2K%hs~$lmC_ItQl|&x^^o+oSjdk_9XJcgLi_(5 zjy!bEAxA47O#VC0IYW6B)U;wQfj>Tx-URvk3uA?pD!~ong$BER;)mNfg+EGmq5%{3-3=sA zq1)4Sz4X8MC+5X)0Qm0=}p-umlgg_wfCA$3I%=vDz9l3&H^9e+t?ZMdZ&m36>PN+wS#I+dQDC}~ffhAn8NnB3yNK3M5VtIM98a9+x|61KTYN^`ri zt*zvKxK)QWuOmZCABGOsftEJcTy>S>W;};$ddL=KCT=w6=AK6Y!!gLn0R9%hvET~}+R`Zp+aQY9z zu4$#E>3(mIyzej9Ab{m|tG>OeDq5qQ zGv1GzNiHrX28C`|7&u^3#Eb~)G%ZG$E80+)YBxpH%!I5 z5{WmK4o^=`76!bD*l{MB$(YtFv+4~Xn->?*S~RJrX7*I6NpVZQ@ZY;~B)^c*)>ZuK z@<(UyiZQVCLHs1Kz4!|BtL+t6o*zR^Nn3|*L7C8pbl;ypeLB2NjH-s|Q-=TKG%)l{ zz;x3iazORnG@xrp`Q-#@IRheLX9q@Umk8wYA2wfcK~TngCYhZm?LWl^ z{CJy8L9rZSo;_NMgPBcK75DQK2sW5lSxIkfZ*EQ*P2&Ks_PV#_zpIHhflKthip$z5 z&Woa1-pA^=fCLu);FDv(Xumg>A!B4js-~vK>PS!s2!miFBO{?!<;r#{!e3ew@FMVL z{2Cc_vC&hP0u6{c=^8lLP0}I%q>ezd$9)zI!j7$zU#IN z#TH7*h|vF{)i5mLfp_byO}dV@-Ok^Y>zz^5GwAQ;mxzox;gdDhCcdlr{`Xyv&^~U8 z^y7sgGY)%>=`R#ia@p_Z2XU4I?_DS@Evw6tBhEAj>p`K%$$L!>+Gi&K))kl50j8CO zUnB;gfUTV|>v6;k_$vI~@BQ{k2_OIdAcz%p+;H#;k{M$oVq)bG_fSp34CwF8gkoa- z`rTL<+FMc2d1lH{na!0j$$D;i* zo(A4TXP=717OG7)n@$Jlc&|^X1_O)5U3Y4VfBMHt|G?;(YJcQ{w3+h+p{IYM7IWx= zEW#oXBRrF+7;SvT;)Y46_fiy+3J;tn{Y3k-yJ*dUO`AtTqqRNSh3EMiqDO2L6q8*& zhYD6^U$qVp{>CMJ*`z5ZJ9aepKJ}rzC=e*8<`yT*G&~oCR#W<5`j%3yTu)V=Jo|Zh zZ~!6#d5smxKieC-r#>Ifu`fc`R;^G6*w9XElQyi%QeW_RnAALw*kQ?AUh#vqOTZO_ zp~mIqc`ro)Td_VMB_9qN-lbWpEIo=?Q>6LHvOVxivJm&tjg||W%lzHe3bz+rbVi*5 zE9aHh8F<{xtoy}O#Xo-*90K0Q!wD;m*b_>o2~as zHTYD3MDo<>BebNSXSrA9s}p(@Apn?o`kr(IKZdg=?(46^8Cs*qolV4EqMkOY`n) z{R}U*tTZ2nMC{LPzF|{EztcBn4TR54_}}M;OMR)&r6^NceLR!sOOz3oA>j8?v>6>i#3BLm#@*t+F7~?49n?gC4C>-NEcR^ zM}B$~1a0ADg){0n&;c{O2Dj*|n?eQP!{=_CnOCvfpJuM3%AiXjtzu;I<2ch0FZK|x@!IyKIU^;K-%U{~s53`tn73rv&^%*%~wpHU$9b&pZS zZ(#geH55B$f1?Th*xBj<>1+MVV@Unm*)qi24VuQYY^HA#zRH07!HP(LM8VR2cEOwo zGbBl#E_VG$Zt39kCY^Q3VAao4mqU+bZ(Ar}in`Yfjn%l*s=j>Wjn~-}Gc2#ew0r5j zpQ-*h365J^MYue{C4S5nau6s;!jpnl~1{*57~)bxDTD$~$I}9L1 zQa|gNnNix<*euQie*kzU*<(&})InciCC4qoh>Ueb+#c16y;daF?vz94-S9$8WEi3Q zHJwYezrpxsqQaw+`ikm!Ncyjzy`QWu7~dgj6~@YNDk#w08`{M;B=mD%6RVWoAF?c( zpUvvuHoZ8ZS|&t#I2$@a1zvGF2F^xKlhl}}tG1O&U-j@(zm{JXF7NaLb4BN1**rkR z5p&Hn+V<(aX)CXk%00c0K`}{$R%(lMxuT217RqOle zd3Ul1{s)0$FdheO4iW&Q|A8+OZv(4&6jE+!UD2DISew9C2c@&QivYZw&X&wW7pv z;6j;_?xCfD&thcWSA&$zXD!F>n?YFa51S!U6mdGh&w)LCvxXpX~ zK{3Vv_~C=q2b{)BPB=eE68|43!T5?f(illZ<}1-|=JY@`WEj>|)xv6MM2R!b9`Ib) z+SMyN(8QobS5iyrG^zw?|0=D}fTTA4&>;19+OAznG%Ic1y^_ziS$`v;Aj<(stO&H$svm%Zn6UW2Z{ z=a(!m{rxx&f#Gcn^-W#IvmFiv>s>%^DTIWWn@r(JA#?-pIJ~X>a0^_IBDj>lrD>zYB_x zj>hyqvIVy1@2a!)Vr`JqIA797@lGCH<9Tb@are9GV=GF+AGq&)_u+qu7)%bNF(RhWQ5!80P6SN)W<#kV0Xfnx;w|6L~8)&qP#-xy7PSUr8 z-@Fp(lQ+q%)#(B=u#N+15ZiyE!6z`0?sK=eIMjPLK;O6$Yx99Zr~^u1;55wG@jybE z;XL&NxB8#R1PY!U3p2l%Yg1wxs4c2>Y+KlsX6!Y~!y|%=scAGQK(+O5M)7+*_|6aT z>AAH;vTKK$wtRoax$PuOo(Z@Kw%dp35`$fWKRL&{;0jmyoo7YN-lx zsKrY;$rfrANrhw^-K;51VdQm}x)Y;N&>`h{+Je3h{7`V2>@?v}5+R;sVLay~+iJ?_ z!w_esGl!k{mrSvqp?B`)5IDba(v#+HzYJ}b2()mWyXbJ=B631G;w1m-^!v{tj3%UH zZw{*@$phxd`Z=U>+lkA4c2!=+9gTMTdEu8di12~aoy(M7mQ2sxb5WziEp>Bxh*mjg zw+V|9098ZZWx)$0vp=-lLpQ~*bC8`L^~Hjbo8D2$dO|PCaK)khT28fC2+(yt@Jg&> zoB_Zm`xAI{nb+e2Sy^v4MREML|4v`%LFsH99CaRbO`oS-GR}R@y#QDpS90OtKJf6F zX>r$`r{z|F%OgqxII@OM(2PhGQ{~#wkTc}k+A$b6MZ_^7`J`7(iGt}u(h~@=BGl@) zm(-~D(yf9HWWjvLkpe5ZamDY!6*NMR;tb9ie*@WVmMGDB#@3>vkLP=KyS#YVh%L)* zggX!ZZ4SQL>{KJ`y{y;Qc&b)D%rAcsx_up}iiJGkq=A4r_lv^X|ME zlZ?s9#tfsFx9RM->ur|I4#b44~$YU0jo<36Utq}S>lv&FU^`*AM0z*sNe;RK54iQXhK7-^%{D+cRg((5=_As|-K^{jjT;T{h^jjKouhGUJp@q(^GG0VkxED(>`|bne zgWIMSu?{pDH6__N(H=ayq!0jN6mv=vvAF;B(xhcPLVdbAy8icAcqtW-Dk3%j9b=~# zkcF1WR%@C@uj7RWh^XB%C9xH70EmCeI(D+NQo>l*mUnw}$6VKEiLn$_zu4FC^s)aW z&0HH)ur#o@z8p=uUt$!oNo0Er{fFh zJh@cD*8_<-epe9@@YF&K4F1Ez!<>SG4=x#5tY*Y8I0HwM%zoW4DJcnex-iv9)79+N zmf@U5Q6$Px`k|f@G?kQeq5mvfHCL>aw|pA|eo>}X*+SIjb&JF7L$)6>T{S#7IRRQu z=e{0^aKycJzyy`Gd-H*PfSI0rn-9$89eM9k(!|TWes*8?OxwjbJsyD#k(~Ps!l0m} zT{mSCCA*4ovF0lrz%CLSVdVKzfLm-x689>ps-|jy>yXHs89Sz1EasoDv;FQ{;V)Lm z-I;c*&IlNHYSB=>TaEFje_Y3Y)R?tVsp6hKkGTfuu7Otr(Y5T`9lh}fq@H^I!=%FsvaBh?Zt zMvUb0v%fM&kyrbmmjN{@a6odJEamH6xTLuzX4cCNxD`@x7)5nh=OrlZ+V96U+ zQcG4bpepWx3G7o4LlHJ$(adeBp~vGeQz<*a{+uDBb`9rz5XBp#DAZMe709)$$1w+~ z=wpkGx|NFxvw4DL&}$h$m`FfQ08%#gU9Rd+E@@~JSi~n%fWBrb*7%SfBs$gxG4Nyt zbnY+_G`n)&Y6N-)i}t?YeA0LfLT6Lk;j76DmY5_Figg*~^tuE4RZglvHF9PEr~ao_ zdjFY9X-2fBpW)z_WP!juDn5fv1+@n}J#V+!nk%Q(eU#&x(ka!??pArd-p1iT679sG zTq=Oxi)#rau!#;aP5us>hk_dj@0qxjYC$D`&dNm5G#3A_prrew4=DnU471xn4GYu> zH!s|84KDFK-~;xa#yq7l{)8`3Ad!MDx^M%Tcf^^J=XYfiU#>D$Fu|Ep8AzsNw>5r!J|WK}Geo>sbQZr^5!CSdi4TZfcM6mg;rD|D6h@)Z{& zc3BE??mUje)>GPy2*xZP-#woW5TVk4$NDHuK6%-@oR4uo)~m3Nk7#n%TS3z@+ZM)KUV~GRO9D4ke8zItMbszPRNS5$zaC$*A3V zm?m7mxp-YIog5<7xO(ZYK@_2iPxYywZ!82MfaJ@DW38WT!gcz^>^C?^*EIgC8-Y1K z*y3Vyd$2#r-U?`A9QmkP+h%G{|B;xO7dKR^$MSkdpI3-}zX?U%wJtfzKuSa|5#}H+ zcynlGjJYr9pG&fCLVG4N~?n%c;kv^!_S*8$ZOHtOSR0_Nkd$NJYf zNIq0{K(il^5v*?jWyhbxk=4JIz^M&jItKRl_s0(WJKgN3JJnX%0fRh{%H};BTL8}u zvi7`oKQ%u;5XrG7>E_1qalS#~fB2@vj20$(g(*7QYpETa>Ul}^r;^>ICsGaJ2p1HR z#2{u_k&Vj7#krZ-5^lsLf)SPGOHoEF3OqZ$q+pN-XdO=abb`9(qQz%6wpqisZL>-7 zkZ)^S&LVNWK7QTJZ<_HFRHygr!t2ZY(DFzOA=13m44Kk2F!E+8arB*<>FT!jTrm@L zpG&j*rMc?+4_0a&JHZNS*_qijx#M%@_H%QF5Wq=46zKq;0SPf|eY2k4RF zT-`mst?PkrbSIqZcy{|cMB_)1KOSYxn^GDCnYkXv*@4-A==W|fFy~GW_Ro*+?Sae( z4gN;N!JKf}9^e@Jv1)e5Ymup=$2*VR-e+_q$Hg+>xeWBs>jy<;h>gBG?0B@?(6q#| zi@MqDS9iYuq@@qq*IElYtg4jkr`@M2M`}96p-Cz)X4_m?2qG*BZ*)ZLeZ{opMPX># z@Q+=N=DS;{eg1fS82Eb6HqIcRARtnrLaO^u$5lz2eZW6I6%Wtx33?_p3nop40j*Xi zv4$E& zFRyj}0FyV6A#w{t$(sBVl~pYrkd*H+jFm-L4=4|y92hDwIdx|nruNZ>?{=&YJgqgh z7a`pT#B>*G6k0Y%CkyCqkF&X%Ea+WV7gCdnrl<~u%O(zJb(HIrTR%UT` z>=@7Y%6FYOJv)$dx1~rGh(;!qazkMRhnOB`jnY6L4%a!ZwxiSaCOsq(MYD&gI28mIU{!<>Er>chBb?bT7huEe-k?1~!twIJrpz^Q^r4ZHg8XNZ^?L#DI znGcRlB5VaQA6H2BIJ3tLHHAE7ImdtwDa|#6u)HnPB^t52~sk+^J{`b`sX13H~OFh{zr1R64S7$W^(Xj&_+fwpAJ5aeQu zU9B{O&C)1H58VC4kr?-DDh|o325lHM8dXdQ*5U4K*ohL@mdE(t&-C^2>q<{EfQ}Hb zN>kG!n>N{1{58_x^icjju2UQ9mZ8%!F>g-R>EkUx?(`p;1CU0O#i#CQXBR`#b8i_&)_g%0zsC6<_>(9xxa+o zvIM|-20Cy*X=7LxCee%){EE%}YV!qE`~B=Pyf@FOOn|sN$>77j9520 zm=aVhV6F1$*|%J@xWMaU@T`CB_*-_hpdY@sb%z&_aahkaO&oK2f`BR)%60caN?&C> z-8yzFH)4t(rp(lIR>+raaE8CWzx5h!#fCco;==RHPSAxxEDI*(p_u>aKVxBHo7v*) zYnR1@pP$)pC#De9em}|!v+LI6H6tux?m)CeXe1v>&n7Q#6x9rwnv9pPm{hivYIO*P z6bkvUmK=Fb+^$Th{&tU1qWvq1g7-rb8&Z<3fz^3$_oC_@iT_^Lld%JnHN$AqW1ODq z(iyv4tv|&qP11bx1Bf!;o;sRf>t*hVSiWsamzTZ59kI5kgPX8)^En~^W+I;mU!H2 zcj}mbSTKHk{$TLn|9dIZ-3odlWdae+GF$m9h#A#I^Cf4>Dno+z(yy7^CboIh6dxOGvLdT5iG|?3V##P5pEX;?gVaFajqolRQ=cq7Z7IAV1o{!!hM)e^9 z*|lrxT5QuKc^+9g{@v+XNT)7(0J`#GvBt1~wuM_e*YeWcC&Az_{V3N4?%2osJXPgm zE`;y)E=dAJmmo*CXQDkuX)5ZdwW1EmnF2zvnI@pKz0l-9YBg8RH6ikERm^uhxhMIJ zFfjuwuRey5ltR0_ziP{vy%RJWj3f~Cktpp#e{KR4%zk4D8WtiVqYW-V2(G{Na%bgY z21nqeB#HLU`03y1U(^Y1UUbSB`A}d0jHNte3z&zZaf=7Xuv{NqrUwGFM!*1gHGWEj z2LIuMmn1_YtMgaABFQ;ME(S{73$?I|t<;t+*OXt|xg-?a4`sb#KD<0$TEEKgxPZ?> z>-g2^XW(5r@uF2iZZ76fhDp>h9zI<&pZ9|U2_ebGdcAlca*;Zj8o6I2lV3q?6Jto> z=9JmPlrb6-#xxz{!q-3PT7V)FfGoo+nJu<#4hjuHjkJQWY?I9Q_w0ZvfTXrHdstxdBBuveFvM}it!at zvF7_cnks0zIStQw~|{kU~PV)8{QJI^`dTF%^b)5Mulfa5}}^s?uU-d z5J5&e0;gviLES?Z!_Da!D)XVXlcbO|`W@0w!$Dw{@@wLWqSy)@YmduGtk{7#CET@y z={x*(*w<3n)ZoHmwue=BzOfhYkh>0cdId7NMT+Hy5sw74Z1jYKDI*YjL}%vvJ;v>$ zu?i9zVr>lvAwqJ-y+lpA%tVuu#Xs(GkUseWZVOT)bT421wMd1Xq|=T5Z2u|Qe`?;! z52~N@Y`&Tid+EH-_!G_Vjic{*@1Wyzw@LoY^BS5+VrFLaM|V;qm=wk^Y$`4~-T+l* zXJ49bWReB@=jBLYWi3YP|MjJyPiH7PtS7w6a1XPpeEGIkWw$B0`7)qK&`& z{AFa~4nm%?aq!5c(%VyYMw%+gtffemzm!IJ#5-AXFX`^6#Jb*ci~)>cv+Sx-VeLXl z!iM%n0pL53nI6Mt`tINHzt1rgjHC~-1}3JN1`|MI&zjO*38(({_c=)z>g{W{zm=hO z@sp|^qRAOyt~gkr2j5Y(8{8v1^L8_*kY+pr{7&2VVA2 zg|l;^e@tl8X(aR%Tl}dF`#IJ{02I(l-Fc6Ox-l23TX^(!R}<^Z0wlEOr7C*2!v;&l%M%5Xd^V=97#_S<VKXB= zCk0-%Jth{eMs-4-=-Gjvf7ARpePG`2FDSIrN<0>R{$xIbiV$HUZ2I?^?g%P|#Z{x` zSeL)zBNpp3K@!*ATq0BvYV8Z{XTES?(wU44_{1%ly@`M8aT0sxdgifcxi{smda`zv zEpS5B+J?=yq?6sa-G^c13pN%hJD^1oi4-AqN{Fl?+Rq}h70hJlZ zjl2DZeVg-Yu1^DiW9`?jlMMQr&<+zYrz8OO)bYt{O7)n3u20}uvjh}tDZLciwhmM) zQ%|r@4CtY|9^zDfPi*B=WRK}#!U6r}Yfs4JiXK}?<8f#KyB|${U9@a})OXMJ+-klq znjRPy|D&)JO@7lUJ7%*V_>GQ47mr z^5Wz^pm@3-;)^6ip61C*`)7HZ=bb&XHKgyOlF%)8-wl|);?>}UPh4zNs8;#J_e1%n z5yXepRiW{5MquKM5!$CXeD!C?Z$^=c%-Oi+@~~hXpvM_7Na^36u_(FCEuo<#m{V<7 zZ9LZ-zwPucG=Jte_bk|QDC{pLxd()o7gK5N9<+Zx-gIDtj9rX0+<{%?s=rCE*V*ju z_BnOBL3;PPJM1=BSoFb5?Bb0MVwQK=Qf_WOHSU$0#SB7|!FGlZJj4%+QW>*1rJ_`h z5=Z?|%RemX&qZ-%WkyO;<^aDM54!EG22!|~j?%z|o(yfHqV2Eb8?$uiw?!-6x69d^ zpyqV@!hFUB@$BH7>9rqb<6lwDbp3nI2i&hf!Ep5NBBa8dMC3dFf zureBLoVe|?&x`cfFUw8LLsnIxfDsGc>$&gPN|N`H;syTD)5+ku0z%N`?q6}(ZD~`P zGgYb;dGT7=Ez7SiR>JHxWY`J$&n1C&JGbh5PRT(4A7`suG_~r}YLs+~?BRj@h9_haj%xdDjJfS?4R&X(9x^x-p}Kx_c}sDgIZ|6aoox(*s= zwb9*~y98zi)%m@4HXpSa%G*L}7-M{; z6~%JF=2L(7kgGTXH7td`3*30 zGu<`mc?$D7womBT>ckL@Www*;(NXco`RjVy@?m{d0{v;B%!v!@jt7x9`13iLuEIsH zTzgS%1hVk7fELXv;{RU4m=h>fcE+r4i|7)+5t3ohguj?&ok3V%_h3-|sjL8b zF+5oR^RKxZE#Qs*?y)PQqYYLX(p2QGS=D)9!CIrpKKUW5Sh7 zgigXG4up>Zh(fUJc;pw~>)yGOYkQq|`$K}LtJb2moqL5&&4zIelD;@Q=P7nJR(bjV zt!#~Su6cj3xjb~Dy!BJceDr(+^5lsU5Q!_1ZZDq!f%v3^{Qgnh)qeghC`UmAG+=4~ zLbr1b9Y3?2e=ZC%kkhk&9=={4uwz3sQd4YTtXy`&cyk%8q$YIp`yfq;l6aI&b0cX< zYdpI==L>c@dR>VmuM*s}e0q_`AQy;qAh+J2bG{89tjm(gkkj#lvK`(6Y8fbGaFyQ0l zHPz;kZ+Gql&^1jhZrj)gmBoa=zk%iEk4s3AJHWfvMv^j{E(6w~LSKmWPJ1{$pGC z_$*66yx6yy0~s4d3QGD#R3bwFBd2P`Hx%vZgZ7;~vNBQ6{8VgZYbhDPKU44h|O0l28hqze>B?QmoP&uEfMIeuZ&R)Un{LPY+uyk)+GtxO?7-8r6WF#nVx03B!O%lKQVes!JO}iV z>;g##d%Hn30LkSt<9WFClON!m{D^Ody5Dn@H-!-W?^Qz`WHVySPR7>mbv(wg)fJha ziH%!TmFsyoJMZYpE1mbBGoIdf_Q>QlHXTdnii1Dd z%y1eDzvfV=r{suud7=L+!#fH=eNNf`NgeQgclGpeyK}a#6fH+BcHSpiG|AS4V9lwS zTrx|Vv$0pYK;t{e36qf2hFOBe>wtpc&2`YGHRx>qZIkAQz<);VfY^sX!FDe@^)d=O z#S6NAA~TuQ0E4HpoP}Hx+07d@QXsBW^(_Gb2$KkUJ6wuu&_g+OIp&$rW*>O zX2UV^Y!oz=M**9G#p6#puJKO!I{Bl?PcqiW^15Ob&4IcTTY|;LPOHlOP}bupa2GS- z&(XcCF@!uie4vS!f>_jMVX*b1cYKO zP|CuU_y1tF(X2G0$&rEb1K0 z6uh?7zU9rrE;h&g!jfWwY905tAjFFBDRP>~Gl9?&g%U1`C9V2vt{um;xN`UaRQH>~X zMfk}w!Tn(`DipT1eRZ@|xr1OkB|K=M?Hs-mdz3%|JNRgp-z^<)RJVQD!t3=+<>8@- zRQ}ntYH9OQN55_cBO&?nrZgk@{?Yb?Kl8L^WQacGVhhXb;?C{ar!?;4_A1e)G3Q^t zfMI|rK7qB9z^1@+{|0-+_srvLp#~AW9~j48t)mWh4eROsVFMc98HiPdE{I5LEY<+u z$CO6gnU2t9eC)QyQH@ZDXcNusqXhy225$Y1MMcBxB@3oJdgk=I>CUBHF6hfsQ>B|o zFA*^c%B-xYY*2_;(K#O!yp^yb)vswY&au_YY@e*7m-PvP<&eM8`YsGF_YQ)!F3MnT z*7n?FjDN*mwloz@GfI86rQcOevjVo16Yu>QNY8q^d7;VGvw7b+(D&c(JX2KOQ`bC&N6Vr}39qWE(YZ%k;Rr2_C5yaklx!AD%aFb-jq`l&0 zL=ir_{Z_ivc24K}H0*KO?{B!Tlt#(8V|2tLu-V6W$9c*)%xzPv;jpJnOeW>uN3+>z zw7YnlX!^2y*|HqHFm%F`VMK^x!KUr4=N0_#-d-|k4@7E#i;6DKG2Q2MbSGwjg5>r^ zXjt(FyT}M;TlE_XIcpV4#>7Et8W5l_HtagbJAPj3{nShj7Y~x&@(qrzwET?KlSXBh z+=Dj^ZdOeX+6A}ihC`l%f+ROp$;w6rQ+j2-r0xKWIMfuRA;x8lnc-djkS$1T`=mGo z<$4edL~y4d4p}!YDxDY1S@MVdA#Xg^n0q;;plX;yc^)q_kIcqlHR=2wkD?+gW?$pcJAW` z-5PDSJ$Dwi;I%~}uk+W9MqSam7D{$pho*N)mog&?3o|W7+qfQz`29M1E!|&MR8(}H zqQdS6GYAuxEa6wvR|FMv=f|$vzbQsZLzY1rEfl#MCW|-~QFQ~k=TUhX?sQao_gVU12br_ulmEUfR@vEg#$)su(sNdG3 z&U`)KdO} z_ehQbjhH(kG>!};cO>zo*ABc6BHqXNn2m<0^0WPnhiL`LZdX5J z2n~{|?V=x|)-2=+=N4B$?atNLha?eAf`C9PKb{~v*Q{V+if+}VOYgJFv%4`m6cm)n z-`OXS9r?;V=r{-_cbdgxODsr9P*c`^Kl=@{ofWebJ#+Oc0>!BZE~jD*mH2_1O8)2D zJ7WOw+E3E$FA{&Z2{aHB5>^5!M^!wCb!Mvi8)_!72q^@T*;0q=UpDD?ZK73U)mfXf zof;@As_Mj`V#F@}8EnQ80i7uEf;S;(Qi;v1xZD_4t*?0EP=w+^bZi`vEyozkg*v`9-S-5}lF zCEeW}g3_rpQqtYsA>AN|fRuFiw~x>J$2;FIW}G3$;Xe1?tFLRV<^BX&?`bf+KDa|& zE^qzi`$M=+Bn+$V`QW*J0kcISeuwD&{rtw|(THCR;pulS%F{HZA%PS%2Gdw-ln+s} zCLgxg82tG@G2M-;X|Gj;kDHve*XRd-Y0}{97;?Livpk;JJpOPSlaS)Wlkz+UZ1SXh z{Tk(Gi}&+q!az-ii5F0BFKh0g6c#n&nT$C9ElZ$YE$Fq#4O~;{i0Gc zJwBM^ns?w}$rJy4`S5VLd#(Y|fvXVNQ(S^bh8^?hi5H1B3Lk7j+V?+mlyjO73DdvL zj~0xHye)r*9sKXi@?WlY@zBgj6L+Iqh3w7LW4hWxWGU)Po0YZZuvDGNjRMFbJ_T~q zt$3$RApbhiuuY4DNv%U%_VQj=vJ*#JN_;9@D@6;@D!I}vtPpN7@u#r3Ye2qvWIw7P zVAy#j25*&(CBFQA#Os^VFg=T^w9c2^o6fbAj7v<5BmSHppY?uY<(@s*mPybIz{xcH zUN;lm80{0O*eWM_ICEZh8$3f6HPHM%iM`@?&8Ke5ADJ6}OZrhNGrzFVJPi2J94R562uq1407_;A4KD0vGh;AL?oHv zjE&Wlef8sMfhhPil1`^Q1u;m}OtT4pz*LFjcg)cL`{m%;s`H(pg z#l8}K&WoSQHGa7jNJESL=Z_RVm(ta|Jp4oTn598#&YvinmdDY}Cc+#Z?-iN#JrxEY zgU@0WC#_@BHSdE59yV_4x_|t{Nnb_Ppe@YqeQdLq*jTEZrD>^V0CApw$MVsMrcu?9 zWlQznK^j`L)w^{pKRyQxYrl)9qj+Gn1}nOAS5_~1-P}JJZLvgNblBHj%_2aI9dnp= zb9z2TqsZV9;KJ898_=^UqQ#IfH(Tvv^W6;rhjL;2VdS30-~G$%rgPqGu-`R_=c^LeD&A-n6Ps0%kdJ}07Ir)_hEnW&sCyt@e z`QB6Gtl9O$E8RoR(e^U6VQjJ@$Ng5iO<36vYt5KJBgmqKyU(Lb)t5yks`q?HOG@6v zPpI8BV%0{lkv9xR@OpEJ`uPbMQ;A@`*z*o+4cqMFY{9mhFch(}wh>GZ6_oePRB!dW zvP9oc4#>q}|Ipg;3=->S!dq+mo4e*`iNQgg|ItG? zSR_h-=uhLH*en6mgDPJ-Vh45zIpzCcncNV?43YqR=_cd^)e~d^nD}|ZDYKGz^ZUoQ z@+o5+Abe*&wwBLfBkhDKU zNIvF8-1IiVavcH;fhcn1h7K^mpQoQZ2%7YeAWZ3!M0-~@fqxd-Q47_-w*820+er@) zWlehb@`{6|>0Ccit1x?*424b~S)cye0m zPFua__O#EeHk@DVb?-)nULSaH$?MZ~YP8TBG!!rjAw$qQZl+Drf{by;bm_-kehwy= zQb2VzNycAZnNAT0A2Vu*=BfYA@sU_z!t`Yad@=h#He15YLmTotOkU1PRl#=Q$vt}D z?^x0a$H9WD#^Rq<+{FdQO+Kj77POz?%BQ=JZ%P6Opz5FQeE~)^4Q^J_b1Dvs)9KVP z2sYlU{n|8IMmv5vx8#n5NN1!tghjUFDMhBW%V(y*l3J4WSZT#|`0!%n?aSVk*%k;y zkjw!?i227g+L^%e0Rf%M-wn=cA_}P8-$bGh$GmzU9)gd$J?RSzFI0<=-HJgMRb&_` z#2KdDRBo+IOQ9RHgH<+-XXfwn?$dge-JR-oYXxUv{M*(fK_9&YRw0@zB1IoV3-7Ek z_j}K${mb&5HRk6ED{~0*uf#>M;Jvw;!%QBzad+R7sf^1NPD zX~GQ2_&O=MeEFbQC*Agv4JoYoV47?Z?(d57dSYal7zLrD^IOc07lawOqWp+>RIr=^ zdfQoV>w|UiCh9DSa)av{^UR%lLsSLgYk7W)SQqLm_z5I#)w6&g=EmT#uRM>?uuj_J z!>s(McGJ(Yl`H?uvM5>{5H+b~g&)@G=g}SM=#&X9k7bVkxQ6wxhpF84@6=!zxoT-$ zAndJHL;C3|j~+Uv{re=9%3fFmrPKc{iNjKge_u#biTir!xckOvzKEt?eEhxr07}Jt zm1g<~S9xVNJ0i-*h(W=io`475 zv07WEu57Gvd+HGnlDJoDBrbTq3KQ#}b>cCF#e~0_ZnK$3!>pBt-3mTA@pd#zWm+!- zcYfS2=tV_8c*hf0W*o#v%YEYdv#VY%=;U|Xyb&{1y-qQ0*M$4PjM)s6T+X3SzV@3G`m>9juh&p5DUA&!yxyC=RkQ_T=m_@ z1*}c${gO@PHJvzqx8~&+s$H7K`^elAMVmk~E7F#S6)wvXs8Z=QS;mcvuN#+X5LRXj zJwwel+uhn*6Ebe|jYe@=x=dau;JSCEE$rTS;s4YUtt|poceK`vA<{dS zB?t(x0+F^MDp@ce#*j?O^!NM?+T|(y?oieXbU|4O9+B>60iO3)ip{z-f#kQ^rSd4s zk&$wP*BhV4=bLtJPvuu)Y@gKlHg0Q?!_sFjDP7L?7BsT zD?WspG}~4x3vajg{HV<~|I>#OekD|h>IJ`RXdc1~t;AU99a{PD$>9>fH)QAO;$eT+ z5Z+}9b+Ejt-h9zKEOD0feWyl%UNhJsc!}Uwq|^8q!}vw!ZX;zi<;wz9JiMs zd(+{3Sw4SRes!D$@GslAs=1zjg{N~G)2JIt!h!Fyq|P*%;GCnOc<+-sWq*LwJ;WPyPlUSr@!8G~3SFPw0M7KuAXQAKRh{vl(X`Zej>f)kKSo6G6!#-h|UOoj=lZ^UWC$z~#?^VxV)Y111Yu%#38!V$EIhHoDW7|#0JuKBm& z?99NSrXfew{Vpy!JaDv(?Sq~4Ku~u*FuRibG_lcp;{)kTq>%C}yF0o?laG zlqgIYS4ca7-9CkSdmBg)DUqlmp?l9IBe9$wSJUHIj{HPA}+Y6SCdp2hMR?W2^l$6a}9wRQgY1OXCb^1YuNxl}|`ru_&$^gPaJ_GCi%f=a9UW^he=HBQt5l=R%<fbpAu`Z`S5u_`HiN zr*5-IGyc~8q~$8w`wY~HvUzE!hav7-oMIA+~hkKdX>}^{Vv3l1-gSYS6aL@1@vQ^uahz~K54YC zLjCI^cXrQXOYFzaT3{?;TZ8SNP789+W&Z8N; z^G7{zd~N5uL($?=Q<3YmKa=r$8QxtUT$+Fe9B~^PPuk&y1+Y!BfZ>?fQ3OogU+vpJ!XaV&0_V{y!_pQ4dhvVqxpf-s zLWedpwLX9}esFmByIYHTnRe*zbfF0tt8jhLur^Vw@(P`p4Ssw4&Ogo_n0$!XQ~VJ%O-#dNI<(QkQAe{P%#rGoQ-pV8r`z^Z3$vAo{Rj@o=S zsF-78dnzTQTsnm^>7Xi72TA(LIQwEE~3^^lvCG@XjwwYk*kG?6Fw9k z6dJ*=ooPmb^f1=3xBQv!(Ym(Umb!vVYRwne=#fonR^1|BpU4qCS#cgW+8m4%NP4>2 z&T<6Z;KmD;M(<^1dr93_0zofTmnD1_Rmp7R`MJxA@72#r&EdRuBup|fD=T{SQca}; zx3=58l2i^$99r4bfN$TZ8TDIqjXMDw2FXn* zQ{@8tIH=E`lv>=53mgj_vsWtZPazLfNVB}s;mS;#D9S>7E{NB*DVtlk%wL6tOb~2SO(=(GI+v@w0cC+$JZaMu z+Txl7>WH9#0_gdn-l%b-qY+Amc#Q~F?_ zcXT68Y}Zn8FJ~psr5b)o{wDz;;{MdOXpAh}=;{jjVcl7*;!$C?Vt1fD%Ft5Ny3k|l z_2k3ZvMaTbGVayuA;?699y&mX_`A{Tm5Qz|twS9|`>2$^hRm%N^@KQJM)^XchS`SW z_v9R~$T8KnIzJ1qU?GXak^7Dr87lP$tZ;>)C$QZ=#;g(OYvhIFhp2VreE-;^;IO3p@MW7$Ew#1LE5>{KH&g-j0`0x@q zpgHsEiJ87K2NqV%$L!*xsa6=`L>%*EqQpsH1v zUcZqnpmMSAwIQ0V;`wM#S2Mm;{pKxfc^=}(inA{RpR4&>F$BvJRfDr4p_ser+lKC6 z@^V;3YR1E{)YNY=-Fe80F1EPA$xt?<4w}|~>WdyOpx#FhnDXs?3Ayrf*NRoluRC9Q z(|K!ZYPxWxRi+NUW&?LTTb=^S5=5am5fLZ+MpAVGNw(YX*_ynOa_K)RYrDcHS_!>` zUpc{C7GR+wLQu&1I!oM#ETrzRXvq`tH*&kwDrgZtXEX9lK(O3a^D9YkfJueD*2XP> zTy}eSDf7kO0X#5Nm-bit%-qf@Ww(BTo-T4frp4!?DeUqSBOX41>L`VaE87vmWMFt) z@Ey?M!^B~*1xCk;Ef5uLC?VMyuhC%Qu_1~i(<^?OalRN3)?7&yS>lD^2Op`RDdpdF zYk!|Bs=g=SVE;fhmcCKtb49QYoc5OsqtgMNUDu*v6J|72RENz$?7Pd8je#ojak7w* z5OM(lGW+%$+PP9K6U_c+F9k@T^GkkjY)qEx@j^q_iuPN2=zP+c3!O4`bW$EX6cm)p z;(jS9Dd7F!lvPx)U%c3NR5da?fDHg%or;O++0f9?dnu`|AOti;P0a`tTw2wO0U?kW z#z{X*luKjM+3%eW_oC@~tEbC{TFor{k}UFfP2mV;?^`71OqLD_GuYrj%kkxMjkz4u z(svu>RO0dkvJ`5V%qRqCcq#M>Jh`LQKfk_dMa*N{|0rD28~nCzpf3pL+yQxWR;=bd z+!yiZkcCy$5wlF*T4*uzPF_#s?iy=n26bhHiOk@LXRC@4B8F(?pcC_9@HJgm`*bGn zH&uTS%tQc$rrX%^-97>^XPl_g`OXG9<`8Cs>L0hSojjxtmR`=oLvHe=q`%a`T-pVX zYCm%8;w+9Ac4NlIZ;!dWI^6Beo+4kDMqKJ7hCxjY>q#L{(up-86Gjn`-ot6eqT zA1Eo!G#?j3Y#x^=wipal{gHBekv}Zl4Q5RlcLu;*9)LrHgG^0LRr=gGnVOq#>`dle zZrHb;AZB{)y}m!odg{G7+XkU$@Y7w#Q_r-5;6at265bmy8{_<7nNT{skBQfHF9ZyR zqU2lQx7wF!ff%0eOg1{K!)E&3TAF_Nkn6lXcKzq8{Yrcq7WSkX<2!V^CRq4H+F-@+ zHrB<1TS8;vscZ}(IO>FLC>)z=>hREyz1^+zsm@U&6(o^7YU zTFSdq^vTCchQK8*ZY$2zW_Y{3V}VzpIdGz1c9@eYj#PgmT&Ld90HIK-*$$#j#VB%W6m!g{{v2exbwv36&~mobL=h)AJAUUA=zA z01(_t!!FpYiZ>pW;e9JGF1x>dhvm%<=-FNORAn=GX1D912M4N==VZFoOJT{;9A^a7 zGl&p5oIn}E;Tp^oH_*qiM5{sd3p1ZJ3%_R24_;QfW^35`_2$@xo$lJT~lLUQXdhd@6y*p&QL2 zeOrao-|6a`Vy$4CiB z2QkZ7R`bLuRFQw zaD3ANpXbGC9XqsZ#oLDlLdS=M5nGhEwfHWY^kt$(FO+tqOW-0aJjl8A)$p z2c9*B(s4;mkgUZV+pkre*Z#8I=SM^dhD~hL-Qm_Oo?mE&{WGE!E5^-8i~9*%j+{H% z!6tw-wM1Mub<8%aaC8(JcWE%q$AvKa<^*vKAuryl-9%T?A!AQ=f~^$#d{Ge?o}&ZM`dEVCAtMN`lS+cU$>8HO z_qFjI3M7CI;(lp$c#?jry8lrI$km>nORMdqsV|c({tVL6UbIxDZ3(Ae$r6^KjzT{5 zcRm?iDO25~eWgKu-<1@1Ej%`2;~=QN}{wY^@mQ2wWoaKlfyB|HnCC*y3 z1zO)=CcEkd?(lQ&`#iqAo-fu-JE{yzjnLojWW>vsj0gat}a*p`5sfgqHC;NGNFl&XcwCvUMs2m>m?gpu^L_kCr&^|f1c_?@X5l`iQ>jp1v}Dn`055dFzp=EH`F4FJbr(=Yy787rt+_9nHUj#B0^oSFd7MYBn|836 zaz$&gN*c>iZ^mo#5*YZH&uTwl0uw$tE0kZ4fGXNXeV|foi`}GLaICpWg9&TuwA>tK zG^t-gwxEic-y{3w@}xchYzst&g(%^z%oCJ$vk6Pb)VQ?rIu8+~+n_THS6)eh`mN4) z9O_b$>e$>D5-{4NktSc(XEwJTPj2HbO>X%GMVLld##mY(=4$s`)CiIZE}M=yfpqDLC<4+wW{kq$Q7~AYz&a zDq&>-zSKDtC%nQ~*$%|mMX`(#Z}o##X^bMRe^pV?(mVF`(f~oLqj`rn$b&|Q7cjKh zU{xk;)Y&t>ai32K5i;5@NB&uj2q9+~7Co-NxU6m#^ha2#buH;pA?(ws=h6}`A9zr8 zea%D_JH?jU{uRndn5p!5K2$gb(TGfULqObv$OPI_QNs>4T&R5Ric|cvFn6Cos9T?+ z!C*?qrS*3i)-X%TL~-Px&7h0vsER$!G!S{&LMUnEeYb;`+#jFK5gTh#t~Xmk zG--~y*r?W)#>w%w;P9xN2K6Dv;~)-G+e>S9SF7b;X2QkR-w ztk?$-7~Ts3@o(xO@_HxD;m`A9i08joRyw;{s_A0?E9re%FZSLA!j{AL+Z-9OfbV1X zkV1DZW{@}`e5WV1QCi@E2@Hf3*G5XVR`Qbac0*(G@`&GpVEu-sy|BHC9GQ;oOU9)X z#=~7^872Q50wiAMB~O21RF6LiJmUuz^BG{s!8Hu`_QI_z)mwI7Z>fzuoG2mW4tMT~y6pj#a#=!orzFq!v0erZ1!v4(&L!ggQWpL= z{|5y&UB8Go455rnzZS6HV50hL{e6v~uwT(9Wd!)0{iEBZ$lwGcFHU1oxWB|pfCq~_ z_u6P;*j8(mo?@*Uqf4h9!bngdCB3OO=uukQt%?RsOeiacZ@cKPQYOsa2$v|EjbQz) zPLI)M7LAZcFw-Q0=(ueJJhh>hmA|>26^f^;F6*vEG|QV4JS2R} zqkB_)e^+pzxbVGX!0Gqg(Y;gK67|{icnDN0pTj62?gXbBl&8d_z=E@mu*Xqq?is=1 z38Wx%FB!`anklsJtYqkM^NO>MWgVEdj7_Twi`)e$QS0`o=ZSL6!E*>ASh%Z7E&ZrNQ{PY z2a~<9zzQ1*1>Ue{-`bOsD)dIOW9j_X$$nQ(0}+a6a423I{8C%0Jd9>e!7zBiBbV1K z_Kze@nfv`1hZ?VRi|Q^>%b)6cfF(V8wwR@^FW8=;j1~$y~^VJf(u!H{1 zG8y_g+qJ_H%JNbKKDvCV%74$yr>90dK~q4Hn|a=$da#g!Ei2J61AG&>Fea!=vZ=8f z#jYZYg13DlsK(ij^Q)fP?SEJnhl6maQ@JD?Yf8Tw#(P2fVH6n_T|57Z3;8&e?D)(e zrpuz9hoFZ2%L{oo6BgViaO#5<7mWVQ7XabZNvmT#Y>6LFpM7}G=>Xb82|z$-`nN=i=S{I^%%#KqdvOK ztgw{`M5Nw3f5O;4VUb8d#q;fXppdCe{&eVr6jKVI%RodWn_lXkQdWN8R*Xer^cpL? zr62k8=d&!{9zQn9{5vQ#U=>1{Yx}T6m%`balqq>8<%A$>^d{CNFw=bI-QYnoYBtAm z)nW)B2{q$oNd%aNz^c3aSc)ndCp2D)QC8js&*Qs1#J*f5JPGLU`XJ?yS0I*AefOW7oZMMy0_RFkk7{IO)abIS{J<3n z#QHn!JM;-eNY1*DVZ=7u$&Z6Ig146IE`v76h!E50#txq>X5=y%ddvkBTB;y8z6y6H z7CodlVg_!lr%k=y^17w9vU>l#)DAuFH#OnBW47E%Up5E#=4}%sH@I=Zi0i%@NK^_Z zq8v@;)g8H_wT?`flvuewD2sRTU#$dBOek7#Ja^fdfZ=sNJlA>nf1pqvj13O!BGk0B zB4z4%p!B}b?%Oip1nTWUea6DNy6FQO;8JIVKC+QhB2FNAvP@q|WQ)4v)k;Yi?c2$R zM{a1tKt3s~m_q$14{2l&^^APH;}cfRoh-aC9s~tTW&q3Uh17!LtMz|Z zJEMpaG+v}8uD-&Lh_k(v#5q;5y6pB*>)yR7PdgG$$XIk~~v9u7EOl(p(c%=4ohhysr*tt^VN< zfvND20A0MF$v-C(FO#A+bT&>!StxT=kl_1L#wXrr;7~K#ED{sik!a{=^?Z6U|F-B!y-?N+EV{I;-+XSJWLpO)b6yAr9mZtA zl%!ms(4c_h4ng{kjh9_&Tv8%IakgsqT4z6YZ@$W8pweQJvesr1XU*@9sX#U@2uQWUHacZgpRQu5@ZOx1NFrv9evl0jvfwp`nfrPa%sS%DF$EpXpfAt0yFYS+Ch}t=(6%)#J_76rcHQbWF^^-BCx3-@|Q=R?P>^W|!ig^Yo?3LWP%_ zF9w?h!IJ@d>_ICDk&gL*Hi2`wGx--<|q2cxkp%xRnm2WSUElKdqJQG)_L z#BJ@9s&n*d=EvSJ1y`F?#?jBUxZ9KPc}!TY0V@+58~dxu$IXok5rZV6-gfyn7=0}% zA%O@rxnNyI?UJn`ebUZc*$Xx{w!e3!>I~rPM2-2l)w~SU)XItrcV}bK>E4bLB1Lzn zd7D#I=+HA-Z4<5%l$Bw6y>Z2(4mT0z zBm^D`068_0{10xC%%l@GukZb1gA)}MwGu!fV2VEy#Nqzh5qvXXS^FDG-Z34*Db=ik zMZzHIOnj~KKUl`hFcj=m65X5-bUR=`3*t|cx)Hzz)o;icfR`YXN$>?FPy2I z)&ziA6dpd{%PuJaJ}lx9@#hfIBisR2T>o3kiJf)&oJBEF%-O@CU||)JCzy(n?fH*s^+Zqk7r!uqm1Uvh;F< zFbFMf_Oyon93dOGA=h0YGsNRw&D4G^3}pR{q&TZz&{m#;Z1WnCk&mbOXW%|7M+36( zX~pK8yGr@?2$I>KOUMVh>rugl2)rGuTPX;>jhMc|p$-b!xRcc~=sX{#g#Sxls zf*ACa>3a(FL%jm7=;-;=pIL@#|D|AhHKuIjZ2kThq2jRS^0yTNO|bp-dX7V;VxSq* zgAu3tMuiUJu-caIChVzPFKQ+-Qj#3X_nlPheuHfB1}?1fl-$V7)JgZR`-|FFmm5y* z+bLgQ1MZ`Ju^OGJ#A;A(<9&M5n2jvD{9Nd{;aV~AXbUNS4eCQWgHzfEMk91G_EMd0 zB=JyC6Orpg=c28YeJ-RGO%9rmf+{v#LN zg1|Chr}D!m(0-?^U&HyL2aXv1A3Lkywd&aN?C6>KNp)>BH{Iwq53T-VQ}>2vb=b7K z*}ur6>a~ndZkbQ7-shsa&YdUcWm3;ug%qbRR%$G$u%@{$+1Oi(EMuYE;fUusMYQ#T z7l<*$pWEU_;E&klb>~C~$jSK>pP9U`9se<Mab#elA8<#LwrUkU5^*9G|H zl#&dd4&&hm6q`XCYs3FmXyiK?iYTUDn#{OR* z5mBwJ6<#Sq+MMEl^c-*HB0#Cvf@udNXH$dkm`T)6+VWrIWRDBMX zI1z{Rp!A5>a~ry$tqE(q2sE^SVY7N^z{zen1`dpe^ zxO{k>GewE`8Anb8E*9(1lax?0D4Z;f9617IPQAnvU$>~cho`RFdMe>|=2#p0)R~B) znXv~UmVI>CfY)^B>V^3((r^)EUZ(2#&XpBHirsigzv>P}x(LumafzXv)!Zf3Ir~YA zz#WgL`L$fVz5zLj2EFI`trFH4-;+*<(QdThK4C9OnnC4W$rahb>tN4J?2f zNDQ6a9l6|dZ$eelY)DGqr&n;kSdJM@!8=!logCUs;i9HQm%aw??fJSieuVsWqL8H% zPOy5QVFoIjC^;6%;R#8ND8Uw`FpLFkD_0)l`yHO2jheA0eOE}nEh*al(5xBY1?3Qz z+fxk1gaRQ|n^4judTjXYt7nhJB`|1b$&1ndL@p)OL&tx?mg#7?FnSOkMpzqP#0B4+ zjBAKAC?x3<^*I;#yQ2EDpJx~U1P>e!h)rLDWyrygG5$=jGCCrh~H2QGVYyD8!nUE^WBA7OI&Dxonqie zOvs{Ndh`MMSF?KXkm>kaQJx(?)=%g8^Qk}jD$noL4M_%(XKQ^UlcPGHcR7|XE7}r+ zhJIr6K@-TGSeq)7r{(6mZwbQ5Ggl3t<~A$Cj53P?HHFujD>>e4?%+^!pt9lNef6=@ zur0a6?|M;ru-uTR*WwoO`7;6pN*q}8y&?r;AIrV$h=EjaTEf{_7 z>u1SgjCwtzqpxlIEO&@zdXL$-?)Fr-c;b}Qw%Zno0P2Cm;_y~42cm&vkfoS_WaFLtAqn6@1Y=W4sCumeJ|3&{A6c*hoJ?F? z++s2p0U9`LZf@??wk+A|G&)e|``ws9F?HZw!6Kr!vGMV+5HN_olpnN30XpnE=Rcp; zmVkJwvbcCHXJrP3%u*yQ!-hbx)?iZ=ucTz~oOF&SQcgmx^0(tW5XFkY#cU<3S>vm{}sc4*o|FavJ1UvbRII-b>9|FAgmMLWqNKL5-PU*xh>( z2x}U@f_b^M8U_vl0Ri+)8UZKK0W1o>agmpo$MCv3{0sw!Xb5&Orm|n2snEJWc7 z+6r-@fCtYJL!Y&mqcj|6VB@4cz)GPTo{Lj6ALoIZW-m-+FDyYR3D|-IF)zbswIAf$ zsii8l%Ld2C!yTR}R2AOhO8sndDgG1$`3 z9UUEK$q?wIa7I6{gEAN#hw3)9l!%fN_E?r6CL|&{x_lc{!DdsW1%g9QDTm$kPG&PZ zvg1duWrz`S&I8IYt%Af~e(KP^>~9hrTT)=b`e|SY`mY{lF258faH4eFpUOb*0PK2r z<#)Y~)Z5?xJL~EG6|LNtO?7oV@lu@z3}B}Cp%s;@HUF33)FuDpZLV_)X%gp6MgT=N z0(v1o|D3}D%kO535F#Wkje2>o9L{b&M$XEL1+DFj3va;x0wo4&Ut}0kv{uU6I-T?k7qjqi4 zlI`HwSXiI2aN9>+mSp-YQPB$h>7W2 zz}b9+%xb?sWjGuVPaA-kEYee(;|oIq9Gqd)_gj83=y~6l)t0GwoQeeAeZ0t)_o@YD zP{6%)TAHu7)k)`Z9p2~uf1w%L{zo$u1+}|n*WBC{sx~0*8l(zOCnkt$|6-`*V%_9Y z$255xLTtERFzq0saR)0^g!1S3*~A(_o2(Y|WhIx9Z%M)vF|1^M>%{nMtk_9(-z^40hWk4x8orXJ>zY^u$p~9e zpJE!E!VpT6fNVu_Rw85;QmKmRdHWXRKy4Zh>1}wJ^ixUKE#UG4WodgVje7Q4UUOD^ zoEUT1F5$I#U3^#KujqK>0oykZvm|bmq@++DFDm>gC@5g)lTbf?{0J>D@+UuBZ^Td_ z2kBIv4ZX_h-G~vyC`pn^j)+L#IS z*6&8Nm}iOqT*v!+EA9GN1}{{L?k<7pgn?LET57iYa97vVIG*n)6*@P&A4P*rL59Gx zq_Uf10OK2pfJR{WccIGeA}Tk+^XyNK{aU+xl%V_G49bhQc48hL9(BuHUgs9NjSesh zf)5;Q?Cg_G&gMR6Ki=96$%6Q-{51#`h5dXRa#x!WcfU(hR4l?j>&ju*ax6?3>;MWA ziA3fBkx60hTdXk$mRe;^?mL@Q{4pTJ8G|8IoaGFxtd$bmUu9|O@E)J;c&%pWOwG)U zcBk^8?jO9)_L~Lw-@Pb&U=z;Hazn*(6)2|||L*;$+2QA#hlz@dJ^N8wf2sKy);Y&d zO*{vE^+#m^opRCC#d^dX^?V^^3&UWowdUb6DC9yd<#b2yy?i;!KK`rsBd*={Pw*n( zk;#da2qy6|%WiuISHMnTQtw~L-x#DO=s`R}B1Ix=QZl8fY$N{!cv1C@W$%zKksi0L5;MIikl^2m-sm_B^ zLGVkzYZ$QDs0TD1#HOZ3Wb%9Gx*x5|Z!r>}0TO-`(0&3?LUAsUq)k{a_-fZZ@X}xR z05JUYROol3@xP8<9)Ygk%Olb?QUm%pa&Di?w2)K6X|t@4&+_NzSw)h-%ZXixQQ#U) z(3SADX{@h*wg=5QMu?m^ROirN83h-O`GRep7SjdKv>WU`dmlC-L7P6P!;P@P7Wh6; z?1#>gHv(-AgM)*>twugvH#%+{13R}a_;iDbLGmu(>G7%qv{tEE^-ShTR)L+QP=UV& zyO=5gJO%C#-T&#~E!49^gBcI-uoJMI%d2byyo*?gdRg{-3riNO_oWTr&1M3$>4rRI z#1ZH@h-HX9z&587AacLK=kU1QEo__Px1M9Xzgv6i2mU-1ipBvX&gJXJb0C--IpJj8 zz`C)T4q~-F-tMOg`U~iLZsFepr$oeNiVWRXLn7d_;3p4Yr|jw7R=;PYJTv;2BP<=_z?9=stVC$*^h${TxB*--BpP z{yqEjNob7-ZLqY_K11Y(yerh_3#XWW+kmByGZ&`l-1*Gz!k^94U))%2Qf&f5R1N4|+YDV`_ z00sDk!Me`()mn?X?UmgU`@y9TB7q+QKek0oFmzu5sNa1!@oN&Gt8uaYG0b>c!s9yF z+>7%C9&}(X?oQLGw=Gh%_5+*B!QD+vKi>~Nkz*WmadJH@H~Gj?ag^I{Yt4bH78d{a zH#G;+?sN_vU=Y}A%QZQvmT2^u-ID7Spe1OA`u^j`rQ12>@YFZ=tm`R1ut`6p9xicg zjuyAWV1SZ^aL|N)S)oqGWBs8i?=48Gp`Wd1`g6^LhM;k7MhsV2RFqcqP3uYQ5ddg^ zrrSMoroKe*v%coX#ln3__2OYzq+FIWenH4gOi5`tEs~rC5Nc-w?8B%S8#Wvtt@5y1 z8m(Gi^u^b?A|_K{l0%J;!+-gGp`8Dr!@YEhvU08Os&GtODV!ybs9$JuyJY-YHTek? z0~#974A^1J%7t^)8Ci2z&r|(qyFh_$%HNkbd;mQb>JC|QaO&xs0AHuL;o*t7{(fDf zWDYN2!P0-9+h@|4$x6NWy3YRW`JkF`ZW4&4$a9a2hsRIOd5OV-R;6=ssW9#3z4scj z_zKIVxk-VQAU&f*AkBI&JInqGPQT~q+`4!G1GVm&xcYFBojxXX-|IGUs46k57j zf}oW!P2P35_zs5#?f(9bBRvBHq$N5=*+->&4+mvF|F{eEKXqG-1QF(1xeEyn4h}gr zEzMRPm7VfM(tg)m8E%Q&>~?yGgq5air#njps}G2-OYoezcccyfqr-QLDn2)r9kE-pMeUQMU0WC-R^ z!NEz!Yzy5%EKmLYx#Z;F9KNUP>?Su1EVHs8B}3oSOPNn|LZ`15`>G}@fF8lC9Xr!~ zj?f23L_}nu+%>D4uLa6Zshnx!h}otpkBz3e-Yh*`T?_L+E6}1xtwN&}Ee7Ma92B}> zE_#=&sg$7L*J>}2oIrMbaaelxo`}%qTu=QtllI`N_#x{<6G7ME#GrCJ(Kd_j1f7a9 zft_92UCyKCUVGwq@7`Sw@!GAw!?XrLGZ^0CBm}x<0Gc9Bz z?VS@EARw}`?L!7kgj!nC?Kd|RaUpuis&7G)M+z<+zqdT@Lp2E%*IN0yxfTV42?7Fw zP0(OEt35qF_GTGniNLqDMv35ENTo%$30ZI%z^hTz@$dP~to3jKozP;6Y7+z)A*JQr z{c*wLO{S&<>bwX|gkI&8U3!X)3P5u^sVkfRv^dN1da`UCf+FZ$on>TyR zAC-En8zwc`04PG@Y_ltaBb+(CFj7YM@nHWv#LE!Sq^D5K2L<6}bo6tef`(fJO$v!0 zj5rgaxRV5uIBJCiy&ney*mDz}N1nqsDdcWo^IW9?Fa8^ovIMX^1Rg)vb&_~mNJnOk zp4II;4bmoV4HusVYQc13I{<^8`rfwKNc=ORl3`gIyOX6;I8W?q*UA zr1tKKVmP>+_=Vxa*S5B&wYqC-Hy>FRnKTPIg`_(!7*#bDbt#Wb)p{BjqkQpDOHk zMlT)|w`RmuERHpwcrExNG=WUMjwc)&8;fgP^ff`_%;=~uV|dQKS~Hv?%y%5dJSnb? z9<=4mo|(7v3h`nFjGaE(_;i-~D3fj$j$+N()c)>`BP-4GhD{y;w&tKluS~uWnuLOA z?b|Cte#q?5Z<4cQLro)YFTS`ti-+S$ooHA_K+G(4sCy~L_fEvPzTuO9mmTAv0V9!& zNFKLbSRiX6em2(x-!fl8LNUY@BCS8B1tr>MrFCWQDV==Pjs3Jc6>qI2exiv=oWC`1 zVsKqxp2?1g=Tux|CNM1T>$ZKxXA5R~ieJ7|z&V8}#_%9cRoo)X%iA|Tz0z9h{oXHJ4yB$%=nlzmBVuW&NE5>sHTkp9adz3aoc3zs(T*m5&{S*1bc z@NWfO;!wAD_iers*HFYokC(HPDXyMUm-o3~^fwf3sk*#jwIn{(zI*@3AGJwZ^dVdB z9upV~$o|LEEjkmjmSyUjK&>xd3O@Do9I6?gBt(x<)8M^2U7Af*$?NUA|MTPwpKuWg zy3~5Ti#UpxDNgS=*GH|wc2^GaIa{-S{|JWu%l4g?OYF*N-io}O#?1-n+2~mr^!jqe zn~_PRTy8vc*(n?9AS3h~bD!9uRP;MY^DX9m6#M6IK2JqEWp?{R``A`}xsXXS{pYoQ ze(t0!GoS)jmg}j8O8`dX1NWEEegH9(gBVxPyn11y_>#@;<_e?8b>%p7Acx)y_60B+ z7fK;YAeg{brky@N^1Tw@GYTQ@{=eEpkeV?)bz(`5&O?e@3>mpgywqE5uqvQ79qxy! zdamO3rAemjY$~vSG$Qy^S}F-Hslq$M3{d5k8@&0Dm_C2m#^#+t8DOOGc#mHSw{G!? zF;PIM!9vZ__hXTo=4A-^6RA&i$<)UeM%7luY;`dhJ!K6SPB23ijn$l*bg_chkSQBC zy6$@9w-~c>A7*TmwzmP!V+;UN|J))O6;*CppO_h7ULZH|WjDw18X6lXAYeBE%Hs(C zEDWM#+6RDwzO@#4jz9+wd~E;xSpfhEDvnVGpO~1Ce6%U-?c+1QU2uI5S-VL+I)0X`2HqnsONd(VG#F`i2K{vBS6LkaxiX zHifv2YR{d8W*9vYQ1Ppr49#b-Et=_gizmB1!?m*lZGcno$r?bI4s35wRg0-f@Y;U{ z6vi>gx$af0o;&M}P;tb)-}}O!%Pu9w^!Dvra4-3Pw$AL6837^ z_oz^BJ9H3^?e6GPl@`Qhms;q?^{r9Fo~C(6^Upt((40OsLZ&(Fz{q~*$LwJ2x~UKprC4r!?!$O zT@gP9V5q5TWAp{&89a=c*PbhIL3H7~T6Jajrk#F`oI`H|=-5)ISl;Sq5!s1MlV{ORh8*k$tdYs`1`J) zb=Y^e_Z7L5F?d$LHp(^u7A(>)Fu@cJqg*r7z4^;1-6IYFQv+O3M~Vhr_LTy{WYr)= zdwT&^&s9B-K}2aFkCE)wce<366vW2@aDd?gute7Hk{uOPwyw zTkhcGBs}`)j}~}Uz(7F=Nm2>nEv;yL3{Ww_-9=fGX#E03fFANr^~}sIG_U}AJNZtY ztOYZd4>KsS(YMa24OS%EL&LP~1trL5Bsdj1Ku zpYYx?jIEve(Mssqd)}2^a#bxtn?`qoi=2?05v%foVXy=b7k93dKpx6~#W86 zB%Q`Fn40VP7GNXV{2s33^t1ICW02~rSFhx=dX0>y0ReRY7WTk)gagB%z{`J7!3dC8 zZlCWUPDNQ+GUw<_7OdrozN?On@HCz4DI^9qU;#0~ODuNWnCm42Y}N?Gn;)XTF&0H8 z$g&3f6Aq@_W!4)$itKl2b&BiLa_?-o2hqA+&MYZZ0fi3>Q_wD(RR~@%+or z;)1aNLz>k2`wIc!T&%Gcy;RY5Db8URRtz#TLZyo8Wlt&StCj)r0-o?c0|&g>|8f5* zn5;Z(et;M^2@SA~5O4y=$1N&hz6OjKxl2mR%gc+lO-6F@>x)6#l_UuzQLbqy5B z!a#4}6B1sBN(r8!bM)`4F9A10DaD3F0U$Pzc=hJZ8w}=+rXNHcObs-Ib`;(=KqGD% z8yk^`i$cgFh#%|jV7c)#t8%x5g~=#xOJD3b7L43S0Emlq7TLkErDiWFqG*Gl0q5@& z4y`@*Ogkk%@>;~Lz)Xxp$XlRlH3Q*e`@;m$JS!baRBvHw3K(!2E}MH2r0SDk5Tnys zWOo^9I1uL53=Ln&V2ytQL-bUWS4m%=7G9+DT+c~gKfmiR7l4nCPyXZMy$(Y zJOzB5`bf%?>?Z^nht3AhyRZR~cx(HVoGG>SHXD4H=A{s>;obBjqUrR-)hP;Dz6x2{ z(?)K8cao9eVrG7tr*cJA`V`!iL?#%-YsW zM+M9bH6qUeOxrr6ogf=<>~# z@xZ0`m-7De!G9d{z)EwSIYalY+_hdy;+roPi)HG~=(R;wE|@qH(A#SGZ^|P6yWV$F zyc)F6pFekk3I?BmV9;j_%oB_M#Ig6sK=^?zONdccQQ;5}AcyUjsTZnCjjybxHU+>7 z>RIce`Ip=zpkO5NflYRemyeGa^32*da>#H*e>3M3nJMQXsB9{VMe~3zE(<&CS)NlAI#sCD^1L3WAc~4e+Y2~l6P zuyeTn#t?^TAV>aprQ0q0L2u@zC|OS_1|hRRG^PpJ=zy04hZ($8VvjPQCoD%wosBv%1&zsp4C#NzS2XI$-X&2OC;l-;Y8)4=y|ba5{TKWXk;k)O1&O zH{x9DujdbJAPy`eBf~iV`qSn=$e}nL??K6>7QAY*k(FEh97ul4I$shH;Oc;&d)sTf z0wWlPS|Ht7Gz8Efjkz=Ha<1*jNmxFez5f5)_zNK05Etx2NRgQr`~PLHHXZwGuh#BA zszL)Fse@O5TT}dN3|=y-*}+_0V$zSAsLGlCZX%lA0pODZ0ccP_gJ6EoZ=z zrjyX5L?z=(!F<;6wpKOB4~O5|pUVdPq8~6eeeOnMvYy*8d~W-iEI6jY7y7BKO+{Ci zddEXqaC1c8s$n{kDJELBpn83Sk$cr=clQxy`JxSqUP;@b2?-%JH8r+z6=>ChFbzU@ z(Ncl2z`%zaWC%aOJ9*d|75(o{U<7?oQh@PS$wM2|3=D*jMzXU7O6FllU8*p{?#aM6 z+wPv4Uqbf@#FnJzw%Dq>48##gll-x^260nBTp~*X%!05KX58&tA2J;DJey)?SQ4KZ zKYej4I&$THcD{_#qsN(-smkz-5DyZ1w1T6NL)~GNM)gV3071SO8XzZyg*^8{QZT$*VbP@Lm6Na z8utXm9wx8Kt^L!dnw|`UtV=hK_3y`@5Z{w1&EGR!Q8C@~%xtmSNgkc+(PQ2tG;dic zyED@NKKEv^)bvVOjx6|_vy#tl4^+R(X}j6izYo;$V<5$&#cp38i~MYTWWUSy?udo3 zShQEesfu<#m!P>mgWf-nPMUX%M#PO7$8*Ta_42n*H@_2%XY(>k6O89DZ|VDf=6So$ z-nC}7b{$tW>d4OdwOeD=Ro&HCECy0x`67}1lM1*cP7pUi1@XQns0O+6to;BMx(7gq zxIR>P+hH24B;%lctS^EtYz2UYn1n>((IaV$1+Yr$7cVM-ZlSNQk8CUuG_J$86fajp zx4ysqpge!~|LxWnZ}z=Lw_e09BU5D*BEjJ5%(h>y>N=k>=T`kH_x3SaIaej>M-Glz3I$dw%&eqH+W&S#F{F9T*s_vmrJ zqxz5hJ&Y@5sZ9axr^U~@uC2N{Wh4iO%!^~|5QgQu9e(AiMySiG-TO7YB4&Lh? z?m;fulAeK{-ZL#PeNhlS{HtH`Z}%i@8Hfx`hl39O{u`H|!{ZcC(uMH^OwE6jDN3lF z;=u}aSq%UH{}10?&nI9LX=`h1OQ(X*gDl?jDOT2#2>~it7LY9hfVS*^5S~s`_Lnb2 z%5tIO7%Rx|8B4r0;U^XWJ3LIMJ@}PJGRG9?GZ4;zU z8YU)_F|B0PNoa3SgZboWh83Xo!s-~Mpicm$LLp%MZ)qDADHH>`=o3)ofQEq&#T$_0 zu$*P?{~Gz+F2qSSFqYgD;HhbAKTwj|^eMs9HkiAT)PAg*WpSuT2}(g1DHfDTKw5lX z9itMY4n)k`{`vWeV?D$wYHG>A!eOwW#e%Le0dmXQFrTfMwdk=zlhYXVdtuy9mIR+k zTX}^rajlDFF?FV^GU|p|fZLnEUrHYz*)wLZ(juR6#@$5FaZbL~!!+XT6Ws zF-+TZ(+Z+UxMCZK9&Nh9S_ThrKk|u1j8k#8D7xS(Q#ER*O7&BYjlC5 zGZjMOK#1IGhf)`)QS&KPJ$Z)HTPF)(CCsi5JLbNy(Z{>=Z91*A)OZe@0;?dD^jHSpp5hTq z?B?+akc7TZPvc?k-n*B7qlKZ+;aj=c&_%-pw6OeOVg0@P_d7kmqb1(h8XP8YG7xa$ z)}b|I66iGOS@>}mNj-Rkp0BfK?5c;U4w1*={1EtRnPhhRBh&FlYc?9$T@0-PFJB z7HtWKacC!H5C1&E5ZC(Ui=v^Sp-j^V$nuM#ifpgns zH$wBC82qPJ7)ir~Q;pO9w%+Yj2#|1&tt^p=pM7MLfq3>HJyW@oScee|{ePWU-svO+ z7Z!>^Qxq52ClM6@iyL?|-H@Dje*b(Uf;h_6Q)5SM?KfNhB9vspwGku!6QWzapaJt> z9L%B0UKfSIP=54mY(h|ypXA|D8G@R9Pdti={{MbcOkMZ*)x)^A(I=ZC>Gh~RJ( z@nMkte~Ge+e?I|0$?o`}68f*8{ZF-nLi2m)za;^R%rvG`holQu>%GvYU(R zLmKx73D6t?u&P$X=|JeSgv1V(&(MC@3}6C*$1b_ZEaP4{HkTU=G#@K`2T*=%eT1C0 zrQg@SVc?EFL*SqU;}|d;iO`$|%z`2?WQSvXwYcC8%1@1em5XYk41V3{Rnr zNGZpN3(7sHLGUrFOJi(cE@}W`a3T2^&h%^(>{f4f;!PlmW9J?%7WJb6w>#?q{Dr|r z3w|GE2oMn7RPNs|?%$CE$rBADgu(|OA3qTyKWg*@zt0n}5o0jGamPWcX3^x+qh|8| z%*%zcFRj{okhr19u0IKUoM7lF1+RD>ID<~@-w#MXJUS^o7=8)R)0COVMg4h;}DgBhLXaKyamG!qofl%3X$2P6&1L3L}UZShlfF( ze6wce4<`sJA;hDG%KhCxIxzc@0pu|Z^o`yL9M0_v{APk@S`6M z-av({ZtzZXl}*3H9^(6!b&95@QM~1TfJqhvBq!L9B`%OZndV zu!#MD@GewK1I2fk(Tjg==1#Iy`6t-3+&>G{`0J3(w7G~*GmwECt7mA4pJN0{;9x;o zw;MQE8i0$ooi{?Ek0}ND=3%>P?f)q?1NgYj5jsD(Cf z+nzq~mX8Cc*9b(e6I`n0k1r83Lq+^hrC2#O!=rfHJv;JM#(k9!7SoU717L?6pfPb7 zTFEV8YHTT7c_$2T$h{+Z_d78VJi?$Er0oI4XIk#!JTyHpMh=w?T>H6uPiIcm0bbT) zG05s=lzu3TdLU0@i%rM&QT*5YQGS2%Ib;)nDrwetgAuF; zOBL85YNwo2^=JEX|5$^+6nJ1l2rz=j&)+8p4~MgDGxz}g8XA;8{iufXiox&*3tjFP z2tcjT;$+B&i$^7mL0qi%A~fH7Ur4P3O1UG~{3Gf7H&`ZM%*Mf-J9iFp3^gR-`Fj-s z>6WNKNe{kO`H%(~@6i-ZmxTcukkw#-Lj$r#!X`O_Jw0UxkpsFcC?YOAg_wyP4y9W^ zV`$jeLeBpx#Y^J5j(Az%r$N0u=dlNMx+>J9P@Vcj@L#?T969t&-}d|_i0-}$0;2-R zo!zqFDF&82VICYB)QC?<#+=mGW{OYmOJ-1gg^dU1^0QHK*MG+X) z00D&*o?0z5hSMB7MuI^;Ul8(wfE}Yn>LK)1fJoqwYXxp&N#{xYXbHz)P#JRjyXM!x zt0jphYC}wvVtMiG8NxS~`NmElS0#df>LIwNXz1zj^;UPs++Z{`@O_B50ieT$(hCSR zWEg5@=0i&>#9T?h?joQ(V`?A^t@Ie^OhCrXODF+74;e@@et8}2j6Ixr>yLy2XkwZK z>hVK}Hg{5!PAIoV=8`Hkjb0nEHx?Y)%UWo4(QJK#T@eC7js3)l=4xIp8dFcnutWU| zg}eW=2=5L8U%=nL-aG&G6CC2h|Gskn{|!?DtNq_n1po6N5~l_j0hkhAm$IkC-&t!t zK$8JZ0&5w_=J1eyw7l-W8iNE_*anI4*XWS0IoNA`!1&es;7q{#WB-D|dO7DIgzqmM z>{Eg7*vBNtuhv2ycK8Y$lYwlFYu_pfKZDc>r3?y-AYehupsNG8dNv^35lM|$gFscT$J?_fka~`cWh+pS_CGlD4H_%P;Wu>o z=-2MS!O4shJ++S*Of`Lcp))d%+i-n;+%RiAsjD~WChn(5KZa@YXt zRzRkrWYhQF^lHfIRc#o9(mUr~u@8`}1I7&i#nyL~d2TpeFBy?P$Y9z3{nmw~@`l+a zII(LLZ13`E-4kHFSA~$}`Ek%*6%q`>mQig#ZTk;Et$hoiOE<$X;Mp?;=*U7A62qTx z^a9FtG)ImQz?ljlSzCZp4mjPS*Ml&U7{$fJM0Nm6P~bA@NUtat4TA)%{QU(=r+HB7J{2KQH5-h4LL_}!dL--KnW53ej|0lYqlN2ETrSU{W{Cr+f*6?>S3!vKL5uxUfrA3204oH%WN*x9I4AkSDpd)gcl z0f03jazZ?BZ|~5^NIw7*+l5~(ot;Ihw|+8{Gl{_{2oUrNY}%F~fWk4l742X!xn!K1 zh15_mPk?=cn3A#yB3B6TJYO>ch4%FN6FiRfyRw4y&I8sVSgR+Vorsq=G^7VK=G1b! zWE@&A9Hs$8F#@3kLz|T%e!U3Dgv0GPlbFgEi-(sXURy!rMGm@!9e^dzz>y0c#dz#B zv#KYb<_-T!+ZpSlJFZ%3kbVnaFMSJj3Y2x>sQ5TSpZoSZ~Du*Wg;?d0AS=uol(R1y!I5;PosVf|#4mwKu zJ_X8Am-HMQ-s8)c!;bSj^%3Su#5lM@NKi*y!r}*6 zBjEaDw{Hklt%l*tZ$tfba|~SAav9KKaY&ED@4KUcYW!^Lhs+ok)y2SeW$9%;G%{ zz~9S>6NKq zQW^lo=nw|w{N03kKnS%K!UO%kNXW_YF^_f@OJJDGZ{y3L5Mjn)sk3Dp*%b{bK?pIt z1iUu}Q)RYfumz?R91Ku7zY0Iv7prRZ2?C4t!Qi9)mscmj8@+C4)C=%qe$iLM^9X;!wNqn3#GT+j+h0H36F|eWUpJxCo2-N=9L4|}^@euqL)NE`ZUgt_c zj|~B+hiG_rS~agl%omEyQMR052s`Iwjv_xTKyjIUS*WVvzzEYtJDQh{2sHl=hd6!h z!8ccQ_vq}Bw{V|P!j_6f^XH*H#~@dY_D;kF9@u;~a-(2xK(+^{awvf$L+u>`FoWy1 zOdXUc$&hlQB%L`CoB)vc2wopKBdjvL98pGc3>NgVgf9ue;U)2b1BgL;CpjsJnx5Wz z>A+A8B;UAVS1A^_MrYXDr}|TkvS?UYe-#QzN)Fr(st4MS6I^8`Iei(i^HrBre~YrX zLoXAu!XZu=T1glI$UKZN(`J)#r-#RO^5kc`W9B=_y%XCz;Dv;iq~*<>Q0~vHrQAtF z^-$BiWP?+7m{4C0aO%jD%@^@Ss_YlaX+XtKL0GE+)QnuiNswAL;m>d|OhJTbm>N)u zXtPfO#%TsF2U)I42#OJg*=+-3x#t0{A90%%I+NmzO{+XOuAp5Y07+041h$_FW7H10<`{sXy?@aIsYf zcmv+o;B`Bn+JIc@NlOaogoF z#JNPnQ~CF%wL_YDH0s`)R$yvQ<9!f9$zG3Ma6JX{~V=?slYlaW^)YQgZmg76X0S)~x`FqoLLcj`cxMKBQcR{>dw z4oU&@sS`vrJPc}#1LR_X&MZ<_uYpv4_;CmdGtleP=7=Wje?k}tb$d0y^6vyH86D#T zg_J+6O60METr*7P-~zpvkS`v>*E(xXCo zdgt>&89;&6Y{wiTFBF^&00|d7GeMUhhnvZW*kKj9jt}J7G8_|!YDm;8R}D?;J1xUs z-Wh@)4#5p%O@iB)y1xsBN*&NFm;qe-ON-q_w>ohTw);N{L)!y71EJ*GeP+W`N04C@ zCaqNh#~{0b-p3U3iXqc3K=KK=L)3JDz-CB46%Zp(bfQUnsM!;|AJFXi)FA071|4z% z5*ZLA*HU-5l5EqPTfad6Z3IB)3%KkuyUX=q*92dKY2;e!pVz0a^5TRBv&sD`Jb*p; z3bHKMNqW+gqN4iIqBowu3JSi z^w0-DRCVnN+dM3~P+pyQ0RkeTi&AmgOfX5xWo1$&O=@Rt{>0jRe|xnj^dFF5VXB?z zPBq9FC)wF+7WZx1<4>3EAKihY&bok(Iz7AQeo|OimkBU)Aq;sW!L9=DeR=!L4xl6+ z1?m*g{mNJ_YHLRVFO~uwD>~1gJ$os@a4~hw?Kkiw7l5&PX?QAv{`heL_)(pqJWC&g zH($1sZvd(jYHVt{a4O=$+_#F7LfDEW3A3oc87<*-RIT^*GT#l5-RG*Hv#rLJT(DwJV>ld1ujLXzD1X>A|Xt8;_E)T3G+h$1|2 zysmvajyPw=b9cE>O$D9m6?GpNUJw8+g*<;yJlKta-_Cu+rCSU2F(auC@azQAe&Nq; zE4IYf0S#CI=MTbIk*AEKBOosy2p^MxehP#ll$4A7?Z8~RRSW-&T+FD?(!16I*%k9Z zs)Gjn%<33bR6VbK2iU7U48M-f1R*E%(U~XyVl9d=Df0P(SJd+W%6*^-d|}!kH4u`@ z5X?ut4m>}We5^inWwAUMsl{;Uu}nn!Lf;G8H6UD1!Y|@tkUmZ&vr3KVdoWL4hx5)t zTdEGinORvW-)I_LZbLTaB&lW4=};L7q~{4Jl$HT1Pa(H3aK)f`Jj%3!9L@;pil+zQ z^iAkRUtaCj;R07OM%ZCQQu<)`4~RTfprHQTA9gx)OuF6|G|E6c&-FmNl5;> z7T!Sx8Tf0w477z(DNz3%0HiDnjey{5g39&@-|xMAiG?ND-+ER*pvgu-Q4ZP`QGGF2 zaSXlm-uv6Dx5aLErBugsTI7F(1_I?E(;Is*kbO@+w;wNWFOPyyDd!rb;YrwmUH34ub*XR@t<$|V7~XaNQi zn=E`1T5E628$Lx|(vEa%V%?L}iIGZz&qG`6VHPsYT|I2}YWs&hIHMH~(Pd5lNpFKL zpxh{w=$aIjF^@bip10zi;l|7M1|hve+YV@av=7*6dYLuu9;cZ(PAztsn)nqr2^RB) zI3N{A3I|tN0n0aJ^Cl|C=v7o}-UZ&M(+BONZp|%v{qqNJ-YhQm$eccRLWYAQLQIj5 z#Fv%^U-@~G*h-SZL=)*EkQm+PX#th!XbTOuyc%O0 zi~BY>+Wza@R;77v+pn+MTVEFzezq30PyfkV)1Y~>^6q1$X&;Ou>TS`p62VTFrei-f zF;s+TPho_gR! z6$LB)xXoz&unS%w36ONNA!E?GgIm)eN|1e`U|l3J@3#HB0qHr>=&c(nPm=hu&CS1? z#i?nRUZy9wTPOAX(|Xv?OU=|0#RJj~UW^_dBe$HQ*jq?bK4}^CG;4Mk&^O4uzQL1> zHKuqpHQN6tI$@gn2>rQWC3b-c=7eaL6TD5|nzg2D|6nzQEKKZ9L--1&gH%Pfofnhm1RpfMg7p*SI@-IFtu|2t z#UHYFQtzmJ`7ruug7d&>JV~xAvyQeoBlq5TQhV8_-R$hMQgKsB!F2@TOmFF;__fnW zO@_?8q90>%n-!=_)e~w0N{_f@uhaFclhcdk=6kG{j@l?2Q0yq_pK$6R!P3lnkeqss zYp(9w%;tB3I{4y+J~C;WpxpZ1XA}dyKb}i*!Ykx%jD2}M)*viGuH^&nNzt5}10fRk zU5>8p+Yb&W{HYA7%s+pdiKxL><%z%0C*`GJRp$9-KBo?{M}-o}+T)$mq~oQZgVOUE zIsYtokmYZjo^iD*{z@9B9Y7jkzjr)P3a9*(Mnt4F!G36fg|W9;bN%&>OKk(_NqGmb zYsh;^jQ{yC&A{K$(Yj6ju8zi;ju6lO7#Yb^c%S$$DQQVAembfhO<7T|7wPIX`)YQ; zBadu7nu*vOLy2k6*Efj28FDU<)$=$J@o|3BALl5K=bpStkaQzQy8Q`J&B`OgN<8W- zl=8~Nh6L5g$AYJfL!aqUr6tuIrJq>$pP&%-pTM3V#zc)5`b3x?EslHfTIY!npXh_P zr_S0ct6fpm#9E6_&ahC97lo8tbdW@EUEc`Xmi}Ej#-dr_Nq|j{ba`>tWqV&%r^$Kg z*ZmuV1=b4sD&yw_VkZOnvwoiAUUdkJxWdM|eEsKUc?$b6Pw#-+JtR?Q-m=>G2o4=| zO;K^t)7(_KqQEEcww(pr!sb29rn(WzCR6`7id^w~L0x^~v}G4rhgxi8Skic@M$j#n zywGTqEDkGUE%jI9RL^Hzi7fia@)%t^NN`sbHCqMOrZ4Kzm+o55JfyTq4nCd)Pf?{~ zaPZmJuiC%~&c3C_k>eL)P+2FkjiYnwu`yFk!?PEsk+76|=6m?Sao~ZgUyLK5RC9~w zZ23UWVXtIhfcG$dL@UB~+PLg}TS`bIcg3FNrXr^`Pn8s-tAVOJwJm{hujywTrznc- zqgE;A*4h15wkyte?@>f*JAQTBRT^b4W@YqEmpVp=uTr!zwJ%GxU?W4JrHn!&8%PyB=lm#@ZENrglFMOa~)HpXzlGJ_#E61JGe*}w5(4^ia!M2qz zFO$pttjrpZgT^MA(kPkr5AWF;+aAjfI^S%*TX`}1q;EL%ypu4ho*&dRNeeRY4c8tq zbUk`~@;R=fu5aoWd~FRQYFZxk54dQ*zgL!(_%6*8kpCH|qCA^6rqXMIJz$`9msAs~ z$J6|7fSFm#t{k}a<<4NV`%hkS0-OXQK7${Cbnq1d??bQ+;G}*?N%;bY@&(7Klnn2L zxh4Tu@BE~40`UovO!c>ARFoUY7g1Ox$Z{#NSP5Sl^DYM&X3h%mbovs))YDx_if) zB#c+K!rx5}th@J^K+q5m)T-EF|4hm&alT{p)>ApYCsSB?oz8j6#Lt$B$=~p= z=#5ipMD^JFx23CnJde2$VYrvieD0U~y zR`pyza#>_lo-r(`#!}nUzKHL+euUt?5;^j!;B#0tyk7U?BuTc^Dn@1t8~M{l=R3yn zH>G8#yldR)D5+FOe7R|9VbLp}LhpmN)}1rb>-_QN{Lw%H!GyNPh=3Y0;qRMTZyTPy zCbhnzdbxzvbuCC2!vK;xqfUYZ>xa6L%Y;w-&6;EH&B-M2X1sakPot;*iaN}AFt3Hp zujQzs?kj3yje%7S$HHra=|?TZ=PX4?qaVN4B&T>L$}6=9+g5m%V4w}2Y>@LX`9)De z83l4T@6A#>$y54^^RD$N{VH|l)LOAd-;!((RD!2E=6_q+UZhagP=(h?9ho=(NjT6Q z$5NxI$+W7B_*`wA^&s{v&$#1*yP5YFKG*Oi{R?m{=lCx-1v_Xu4dp+~q!VYrqg?k8sYqtMAI@5vhwm6l z(xdH9qM%1ePatl@HL@@4ZTXXQw0p5>|FMwS<~a3z*D~Q5Ey3P{#vd&f9V^7oMBV-I z^3Kk_ogU<>v2I4kL9XOu9fS^2RsK8hlQgv!OQxPgO=_YsTZ@6L*kdjjHTl zj1VbuO5@O+Y3i*@oYJ!sXISLxB9o4`ZM~Jp+-c%S(5^riY;aAd*1}jZwR?|z4C8zd3$K}&ychbZf>_n`S6Xh+|VAj{lEG3TV+KG>sohIUsRuud~+7L?Y zWlemZk_rz)_jJ#4Vean4T8RFMYRB`!0coS(5;o4BXS#D`$0_sCA06h&c!?n`$>J(X zA;*xXTBl=118RJX@4%^NZX^7L_Nr4pSD^@z+a5crEj z?l=RUJ+G%BqqNw#ig0Uj>S@QNV>Xi>2`yBRCwfzS%*Jb-uNC(C_NKIK5gk1Z;a=fH z%_5cB*Oh_NUbd@Wcrd+>*ida|A3-M0Jd-gwE+>K2+;eH4a_LWqsZd zDj8jIWq&dG=`|-8jgtBV`|AnS*mu<0bWL<%slo)3Q=0-qDy}`W?S9;2Op(X57By<( zqvT^I$zroGd%xDEZz_o88}W;5a~qH3BLzeA;}ii87n21Gd#@x>woy@09C0eV#}y+l zdGFetp~a0dJZD@EjQzl15XJea@D?`rJ4fw#Wu%`tbxBDEuklh`yenq83oCBrPxDy2 zq+phw|3TSLWyZB~NfF%_0{olJS3Mwn2txR9{<*ud+2c$%PL;oCe_j~7u|WZe!CQjw z`XfBMWwuD`R;J5DRl&`Eao-+#jbUVEMFW{$;Mf_ly_iZ9pOQ~FajKGK=8uJb0iHi$ zrgPpR7o} z1M-H>d{;BLv+!r?w?B?==d`L=uM9N}SCA|Y3klE4^Q3Y41R@1JRkdV_<3ux@wQ3kT zHh%-+C0OnZ$&?kGoD0T$<|hKYsp|z|xv95U3(iORghVii)aU@12X)mB`NSZl8DWcu560R4H6Wg9V-^u`@I1oar3+_=0ZapQ{Rzg#wX?RJBfE97uE%n1?Qt=AxI*1;y2S%D)j3sMjCdNCS zl=Li#Z-!~ra{h_J@tHs4Yb>VY=P28XY$RI=vtz&c$;dd6?2CITu9Py{tRwir{f$W-p+$i~#z-wk)sPmpX=!}ypV!)o zb~yy9G9i&BWccy*kid)&YB^TR@JiQ`o|Z1Y)WTp5Zk9}Rg}s805LpfZh$L$9uD zwylFqx_Ducs2OMJaTk%7dc*B|&B-Ub=0CI_kyp2-Y}~72)nvQ;;=s*@88gnG*6wR& z;k}-_za~phAZc^$^!qE~XNG>&{2-gi3Gv9g^_;HA=%?9*X{Pu-6CZi?hWCO;N1h1V z3J^M3T(3S`c@we?(N5JlwtDv%k-MWe4m6Mc@jOrK)f`ALvs6V>Gi~{!@k6U}t>3$@ z*925l7@s|X-IP1HWh57C^t?uClWCSxtm~bph*M$Y{y^Mc3#gr{Mu=GTZ`|Vd--?nU` zb`I^zL6+Hbw?yWIt9qY|#&d%c5$nwEoD zdUU^($RFn5SsY*lR9UilWr}~X$;b6ZeQ37iBzvvZ^kG(O#+5*SeKEV_mGiOCiPY>(!(%^$)ZZ zFKKv=jV2QNUrFeq_(}B)fPqi9rFUCOi~DHiWa0^G)tp|8hLMwXhrd*eNHn?h42f#X z<4gIuIiucy%{%T%X~K<+qtc&jVv4IiO1zr#-ifoy7jLH(N_NbXLT%V-a`VYBo1a26 z?&ClyhK8wi|Ga%4rIV)Kv!$yI84sQ3CvJUGG_jZA|yaCD`$h(Zvh|O0BB7 z^W7I4(!D8uy`_3Tw&WuH*N+GD3xT=e>c)>@&4|)a;#8T+^Zl-WmducDCy-n3_>h`$ zflI>jndGVVtJiy!rca9uF`j%G>V61Lt!VFT6=;v^S!pb(i!i%%Vkoaf7QDZz)n>#J zf=i|8)bzTccQ8=zWnSu6TvKlRl#C{FFLmXhMl6I>2%mY;tGBXB;9dDb;kNz@58Ee2 zrmW$oaH8RC%uIyp;8k5N8S!DI(UeIZuxh0O31>aHEOt; z7+EERJ18RixG)rL*hZ86#UVRPKl{AajPsHBC`|;_tjJORWhYM3gk;0cAHchV8}Quc z)D*LQ=n-HUqCTX&nl3Hvno%|sc{L>|HW0*kdCc#2j(F%19wvFbJbi~`Gx@kS#w$Y0 zF+vFe8yjo&Y_!g`T^e~QBwrXWJ3bv|Q_SgQq&Rl_y+}RHV2HtVp~AQu@yiBBai_Fe z2j!|ZeC*SAyFXh_a0-0snBSbq-}+dxfx!qGOObfEj}ZIvM6KeM2H?5u)GEw8WIfZ| z?>!~CASvcs5kJbrGVIfzK=@?3W14b2?O^1%tL5YUhK%%lCfN0@F&)@EZl5V*>d$IK zM4d*CPrEIj^D5%ia5*kC@vE%A5t}9OG zq9!j?oR2-b@awKZ?o_AI&1Zst2B%KIEx2!%67pZGb`o2=a$;H1Qk9Hcik0HNf!N{} zgye>wRePJOb(CUKnw@UB2R-(y<+!j)<%Nht#aS)|R$}304Gd4ICdZxTT0YLS@J?Oy zgZB&VC~r49W~Y7wP2b&4o)?xqWaE-Q&t9S@7^Wnzv+1(B9?tSSsfI1g;wb0-#tY5C>y4y?@X;jii%W>4~cIQ@k2ix zkHt?835cM~r2e}sufIwreR#?Q$MNpHIqq3ND5KZ_sTht^f z9Mxxnr(qO!q4rX?sto+Cn1pYx>sx;vlY6Y9F>`OLf#W$%!b?<{fRtLU42t%LB6WZ_o7bhmtdTBg;z(e zdHl@p+=5`vn$W^}v+A@{FFZHc zDA(*Pe9T4o28oudlqLj>&NZz&&g=r8XYLW`I-aRZKMJfvHZKc)d`RTOtR$P9Mhw*X z@lzIEWTmqO&5b!9o33MP&NN%XLp~M1Uf~1C5lU;I(ex+{ywjPiNoJOCDzjNItwxao z0)_(|c3IG2mPB0Y|0KG6p#@!b1EZg^BH|_4RgcX27PvU5RGgE0oAcFHIVWPxA$DW| z(=6iL3DvR@+r71n)UQK}S%Az6C594@8e6p0%2NF=%KkE{%C_qQhIde#mQ5**beD7s zBHakm-O}Bif^pQux>$&gmegC~<>@oNQoAZeCIM$kL&bfk3 zSnx~B`$)f4+E^)ME8!OA7zL&f#ZV)_W%iZioB;Y*?h=55!BsQ;A?pM1fA}k?5*~j7 zz`QiuyIu{RpXmT9S+L61TZv&5^qh%INr*;AqG|{yLLlaS0%adMInmhikHMHTZ$c## z9&>#=sTu(j+5UYhx@UM&U&`7X7%#(t?}FTEym}lD5?Bi&+}^2`%#jWY+8k4}mEES2 z#Wlst<)g!WYcFo=a|R0)A67&IeWB8Nrij}_86I@kqOHdFI1@qt&IIsqaxxsCQ!-;n z{ts+^U2uCey!o^HhHEkn9X_5ETWIhusYgq?5wPx@D3t$<=5at1xY%Hk2(XSZm=^0t z3c0E~KX#e5QT%}7>p#(zJ>))mDI}->^aA^L0)Ri6c!VhdP1&QVPzt%++4JzA{g=ZD zmOS7QAUyu`!v>0pBA)vA1>txYg~8MMO-{fdlmC6|9`iD+NB0cCe!m9bfyv<%Hz&|; zP~uB#HP26J%@(aJ&Ij@3pt}2mfF80(?C_{I7%gx*&W`cJx{WDUJ}sy`4i(95gbw)m z_$hES%YcRE<5HrDpYXdI!kTfwI1hxL4||tef+Axxz-bFnI%^L*lee?8C?3}m;t)oh zcfUZJdZA@O5jh^zut03o22)HOj12?6OX}Y@16UO)V+QoGX=EhGu)vIP9L)Lo`(=#V z9jcY907zo`Z8I@SGj^>mlidpio=RJgf>z!VOd$)4=Ns5Z~A@gd~=wr)Lwz z%+f956qtg?f&v;)X%>BS63%v?v%Z{PzSr7X{m&Kv{@)q*81%9ggsNp?#}T)CZvJ%U zQ26jdjO!wI@R&Pra>Ts2yG@+m}&O^>fm1i?hoM&;eU5QWY~Q^vR=&9AGh* zPjvrh*?_&b-7kWKiB;W?rGzjcX|l&Tg9dcq%v6i=er^C??rI_?Qf8s*{Mgk3Gps4; z4AF?V3!Q-Oz<2>#O{$tr4rY6lnVrRmKq7KfBr*b2 zLh~uJMBl2lMQzLA&w9Z8a$`l!o)YhxFXLF)7Rw}Yn@ZC7uBW_tH1B+5g+F#+GmRGs zKN%5Xf0qLw@sD~EeSQ6OJ_qtgz0IQ^5f}8|eHS?%J&kpGQQKdC7TD%))uaqL4zl(I2?Oj0d{mXLM_EAWHX==!c%2Du!K_^S8xjppf$rrm!*Rq=EZ zk4JC<`}Zv&e}<72II`VHcbs0wvQFiMlJ<{0C6J8{P{IwP4i5_P(HROkA4nm`N&uki zVfzDjFuYXLn@50~uDz|*fF0nGM11e?l~1n-_ivj8FLkaLd94E(hNifOMCT$nW(l{}6#T*r9CA;b)R8FX%Af zWKe%2Ph6`=J?=wc4&!Gne%mJJ*CQ9t3szgir@_3r{m6E{8H=F2&(L%Nr>9D}P-4Ym z3U1C?sgI#gB8W!0pXek^vcJT=%4<%gEw5dpM1QI9q~HZQF`Rl8unem4KHL6U^>Ccl z(eq+jFNC?O6%XVHFv(T`{zQ-8#bU&P-+_;p13NJ$H8JL^+!nn`q8OH;59`ObrIbTl z_|lB?*RTT8F)#VpSAT(on<;I+vDps=R%?N6epTz=Q`=oa}-n1ajFF?8U4HOXNE7T3^@uKrTK5&c0&uK(*5(Sq>+UvM=;?dXx%7C_pt_?IUrBRa)~R2ee9E00|KN z?dGHXE+9c-wf1&!V21>WcbfhPWf>wSCl4=RSShCfbIRqNq!PdX&*Mi6I4r*N$ap0k z%h*NrD_%JjoCLG7V{}-4k&Ehi3;uPWOrX!gyRtA8NRUgO;jEPOltUNP5ufsYJ<(!e z3@N)cbI}Yo(}FS69u<|H>-(Kl!i0F{n6Txk;_#CuS6XKI60V=)FAH9m38D|2Nhilb z+D+TurE*pACHwRDsCK457?xF4;Uv=JHwoN}oo#Gb_yS(%sQubW)b+b5B{&h^`5bB#tVWZz3TRj;f!_?w__Cndu3Xiw$QAPjPtF?^8qSJ}gEr&{2 zHo~8;aByQKtu&s5Yb5E>xsxnQN{3EgR&FEQNaQ%BW`&o)XKFxWXJ;2miS~DYKQ3!& zaq)d>D)oI5{lDjSO>9AR-UpQrFsUDFgX-8qV;Hx4IEZ=n{*s?UL8JOm7QymX^B+wr zx}#E(@g!yC?21owmk@)oo9w2>90zT{gX|q}%b_yBkW2^~$G?)QMe=2oTJIbltM=vZG(eZ#He23-wV#m67tw?Az%jE8Y1wLr)Lh4?epWTT#hdeaqE^dEB6g{n(54EoJFljox zF;VJTV#_o4U6-D>k=TrxJF&|N)@kj0$@Gj%oTc?>k_9L!-uS{EF8Aiy?nNItDimR4sA({z zg(Q$qZ_Jn-3WB3|=hx6*H->{QI9h-L)7E$_nXf^rAWk*Ddx5vgL=j2E%UvH*Q04jd zl>|#yIxcM4Ym*ERYI9xdVG%Ul9uNJ8$_nUDUjb*&V}}D*H#g%)(JCM~PV7?nH-`FM zX6gAse+_2VsuY!fOI^uuGG8xHRLogU(0N zIU*lD1)NDPPCBVBKzAW7TRQRTQDI-!n*MJn!L{nti&WP9Dz#C!;_QAoHz+M9y0~kO z6ycQyrO(xD=SWsln@0>@-?k^XO*xUm)#|}?FYh=z50l6cOCLFFDDja*m-^l*i*+Hj znG!)u(DTfN1`>@zyfzoVLtd|4mlH-JbeyvIKv2crVIL&`Y1d{cZ(cp3E5^+M&P^L}=d;Jj2nVAxi=L5#65H%CiKGrnceeIvAVAtzpytHNVntJ z^@rUiLD>z@jDC6{29@9RA2AT9NukN(H-44O%X~L1ogEscN5Ui*{F?ZC0NL}f<^#DL zlfT*oi9iyAhw1p;cl-5Oj@ax`8nrkYPiu6e4LaRrP)(vI`CugY8qzX-{&H zOGr+2^rBQkF>S{K?_uXWV)f20XBF%|6#8>SFUi0|fk$5a5EJ@v3qL&VMnqXcN^?h- zPR8SyE*W3rTJvf`q=Hr!%TX*}X}WBNA7a)9BO~QO~AM;P5zhv(|#d z$9ycrPvzS9Uy2r|CH3U5Xb;;taL1TsS49HT{(}VAJt(5S?DsEXu=r0TS7I^xv~XIy z>4n^ZouIAT_vYe$%aw)LqE9cCa&a2jxBAKD3K@cXcreYDBv6oEW%+xTGCpX;LMsuI zBY}n3$YeDEDt_>U@^1tt_Jr+2c>4~>$M5a|%q|RO5sYfPC=?jb+M(1i3sjenuBSFO zX{zZ$s_~V4u0fQqG`iot+SMfi#Ew1Zew6~iecceybiRvZV&3_R`zAIeI=P?sBZIC5 zy|H1>cp1T+ANj8={4#RB7ph$Uz+J$yL>4~Lfp}wc*2q^`Mzvj`k%&YDy~&AcM2Yxq zn8i!N@AU#6U7;h)j2~yZ)vb@J^IdAxIIp-(@S|b5m7TF4)tj88>oZ{MgWRn2BuV=d z-z!?sCAw1}LBQgrG7s72R?qk$_e}wy`DJtJh&fcJK0P|Z5d)s<39b6BRF-~-yT+4< zX|(OR!;ikwUDdB>{#-@ACaw@o;5hQcZ?c~Fvw z#35u#)#zsenhw1J&M#1WmdF>EzPOlqf-HKy-aBsQs0Kd-L&$$qbpE*y=5%&plDl11 z<*eT@OgS;_{NbKg zTh3R0!?dl_!HCp>Y4cJyyo%gz6^;(KSS07nHC0)?5Ei{x2|DpVg2&pwaZv^dVA#hE zFUQ<}-z=JXid_75*~(#$dYn?Ri7APJ(SxG4k zn$9&PT7O$YHl!DVp=@GaAp;jA6Y`g#E^fG6tTglCfgf`Ikg6K>HTdgV8DKswaVY9B zEHcfBe0l5)C>TOJsn-~GL))q*@% zl%qF(rrIF)^{r$GvwZF}Q^fz4B$N2BG0_NVV9Br3t{eu4F7=nry@S3Ei<%no*rbtX zZL+<^z@gkA{VWV zy6k2llAT7B>ym1V?$xlyH8!*_bAm&I1o{WEeLN^YEsJrFU4sA-fk73~qm3FekzQbUI3}%? zDb@up2R>tn4Nz_IuK3dVCI`|`?1;rC^gacjV(&|*!pv?UzJ&w|v`t}Gc6zevEes`{ zzt^Q4`Hw~!wHCt%FVTO;^f?B;;qd;0>j`(y1D%z=KVa#@oG) zg6Q*7vEk$^xh96*79S`&fl*nnR;)N=kC{^_umEHD`qVUCD&%r)JCs+VEA| zK45Rmyo_H9G$pFl$=joULL<=L;+{cd@KKa zOm52?ufz%O7?~ed$RBX`3mAHpBcqX7~A&@Rs_0|hQhMK#@%NW7BlqMAAEBq!DjNXgStaClDOa)@-p#^M_PMayzK<-1Cq4od?=7>&;U(Ez ze>;1hBE!DW8%4s4My{C95Wj@0m*N-3MqN{!}l6oU;&aTXxdO%W+uxokADI; zYF6M~dF+I3H{V~}QxmXB!>!)|4z@b6S)x(rj0Kw=$-q0V#ue&0I48ClRXSgMOG5^Lby%a zP-K!$K0jOKN6bQ8$)ZUgx+^sr=Hw5hINhDH<6oCe%1DtGH#QEhs<_&$#}p8Qkc+<|`Kr-~7=~`O0db;iE!u<+;evpM4f7V*Ku! zK=__FeiOfwAlXZ(HC|?2tO&kY?vV4HO1>FIS7g4wg`Yv5Me;3&df0!VV2x}*JZO5h z{nOg;VVyl73OH9$fFp<&^w^T46X>HB1*pJqmT#7T>LQ@@9c+sNJ`i{<#SV7@*=t%g3`O{hh933s~ zEj`9wv>epz&%3-noQ|CYz*g;SI}!$h)fKMyocKuY4wm1D$RkW@z%fd}wYxk<$HTh^ zVKjX)3W@kabcw?TaB0VjFVKr$F*R2xK0*ByfycLE`H$D|l*`&p039B7Zz!!S~3BQV7z#8iS~T`E;)prJQ&AxSaWq+bfp<>AYRqEJ&tsBgHr`JQ{x@Ii`T z6$^d54F1}5p_j=14X=}r({GbZCwDTroP)#1Xr@m&?m?B3G*0-9GaMJ*|A9ZXQ}ldE zd(SRYu9fLTGMtc4^)jkL;7i!h`Af8AHZ&VG%zzX_~OUAFI<{ zxxtWlO~CU%&7}VquSjXa(Ze3y4jw)4#;t`%O#!9)qnU)#4gjnG-E)H$-}N6Jay&Kz z^ga7zs3`RC|Ag$K_7m`cWebmp0B9hWzH+?kl&5ap*K?B`?*dSRN8uN5%lw#bW0Zpe z&J)wo#wOLL&Lepoz5j+^gS>DD$2f`-6Am_(BQ{Hd4gDBTeLO&|pk5;+nx}eWIN$MH{q)DqJYEv-hPB&@@v9yvg;W>gL;vnxF{xZx&jQ6*qo(!2jr0DWoW-kK#1|O5-4zH z2J*15W;}a|hjf9{BMMDQEu|mSw1|={M|n{4^?u@at^c_ zW(~`vAO{^dN$180y1e4dcppTDs_HO6ELFI(g9>7}r=$_M_uYiF|Mg)&mHxi!nV@pl zP9Rxx@%A^C(}7U3q2k;nbLt({oN%kDBItNJh}%6ZdQN4w^GS` zX;6?EJu_&0ShP@5?8Y~GjZX6K0V)xu)J4KYb5fc-fB3CRfUD?u_G%ql^jLX+7`OQN z4p@Tdn<5AoYjOOlw(Z-~grWk{>1|>&)gV$9L>S4{lE!plddPljkTm@GhX$=3SL3C0(ok+8lERCu2X}|{^ z$ZQj~93wBRYB{D?20rDJh{4=JP5Mi4=}DE*YfTC8sj#i1p^{XB05L(>S^rS+?VHq~ zK$2==9cx!|+U{DGLiQDzk9a5!JYYq|St2H6MMcCP>zd6wkxKko8_O@RLBxCsejDFf zVeh)VfsEX-7E|AlXq$XHlxv5dAdA=L+#bUX7D1k|!LV>F`eCRM^>2KGKjkupqffCOfcz7*D z`-SOvQQVeX8S82>FQ3b{S?Jz+E_UzMUq`0f)I;AZW=g+U{syMVlNxM)e?7y+l%4F$ zGPPI1PZAW!?4OwfRd@m72~PQeX$JRvxr84z26{#Us^aARRYE6@U898u5|Vk(NR!iJ zTDE3#Vt(BkZA?kJrCt%Leb5G*oH>WK&-g0Qlhj@Fr)jK6T{nX}P_3S!rei4F{A{N` zJKL2>T@Cg*hg!*&aGv8 z?lVv!Ohz-840e~%=+xgW@BA;8%b}G|{i7F$2?7M>uS}1_ut!g%TW+}k7VQ1}--EbQ z%Nssh>MHZ!u13A3ovflyIWkpxd%u_x#k@ab*xj+{3hTrLYCJEZA(&QrENwE{87{oz z$^=hcue;dBuwES+KWWYE)9o95{kDe#akOiqO8~t!Nab%cLvF#sfjE3Y&GAP%Y(&@x z5G~-eqhw#{tf&*!X;>0S8-LTEpY6SwGBv$98z)Td!vsxk2DaSGti;k0=1~6)%s2^} z9JL(yp?n*bavc$ZIC((2xE0F~&!;oK@#B>EZU+{<^U}gz^uwLGrK`SqsU^zzSNlUS zOAQ5R$&-9Rs0s+3z;TAKDc)MHzF%>_2j>K;*U)G>;-g7@6hHDU`h3PBU2VKv_Y^%5 zDEOGW_IuL}2kiZP=c0^ZYm;O*X1OF$_}AZy&+b0pfLmvf(0SCFVs^w@QfXLlpiY|O zaP4kOGkeqyL|d}07w_Tz5Z2H1(>ba?d~>p+|LrCIO;Njt(@pZ~-5^13#OH1iG;It8 zy6Lz95wyRUEN&viW_P4MEi)d$g@mtN-G9}b;{H@(jDU%6OJ$OhZ<2#MQGuM zZ~C>97RmI+meHO#>;E4$0j;3I`u~mHr=q{F=z5KJct4LAQU=d2MLDzruc!+lwA2s* zk&PTQb>RRyd(9iV_DV33o0eIh#$Qrb=(#hhMnE(Dy;16 zKoB+SY^~8zPXk)`YFXiqXWvY3KBdYHCWw015rO;Jklb5wfI4^TaAllFnV3Jt96fY; z36tkr%_cAEj{T;c@j;<&0%i zA}%<`CugIJ%wp^$sJ14;?+nhbz=2VdczsFLN^Hi7YEq7}Z0mJf6FEr`_2}1Ehd#5c zi`x@>GWk(In&@8H$_*6Wfli*(L@Op^N0`WV&$-dH({Ilp@E3*MCsvg_s0^0kv){_w^V({Ig4<_LA_oX(3#QM4=!}PE|fIbG3Og*qk zKFBm4+Na1AfB+{@)MF|9Icn5_Wdj)jro1$wpG^v!*PhY#AQCa#Yd1=z)t9EkpmC#> z8ABXTMJ*Q&DJFtNpz1I>oit9}ZS^=xRsWvZ(Fzh^dPzlBM4KuG;Lk)I;A~I2LnHcX z4|=%ZJ-IC7J3Ex~-7%It(1SldaWvW!n`MMc2Qjk*ib+>waJovDQ_a-IOCNyXF?%yH z-5ZxSK8%;HZ17P9ufnn_h8iw0Wo-}7V0IqGUYQN3roi}GT#hOV22wd7WKk|6yi?4B?^qX03kpBNin603?xf$%k$DYgF~ zkToG11R2%UkszcBmYxqY&J+rw!v~{YO3+Ci=~;X;=ACk$_htL#B7w>6qy&Z^t2H@I z(Kxqps$DZwi<*hD!zCupUT5(@8@I4m7N>*yAJU2^nKXggEglkXR(ZcW7>Ru(;lAKUlOkS`dPw<&GbCHL+GSMU6xO}00F8`kg_|7 z#Ly!6Nd6)E^N<->Nf3THMZ4>V*S#g3UzWrnAv^AmiY}@4)^W~jc2LdXIhowEBjpwY z&g|+Ose}*lxnlTR6lvR}qk=iZ>`8~HTu4Ts_V9KPFk4@Lu}}<<|)au*PIUMHDA8PD85rr7oZGJD%GTxZGZi_?(P3x z9{xuYC9N;9`i`ISsQ?cifkzinJ-|TX@7`YTqlfuprVW@;0>)|zYU#4&01xPY>E^Iv zue?$Kjtge{`N1Ex3R&*-6RQDqAgE84f|S`Bu`w11t~P!Td&dW_@@|>B0RUL3X~#p1&oG^3tM0V%~6!`*p$)V*Ai@Y5(sW89{aI$f5ZC@@~}w zcw7eWe5hBM81dr4JyMdIJ(|F6!u>==#B%qA-7WAEmHYsgjH2*XS_YRsV|#M=Natuh zSUXmcD13*)NYB9()cNC9VuRBBm#AkWg$&^O1+~dd+S=L^Dcu=Nx@2P6# z^_9UwvEd6!MZmxBSWmiTZ#02Q?d_@`U_Ss1lZUtvW(xpE-Y+6k-XBoGP*!PyJE$gOOUz2mGxyBgBx zH3(p=$aK*aWe|P-8~QlnBMtOy@OgJ8Hi z;`DpJu6V7c;F#gfQhiM7u7q_C7HX&_L>3Z@{PDc`xv|V>^(MR&I{gs&F+`P~in^2X z39S;VHDTz?SJ(Bvw?{Gw>VK7Wq*7hYd*;l{IkPE06g%S4CCYstG8Zntclh1-+h=RN zYlAN+K;%2QaxT<&KhrZUyNvP0o8>yG{cEt}-NVo=S<=pzKZ3c}-M@OQGdAUERxEQx zEN#tv1nfA7rdL+)bS%EBu(3roofQwi8qJ^jvdR`y`d5T;re6Tz*y+{*IPXucHz{T{mpWkd zv^8iW5C;oulSyw;+KaN{>CcwVAQfxMo9rJ{ktZ-HXh!=n@i}rUa$kqPDn^#Uc?E;nHiM>NdBVi&vr+4P-9q6e+!_$7l=M zqllQ{)C$9}pDjP1o3b+L+B_-+&o7(qp$oIoY55+jDg@59z_l;PRy&h zy0%0FwoyR$)F&|b=g6aKV`;#26i$i+)ZO`)HBqbIYm{Cxt-j7J$TeqMlRe|=hcbwx zez#o4*^hz&6>aZ;TP*ioWV(UU^p+H9*8=hv`doNjWz*0Zzukx$ z1={tDLobYnRcYE4OA+8MHgDZ{Scb$)FeLlsFg;?0WmZ?Pe^>^b$E^>%Mhv65W2Aw9 zX{!dh*!GomJ~o4W`2j$?whK>+WJuRtF*z!nj?SVnTsuHe#~9YyL@8f6tOs^Mwty1@ zvmN4RQ-1OUf#+276tf&EaolTU#`aAuG|fsswEj$?5e)^D%kFO|%}Eza+w50)p8i^U zI^8gr>*tRW!E3d%fi?B0P~Y3*a+O|)dbnxwrTDc|E)|F+|;YLLlhk-2tk))|w#p;@VnnKmQkufyg(%;>IKS;;hX&k0eOhgOED}@}`vJ7Y&#b=m zif!;jyJ!cR@&3z8fOxu^HS+U20cN!ef69qQ3wCa4W6tNO-CL07ACKXuf~M~P!zal9 zig-LUnsHy)%5e zKkc{928gaLlohoP4#4EMSk?CM!``a8gEIYjpoZ#VBmp+?h0L5|xBTP zRoy#6#?=g;s(o8elt&CP`{7}xw)~Rz$|;}PxY^m1pe3Xp`+Jyg-yW!(deNkA&eCH9?>^F_ua?z{~1igZ=7VHf}=gJ8( zn66v|jk!|JxtW)8SAM$H5~sk8r&Pk7V?u+S2x>5xZba-@==^qk8Opb;d3iku`oA~~ zDIvUQFd=InM7j<`Q8jxSA|hgCGdPy5pQ>G+$H1jp2z-EMXhbl*q4t{h$_>^j`S7n znTQVE{u*9#dAzNGi3I|)k4?mVb#Ii~)IE4&bknlyWGjRr+`O-~!RC9&XLnr}U@1BG zWpB4+T|JHC$>pUwKzpd(6Fs7I9Bv&p{Y-}^|Fc`d2tnXPh??a`h_+}?XMfFP12ST*X9s~uTBj;F#+378psqOOn7vm-QS&q>lB`|{S?Y5WcBlP z%-qU6D#^0of&vUWhYlnbVMd*=&RCF+77;p&0AnnSR9Tvu?CYPUrtv>p%XuDdr4Jd;*^fIM`Ztk^x*Wp(V=$7wYnnX@whobgr{K1?g!jcM9~LzE6S_ zC+i6Mqft#6W4dDkv~H@EQIG0_%Bsb#_Chn-CJ}^@gD;a`*#Fq<6=c$2ASDcm-pml! zj{YiL&|Fq-Ir1fFDN?1R2X*2*mkV)u8$*m@-N+v;mQdg_=xIWO_gO|ohKjQ1wI?&p zYR8P@@5pYR{UNl-i?*&<={Y$XsaK%R-)2*ohO=gsl}i&m@e+e{gpBU*A=CL%P;NVLn^D*skMtNWLln~cdU)#y04~Tz;Ew3`P}Q@$=w|l! zH8n$-JeMh()lT0U>ell1vod$@dP3}O%e`Am;O!9qm9@{6xu(aGyYGgw5N%?X2mL#v zT&T;pYo@=CnU(vIBte%$4;&tb1wMS^Sl*5fyGH}L(E;{9uYmiB0FHNll#X2Ey4-P3 zc=;=!)KE*FA&h}8JZZANE}xF@MA${@nc{G7=8(5q8((ko4WS&BQpcOzsHpgkD{%F> z`#FC)CEoreNZb9bYBw|Of8j8xdhW3yBy8;6f^Qll>;>13Q?(aO9ceJm=q|Zu;9qqr zZizs+rS~46(YdX_GLk;3&^XePs<=d*&(mf zSsQ7XY&IzI&=ZE-_HZDAB0?z6MRqy_qo>z1x3&1pLG^RmI$j*O6paOn=t=~G08ZH` z%c7D0jT!HL0ugHcxen0%SL&jqeuk8cMy1q@>L4v#xSadX+6Ijpz-DX$*#l5#nqika zA_^>X7fyD5x_l6MEr^5gXb!tRSnrOtA$Jw`@Xt$tpINu${b>XM-O~4VOfR4mtytOD zPmKl`YwkR6%vV;IIWK;t^65B6*S&DQ4ZHNydb$g3XzlIen!>U3ecE_6U2`$KX3?*V zl{xL`Vey6E*6ZY+EEdub+<7VL2! zp+M$3+Hq;N67~?YtILkDlF2LNr5AO2L8rVUkSNSbqK`V;1~?5>bcbA3$w(8l%OpyY zCIg|j$S*;=ko(K4>t=Kk^$dnFpn1waiG5Eb$JFS8I>^P?YKR@6hPLq%a;%VB5G5FV z+Yq*jJ4kt>Sk`4kwMv_z2LykpQzy#@?~&dAx@O29{6v}&r<_$xLuZtr(Fp6X0);=* zOZ@FD`1W0PuWWh@e8$AQEB*($n8;ij2dNN>cL*JibhN#pTV+@<=jvEGWqUIVYV9PfwPv4;&&e2++uXjABOz_*9r5Xaowu)6JN0_m0a zn}}o+WX~Pxm3*IcqxP=vWYhRj&=&kOC z-DLP%8gy}G;qMDk6~x}26psBrYD_Dl1rJ&W_u)g^d~Ji*C>0PYc8;`?>m0EgZD;2i zz#AjlM!YEA7#$DpWH|yIX~*Z^aVbx#ak$mAyAcutE$QW|IP01{J=(_!O1sze6`PP! z!?G7f63ZTU# za8S_aFM(pS(ph+=ZtzetR**Srlos9jNl6L^hDWGmqnp^&@5VkfIsiFg@V$1Gn5N!T zbm4aMAEX2ovbD;!z)|0XlpiOs{!ai4rit)!e>A9S8714pYkTeRHyly*E*fTA?yWz^ z6QuUwLnN=~zqiZ@ezn&GpbaeipuD2o5lvH3$Wf}t;CzpKhj{tW9e2^}0QGoa@N2pom(uUT zA^Z5P<_3qiPx)bGZ{pmW1@Hd|CxBWBz@XA!*b)Lhslvx37hW^StcVFxco@WF*R-xb zR5B1IQM~_L+Xk;&-dQ@;gsm&H`Sis4qmzv?w<&gT78A8KTyZjvmTO-_Npi2(S1y6$ zrDGHWU5r&&9vxXkFv|-fqj$n?^A)%f~XT&5|*(2?Ztucko`OXMyh{%a9{Y$V;2&l<&X zLZ^~_Y4~cl#C*;UP;oG;JdIUQ;US*6YfwbcQ$WaxE(0kB9ULr0FyXyx<^2#Mzc8}G zFnX{<8=|MhaIt5w4Clb~L{T&xmUh>+dvaC0ld^ z>+~au?r~}kW%MhfyzS#ewajO1e+$&*GSx)rKFoXKTm@C@+4?U(s@&aS8f4&xiEyqbdn~v+Ev4 zb^edc9atl&7f;Y3kBE3j@3ZCp7=A%c9c@q9`g1A;DS|z<;=&S1RTJvS=OHmFwBxX} z7-Gappa8Y2IkUhoIw%yj-dC}b@=nNx7$s4jO*5a3NR=DDeP8i5i>cxj^5Az2)dZF2 zkrk8MA9wpB_|wdSlYl(<9YanGyK2L9^*U*N!rrT2j8UAO?1Hb}S7%m4m=>gGGW2Yp zvS3{%*9B!@%=je88tHpsGmoRp_}V(4mW)6Uoa6g|US{Wk;t!DQQ6b;4glU=Inymoo zSDGHl{7A)#N%=@3J{%3a;$nz`q=HRHiw@vSmJnT`)+tThBipcqN zq6NOs3(h#Puj4>BeS=_ObEb_R^gG?@IDU(>n_im)zYLuZ`e!#4W!lXf0dW8=Y}sw+ zd-CDA8<^x*9saJAj5$++3ZD9fG}fiFi$FZ+|1a>Nj8>pbHbJJ*zW%o^T_>|(PNdb;G(pG5vZU(Q5$Yx! zVcQ^i76Hx4xcY*k57jKCPNH<=aQji2&t*k?puMCFw35XyZ!5X+Je&pzg)FjBPOUJj=P!qlIePWZ+^B(*d!?$|SGka-We6yE4KJGn# zUN11yzTj%v>UlfY5fp@P-M8hrmvOndr=DJY{hoq$)W^!<_T;X`lx;=rc!{K@nm9Lt zmeCd#5lE){J1IPdc6_y5v0RCT55;8trhH6LnE5-y6=U?jN`MLwXSS}O4b^Kr*Sp2Q z?mjuHths1rS(-?>{M6%(X9fi*ula3U;JxqZHpcLJZ}q(BUw`ME1m6WXhZ{dWy59Wc&xG+r*T_#nf zmWS;fn9Bg!!IVg!)Gg9Hd$PysGw$ zK=?Igz;gRPrhEHEcm4V)jNN!2A-XG|LBh($$^`tPP5-Jm+VGeHk$Vs;vV0|R&9e^3dTI`m>(=6JUve{R zj!2p0_B$6}iC(GPo&XT++j-N2J7(n?Q!Eew4rJPU!}SK(MZwueI_LZ;Nh)_=5~cPC$RT)Kv_)ysGZGvV}?%% zz`){`Mg`fk&q(AGlT;!V-$+YSNkCS3B$%Vt`^L}A60m@}J|rbielbh+u?iU@c7}Jq{%I2^>Qng)GmRq$>L9dT)B8sFQ&R*}J8Et3hf3yg6R zG#vF9rT2vN>^FFOX+cz3?^1ATct(+J#?U!9}Q~A z95yeCntH(zVNgQzN8^tgI%_c_->>t%p$*Tkv!+mQBQ| zOZx~bJ^{0g_YxDF{LyCbma0AL0&dC|s|>2fFBG4B^+^vqDjWJc>APtyEvesilMvUK zxPE#7KXBuI@~vK^O}q?G$?*eUec#|`_2^LRU(cfvog%wO*}XMe?4nP6^rq2R#JD|F z6VGiwr9I|9H0oxk(XKxpR>z`if(+rD-i@d8wrt-cwk?0>De@5=o|#<;RcJ zUzk5}d;(EG485U5GBK5<$D^aWfE)og)ZdgZ^!cfCD0EuLw~>PLm9*FQ)ib~05kcBz zGG7(6U78Fx&E|P&BaK@dlABfNjm^mBxb`$84SsW;VD?H;CWS?XM-`VvW4^Y*l_yu~ zDtlEX2%Bs*B!6Sp~2~Bpi^{~|D6D!4~Fjs zoU`)o{rS0@E&J#D{pHsUjokV30Yq9ycW6WI0-|JZwsYU(9ddhsYfbj)4Q8h=lYu6|G zi0XtMiaQOkZmB8t!_}MW(c{~-li84338jAw7SMuULp9@YrGozao>7-StZYbt_y4^2 zEl4NL1Y1eUdn!JyDp}Ch%vB55q(Ml_*~|-S2Q)Rn!eCifc?6qZB?(9AF+%6A>WinM zZc=}&W>gTF6nyBX(wTuF&8+h1>KG|9sVZnC`aHMMk0@V?Xfz1j_G`ZQgg{#G)vL<* zbyTlw@_Vxj4$zGVJC=Mi+BgI>Z&ONj;(MgJ@-N2{Ht>}s2@HVN8S%L#-v{*i^}>cB`}n#BdBe5Cq{vjz_bSOiABx`Q^ju1MzLxyDzR)g(( zYpL|gH$N4hc(Be~bUXNe@Sc6Rdx1ghT@7zMu{b9NJr)oH{4fr)bb~rv>Ed_#48im{ z1X;g02`=Naw43V&H~2EdGcZsudeSI6&ZrMd4c8kg^Po!nB3{?8{SVf`{{f>PW_X+_0;&gWiVZ?cc|zMJ|e+5?D6Sd3rP z5J7FumP_ElJp`zRmm7+mbp5e@&pJCTw8Ot5BxV^W%UW^baQ9$t;D3^Z$icsZ2KdQe z7a7$nc=yiQ<+k6FtrXY-{(p`enf)VDlnYAG)4 zXYDal$Ujz1|Ku6P(xvgV+t^rEB|GU1W#`nDp;>)Ey{AuuI=^!3Q)wd8B7d6mnj9{c z?rNg+x8xJsq4#jLk1qPk1(PZymj&qVq-m8|0ya7U0tBBMd+AT#4Abw4d|BXs>ySNP zyLHWIQGiGd9T?K+6mVN`*1GpT7!3?ESaF#Kac|U4FLWbFMr-=9ejsW!n?6q(nVDYz zB~2fXT5D(afBi+&<9g-4)lyu3N%k6*f_x*In83VUCHiPv%Gwz!6zJ4?9pua1? zkhy0m=rCg?0RrjNRgjj@$_s5RpRMt6ei+Ry)!qYAQ`@F0Ry^&%oJ6$etAo$ABApaU z{L%FJ%&-#tmtkr;4b1)AZ<*tMR!&kqNfZ#kpi|Az>{a0IwFx7}!B>}I+&4&4nW|gR zLkV4uyK7Z(Zc841uAk?m$Wl3$7cr6a^Y#I(O8<)81goouE5`7$%cbLC`vcMC!K{_L z`*q{?D-Ks`43H>h-e1G5J~1A=73_bo!3ZCEUX;BZG&wIy^#t4P6#H1U&t*lSRaukV z$$)A(rR5-`Xlm%C(o{Ll^WFIWDS`l97#=|plL`ysC0lℑv!2{+x3Eckg+h>d3x{ zOqLAYFj5k?W-?`MFscJ50gX<$7k*DW8QhcE9t`8o&A`>x_Y0wJ)aMJl`ZzPhaLP5s zk$MrGmNj^K})GJmd9{D3TdqnL0qkqi1>GKCzKKdb2%8ZHVP@90T|{N2K! zlUZyumP3xtV#9i(Ix%|NHezhKLBLq;1#lr@Rkm*pF452MY1H>$<{Kv*#5ox%%K`6Qm_osC#zKYr-BkMYJ`Y z=dpcYGKCu=XfUxL;RF3t2%u~CZG?ifz5-X0oi| zI{El)ZtCPjcyFk2<4(A~vx+NC+PdV)gs#%rV9Gys13}#s^j8D|&WSnTgS?|clfWtL zi7VpF&0zhOs^rrOnm974EbWF`XU$oea>>Zx30UwLPf4aC^gd>2h5O^{ua%FlCTQ*P zl5BLdeT_yoX(igwNxI?x#i#=B+_rN%yl?qMjd5{G|JD^Pc*CF5i;{o)vB_EQ$HYXq zEev5h_g%l(!bkIWyOD#u!~CM2F1s35zyMzP#tf6lqWqhsgNKzPR*&bxNiSj^HbNJo;TRiebdK_Bu(v02{ zospv-I&9Pr?O2Z0Ej2jMR-+|do{U9HZqzCy1NI}|?YQ4hrXZR_oxRH6He z{SF9D%<$kq7a%qN|9VaeBN94>NNYD#P6!q8BR+6jV^7>GRjZa#thLA{X@`yz$!dIB(F+5Bl_&UT?P%_FVX;|2ugGagC>{a;N~DE-{%E% zM+zak{G72D3RMew_g!X&^&fG%3yqBZfn+4Ye^o+5px*xPXJ6&)EA2qJb#iGcPbZ|c zs^rPcRV8UeTM)v|OxUz(A|LT>#n&zAE-IK^#Zd?pNz$ViUCW0ETB;<8 z*|46!EVDsKThKsnmp(waeMbcc8sZ`Y$n^f!%z#$8hE_Zm2_yu0ZbOShl15LSTqUOB zO@Wh2vSTa7*OdDb-F?3o)J{cK{pw+mLPqsZi_hHhj;yX#;kk3C-bhS12*il!sPi7{ z60mL+p&<+)mwmCW>ib$0-Zf6Vw!lvNZ;b-BuBfe7Br3>pdQZV8$@uk>d`?mYYV-wu zeN8_a-6l9d4vC|?Dr;o^tI?5#PnE(|SF>&4c;Ljsku@e8Au7FNFbiFx`4DrI!c)kx ziSlL(8XV0M|5>&^Hu!O};=w0>jTBDeFC^Em9s(Ek&sP|QdS(#o55pkwYMQQCTK|G^ zgq1fvlu8`;zg`5!H=&Vd2-tB3#-%drGRDwCg^Kj%7BP|)JK@4_J{l-PN*L5psk$-f){$z9RCXZ`w(yoUO>Z|C{7=bK7hLwlN9=i@J95cu|$cX3& zvb8e8<_8b<>%?t@nxQ7^y3T9^5_Ay*@N*R6Qkqu%4|-CxKSgDKffxPS3H_qSGAyU| zcuQlKLSj$tjkU$L)Vu<;w03F=-j>pDI@yP+IS_SxE1u+VE+pgcMKN#*0GM58S64~0 zkpl!E>7}`Xpi4R58`tw|U88tS8!waU!&~?-Ri%77>p`WDKNqCw`#Uge*IPe@+tmjXjkoJm8Q_8}qTzlTXbIZ*B|5F1CrU`STL1}6#9Gq^jXX^%6w~{S{3>}S| zl}*_;NtDc$>`vZ2Bw!?VbVMl9-ImYTp1<~Fb)?!`AKJiRPg1|+`L`0{# zi9x4haeOmMWU^knJ`?|pOVP&D(eaYmuWkO{t$;#@|K4K>cmAVu;Ro%hvRae(sMGt& zJn@Yb+s33b?^Wn0e?UA{e$`qR7IC=ifA$_`dj2`$-wU3w_j{a6h&b?08$w95wJ!Y) zmqdv}vS1*CM)Z-gE}0n1vPw#XW7yO?{<0c@liC|SR%nBT;qDJcr}MUij|j@2TAzAT zefkbs693h;0E%w-P?7ze$Ab3XSsf3bM<$M&D6=d6aGpoZH&k*z@!x>dM2ql#d*Ag3e*FO&m& zZR`yqS%y23`O83ac#Hm|R9)?#O)oUDPcL4k(oUzxzeB#ca`aN40=fa9C0EWz+`dbvk%dcO zGxx1PuFf+tOUw+C%zGLoEqI{Yr>4mLN%xl)h+#Um5r4+~dSkwXAf9}Jj|$iEB7Nm{ zUTtho!{_yaP3q;=1)%*LSD?b<<2nUzD{0Q8e`!RGzu=3W(=outz%1}7Gk=WF;Xj4- zJP_{4Q|ZYlZoogMf%5)rLvuB+u@NN14GhRJkfaokgK0|OrjJV|eeFlYbntE&KZ9`u({&uMDFx>hOaPN&K_d@wFMc(0#PbVc)zTE7g^hxU+|oD z9GTqOXp^s=RpKP|%$EG^+NMLyRYL(iNv*AvL`vcJ%z)vLr~rDpizmYYLz<2PD^XrP z$_hys?o!h1w22Aechu=>SIVXmx!z**u{z-rLB<{=bN}oiMi5WXI5hu=aIA=$3(oul zSP-(1t;th|G#PaK^)O!oJ=KqdI|6u~Wx3(uTS(^0^rd+#lmH3(XVJ%%w3ACj@qR&6 zqQJbAqPzYk0RcjqB<~r4NWF3-uC!fTqQg(M(X1frYg>*yvyTD%5f)1bIdvjuZi+7D z+Z$bMFVkP`ACH<|X#~v{C<>|u_a?`MjaCuZ#5~AgXA3H-%fCEMLizlQB>>n~fa^t% zB<3tXg{c*Sudv7BzK79R^m?+hG>=}_q@MzkiOJNH*-|E)^6%qeRq$u@6hcW4Np%a$ z0#WTs64?khn5rG8lux4QTRNmaY2=mv8)5RllHsCv&?MiRo?XaVxpWgfJ1&0mkr$)m&Y2slF2qwWI2$U*XYDI>$uT|f>yRWT-9Bj+>fN2ax%9!s-z_0N^4bcien zWp6CLwsZ_eJ_$Dsm&{*=@Ea)YSq=xqL_*IH**jP90vd_)$w4KfdUyi82FffEeJnCe+;k|!vPwx{b?X8ZKnwiz)WAj zV2dQtvdS6!xPWeLJm?PNc?N7W(pR0_tlW#*wOzgX@)(u2XF`UCK5vW?0^z5u!ZU{e zeRSpc?H8;9jEesHpTERrq5A|dv-Boe3mNF@IrM+mWwB?KUEi>9s?=ENOa5BUXXIX^ zO8v;c+_5w6pWRQAzvOw?dI@VUgef9~VpEr&I-p6a6Wng`wRcYI7#p2_wzjrbGcYhX z`nBfZE*%od@fl@rwxYlFVmT5tdanX*Vi!OXCTuM>zvXN@E3;oKNK{qyum!!&4$a?S zH1nayvH>4t>Gk^J<*6mp6|x`q&QU?}KY#e1%8WP>d9&O0g()l{m$3FH&(+zQ5)l6x zO8GjVX9+)|rxcE}QVQvh># z7%Lo*s49s1s)iKo(>?$U25O|TZ2Gk*yy=*qbSi*cmc4URk0a3hKd#BX!vIS(c*0Uz zPEO8Qc$XLU3K@=rEZ~{ljyjP+Z?Mm0H?dRJh!B zKNeS7PqV@n3E)b114ie@7Jw)kUv?2YEZR1h{`x+ku!oBP_d3&s2>kZ7fPP7w{AfEB z>aL=+sZINOdE+jccNF-AH2QYRbhOZ<9QJ!#%LvZSG9<-#4dtGA^H=|z6+j#=+8fpq z?PkO5oE&Q&7>k_DwIsWLkC$^8=I%G?iz03OSv>@Jw0x^DokBEs7^Znt?B2`k{$`~= z`~LQIkR+|}*_7tQX8UO&r)hTx=H*$1Z?yZ#mg!iQ$j;v0kPSxj+rvJ9qm72K>|x&r z6Tb)pzXt)Zc>9Mq+oSlF-5%%|xe#iX=?4R8fg;0xz`Tw4F%1g2pzWI15poRBWGNgPT3NeRhC(A7&7>x{a(ZeESNR0QhET>`Dj9p zx_9KH_0Su;{j!&{(xekURhe-KC_;dL`S`E=K5x*iG=T#wR?!Tge>qogOJ1asjs)R9 zpEIwrUumoD-vrA%0<>fK-a=thE_O3NcGP}<(_X1x#t#Vh$NAKED6gE9PcPT7eZv6n z!O;6`+U;ywN3;AbSgAmY!f0V1uSeGmK&kk?hE~c~&z}X* zbEBz`)tKRFqMAK5eOiT+pe%B$h!uYHF6f`#`z7$%XCnaC5HAdke+O1Kaf*>EDP*;@ zX0!{|i|%t}QL0-74>5!J4FzkcU25OH__`MK1DA1nL|1;`Hwl98Ko)Hm`L zwuA!EIl@Hmuf}|14t*|~8^0$gL1+Yj3mr5*0NF6auQjs+m z`+%M+5@h^`bK~WDBn*%)E7LWS&bN_fQ?Af-3nWu%QQ%btHk0!DL;&NRAKApo=?Q{F z7=C41dOBczB)U@iv{Qq!L_sqEjQ5ox!%)tSXvqxq=VDDK_wg>WWz~z z$~fAd33SmP6Z(H$o)_Sx*vd$O~IaG zXKx>?x&o{V5u0w@WE)P8<;ySW2Gm_zbHg5hy9d}?yS5*{4Iso5*1Y?M67Yeo1){rC zql@)d<)6E!{r4Fs1{T&jfV;;y6~lGB7Q4}dIl9L41smvIS;Knmq-r;8%MCE3h!e=b z?DFNo+xAF6H(viW#MuwjTnI5a<{Rv(Jv=-dCS-_Be|#;h)MbN#E)Wg?&!*sn-S{RrJ_4d<$u76ZN`TT7 z6d^wTg@H=4YQ}HCGTs3EfP*Km08 z;xS%6b+Zre;s1nTw6H}Cj2o*04MM8?<*3l%D!@>~NbN^E`LXK&$=!c@bpi!`bsZ+w zG3p6ouZ=Er&Ysz}a~=w(VeZDQG#en@2?selJ9At0M~0a~Q?OXvc5*|HK(6^_v4oP-#7esr;}Bl7YCLK z9Jpg|+fFDkFfdku|MVWL{wYr52Z-7Q!r5}`hpAPNyH*{67>sMc2g}_MaOU54e@AKq zy^`FaItTEx)H&&q10}@Fh91ixpx#@n{!<`*Z-x8R>Q8|rujIYZTMnP=$!s$a6aW$U z>92sP$jiqhp_M>ukCO#)`dzm*4}9XM@ONdw45XcvS_#K6F{d8gfWK#O>Su<~Ucz@& zlFv^@?0xX_n6*7f5Y#nly-~>b8B;NOjXt9YBS)$u8^;w=ke~dt`G;3P`Colo{WM)o1$ARS6pq5JiC}k*4^s zq^ZtBe?iOkjUlqbsgg`@zB=?D0u=85auiTNhSkgd%jsg!0vHaYQqh{{ArN)igg;|~ z^zQ*`d{z^`Sw*|Kig+1OVnx>IqK~pJzC!@NaG zZf-6dFs4ph!<`^@dy>#Bh4Wm2i!Jx<95w`S_DngXsJo=663o@LaZti7R)Am`$gYqE z@5Q)a-{CwQ)q9;8EcW|27CH%(Pl9Eic+7(+JuNK_Mrh9g)dCnKzU|k$3UM3+n4&QB z3T|(<&b=NfxD})V+ za@V#9cb;Pm3PV^RcR5^jTcFCrr0>HfkOWX8{k|T+zPIkRe41BS$Z)`jB01=|L~hpz z_ei^x`HJL)@14tVrVue>B5FUiuW-HHH!|-s5V!y-dzQXva^%R~0HEyx%cLUOe0m9W zR)KnOAIK6(UAFeWH4{TBZ5MgHz)lJ2Uh@MoLkdipLEoC+_S?de7y*3)g9%`Rz{$~ftf;YB`anh;-HwU zbp^w`5PE30KU20Bp7HHEF*oq~Kph_eWbRC^2TS4eHLo{9#cpXKTS3CkJ2p-s{TJ-d8rY4In&Tan0c{hd@uMLM0*?Ya#WoO3`!s@ zdl1+O0gEU(IeEtmgv}p5e(VIlUT5cQ81bq1^=aci*mP^K2pmD8gS($1ux3n@90V}2 zXpUmHYuFq0nb@#>0?SB+uf8WvSe2EPHEw4CX4(WD2vNb0_;$dYCUE9!z+yarc>Nk+ zf`vp?uKW5+&XX#@NZ_ZVFDilGuI5r-H+k@10lq%U^E%!INA6)ja(Yde`iqNv-8U zpr_a5$Z*hhbFky|XqC_K_KWvY#l&E5LV~` zhLOSr_?0)6DHYgjs87;N`St)gJ73Vi!)$bG>p|__<6SrK;{2Yb z$fBON->akF!yExC;y;cc9-ESIhgPS9w@mv{k!ofd(3y8ISl5=hEXf+a~)o}|}YuqhGX2*d=k^hf4*f$;5+b%l5;xdOQAJga# zvm&VYjNgT^0!Yv;Z*c0TlFPA8MN=_h9G~&!FZz!G5)G0f2BYL|>2JNl0#x25W%-{j zh%iR($Un$RX+8gpBXjgWjtB8?)SVxu`Amcmtd?$AwS&!|9s^tW2m;J{1R^?7l9vH& zQ^9O9Ovww>MC8oP&7bPIgZep)))E3MZMe9&T*71k4-O9a;~my}!refqUitR$1t>rP z6SeVque{sn?YCehjwS@~^AAA@?r1CB-VDgeLmArNe<@ya10T;1f_W%Nz0LL)>UMT_ z;UR#OUOb5k`LiUfzS2{8{;UFr9-yrB+mHB#8{C-a%^PQt`>X8xaM*el14|cXTh?Gn zF37%xZh=W2vZ6rXCPE262Ej((ZX-d}`!nF~ju51uvY_$D0IuqUp-p&12!uxQ@$w3v zT4($ptg<_8tT1edC}c7~)#|e1{+iQkTWF{zwd5fV+wl@ zkWUq7IHJK6dSJy<3#cS~2dLQaot~e=%)Pjcjm>Gj+wuC2FenWFFXV>R+UBY)(6fAh zyTf+EggfZQiR!2F#K0Cg#rfk4?snv0Lw6L+>6Od2!5C&OwHP}71_vOiv_0h2K^Wi` z6IjZ86r6BF2z%?E3HV}qUPiiKeSdr(Qg?T^13!hud?pFpV5$G->r9)f7Q*9^e&0Aj~qh9e*kGiVc+d3FR*0 zusM7IVmCK`LVL!?$J2A^GIpg69@2PK)6Z~5uPifGN!{J1W;g()!)|Xwzi91PKoW*- z=yB}?C7n>(Mr{HRMKkQfc_Tz#JKYI|hyn)yi<`hLM9dU1gp?v|QywIrIpyK#pQu5J zU6~rt6?CDa!x7}ljU$}f@1ih6+=ntffdcpu|C~W^%gHB=Khg5`M#1MMM*#7)f;OP4 zR;DEwkmyE%+qbQCOb2ya`cDAzI(4+3mh~xUCNe5OwXNo+9smOy%>x=|a4}A=wD15*HYgFuSA5k7IZ&S~3%S`LQ06sQrB%5*uCIKu-zJ6Dr{sK8^KWQUxwz!i}O z%ELo)Keox&u(aZXXvsa32<)LhKs-d6`A@IB2&qsd2%a$$0D}P^%#5=M9&LxrY$YA5 zhN^C6cORCOFmCE{drd!*u&EkIN+SgiEJO~tGJh>APR^hWQ^dMjuMp^ zemO4D?}TH4xa^10?0(uv15nF)9!APEwdrSEJ3DgxW^crOS^gLpGJSo02n{R1kk#ce z14KL0*a=JTo2>1Li3tfwK6TzC`2?tL0?0yZr(>6?wcl6|KxsgcTLOn=qW(B1h6-ej ze~&vK^g}IeZDqjj>{@du43)yMink!WySuZrw1kW?y-nmM!C)!Im;MOjhGHwMLTaZC zXrtCoM(lV^zNCx70$<5b=wEdZxRd{$h*#5~z@IrjEFsfN&Yj&eef(Y?gI+cR)tT?(>}#BM3V#Y8TS?!IkJ4?*o8(1~mWppx`|8 zWr+LHxNueesA2dSuKa9;#B8VmU$Qldvhk(=yF|ggb#=KBdkhNhYoM;t3Bn}sLh7;f zH92^C>LydimL+9|303q%Zsu})R4a|dVC{yfZg&i3qj!E_5jaQcV!+>q0XHZ0Z7Q3G zf*;&b!3f7D?ZgkVOq{9o;6MQa#(!-Pi(&t?A+U5hzfpOcnw$VwY++x;J!t=(C$IsS zD_VY`!Ni_nkR@<1|0lX$cHizDnFK&Om%W+XPoL0X8pkl-3lmgqiH12Lm~J$TQiK=# zdq+y0z!~!jAUsioL+d?kWV%GarVzechP$er{8skWh0&GjfiWR#V z^A7x=<{$txgQg1Q!U5D{kIDs77eD`MuYX&rgImr@VD|$yZZe3pUT3;cvjRPR7^^s= zX&g!ZzqjAC2tcJ29H6YhvO6GF9(Q`(jV%^cXEVDF(+~bys08h&e7idQWDHiZAD@~+ zfWVHBq}7%6!5L(BFUT^vy|0~>vqdwdyYN7G(?v|6mjKN~6f}L=?dVED)wMx>vA$_5 zmj_SQagCBymqVHTf*-C7Q4po>16^)K4@3V)z?V&((2gXvt%FX2d=NM%o5K}9345FZ zB^>E@IaSzSw-ZQ|{(I!4|0l&`)#h6emWG3JW`!HTMXepX;b4WXIq*9Su{RVLF~X=_ zQSbad8*)I{6!eofog)c@*@n~z3g7H;B61j`+}VjiatdcK0%NG-j{!-$!h0^be`P+D zGBkAKMf%js{-#Jqlir`asdNK_g(QhF+c_Klam?NO=-LI;G{~+QXOhx6rClfHPcs@2 z;^nKh%L=aA{qEYu6*i^aS%iQANCAMzg~h55S`1M_ z7$``wz=`XW?&%RgSvC(fkH9@c;q;n|OcC7;6vA7p;wS}a&lTa{i{T4v3L;8w;To* z8w{JqQQOb}3MIgJCRwCcvjuMOgUG%F-vA)21HGxHruAUVQSI04i}eIDoih&t-MltL zTdMMfIAm&PtUGoQ_b;ac01kGLb749Huki&oKLj({6@}2lcxpd{5H4dN#v}csHA0}Z zs68qjWp7pkpsv&U7d8u#mvWH~J}kqaCJm9n-eE)Hr+kMVM#le~M$z(*3nQ+l>I2K) z?Af!H4b!bg{fuTea5H@#KCoOkCQ~aFLwFd8G9VAbSY9@|P%`CNyh$pEL^=(qFZ{Oo z`6(IsZH6s14#9sMD?MLFqiW~-AcX38Z|}=LRD8^jMTnmeeyi|yIU*s-)EI?_Na{N4 zli?eI@~G{$&o2;{)bFn7NP4|yw?6+Tb2$BVZVp8PXZ+hYO1{?SW^#7+q+fz~&e&JD z1QtgWN{>0}0{4&nh@ZV#2z|{*?I)+HQA+Gq%3qJU&5`Bz1J$4EVfaX?+&E>>;+S-x zA`E=~thHI}+EAOo8T)2JI;nRAu+x1@-4_Qm7%H*p9yj!V&ISW@gSHuowPUsF4 zMd-$Bk-(*lu1`8)PHV)opifX@rVJ*b{kPg!LKN8P7mQA9c6WUSs}@2k6VhR7nILib z^m8=^{`wo78@wq*+@KZl8jpc=C?Y_m+4J~P=%(^`Bdc-G7;xmDac20{5Y^_-r2X8Y zuzJHN0`Aj*qd92>s^h1xA(wr{uQpm3@FzsR%f$}Gf`HB=^lk@=2ynJ?*^h8^k})B$ zT(p19boj>y>f>W7Tem>dFA?y18Z-lsn^5HhIx7j@W=G)SBBAEN|7Vm4V*3M)nq6hG z59e!;fzY}vC?5z43RZ$bhM~{Zrf#hzE-WvGoj<;);7fBgRQ(^pNYS^7U;7xLi{z*> zXuF8)$uPrC(I^uml4&~fv5T(mkZos|d;?T_h9c@eT+i=f&J<1jF=;vMX{ z=vT{<2B#ckJhaZDHSE{At4$wP4QzO14=_^&%CU+m7WE0?^O$h@ zK-n5xP*WIjJ$lYn^ad=e+7K1>z-KG3a=L)pY4Da)2NX!E_yM)Tri{{kJt@xzu^hl1V-O|WXGD)izYcm z+9*=04gkDn4M{4c*xyX&c}+7>?>F{j-3kVRWEd+4A_722cvjzqup2MEM~CpC$Fmj> zMaeI(BPk!#xxf=|tETBi3N?o@34TVs`bi*?x(Mt_xSY+u>U6w6sLmhaBzHIs@9QWh z9ETz!!&N4%*Tn910NQg$qleAM5W#hj8z6My8MdiCj||dIq`L6+X23di=lY3mI!E3U z#~Ea5k0RAZ6VghHaU;3oggm~727z4DUA&V`GUI5Gc!tA6WIixis}#-mHCF9UXr3oc zw`K`~9*W%yadp1|_L$NWj%{)fgfyJ48w;W;Le$s^v^Ul8818|Vl{zyjbKprN z`iludr1F9sq~~hWF~Nk;sy7{fkkZCsCr$avFX1hLn@2ANC&X0y(uol3Ce?BqGpsWG zQ+q2s)L9CaDEvu_3)U>HI5#kZCbnPMJ%%M*@f5v2Gm+X~9&)R-^#&XjOI{&(s2 ztHaaU6x49M&mp=ARo$f(yBZ6Mnl?OUbL0Wck4R3nmN&IA7z43112B1U9)sqRfu`PSY+>SPy9d8k24C{d$VbzC=&yV(Ew2e!`bx7qDU# zJ<=(A8j(lCJjnGU65r!F?OQeh#wS0lA7qIS=%Ysk9sUyc9X{rFU8Xgo|2OTSiIJS5 z->9UQ>UXmI#3?**Y1sv$@-IUPF*i$a0F%xtQjjpAW!_iPv{A)MN7mMOhV%QTY6LQv zy+sy-;s`(Hzs;{3eQlX{Y7z`jBCX;pp{Gt``7%p93^mBZKNg=W5_b$-tnl9+!khWJ zhh{QI4EXPe^>h?|)*dIiiqO&TZF_H#*TVT_>%d4Eiyn)@)1r!fc7{%$GGm7DlyUcs z8c0Xcd{ zy0w*E6~pI;1Aa_?wDw&4d-JEGCJ|$oVBpWsJjO+S_H$;tiS1WI;a}n|*wb3{?`JRNK2>X`YWYO#6CJDoLF9EB+8$-eA@PsUs=>iO5{JdY`(`7)Ws;h zwC%`z@tuWO-;`@Ypg7{Cmo&w90jv^LUHO43&M!;W{*_4@NrNCPTm2QB7f{IRb}d!q z;t5aZMa!MpB`vZ&J(9+`2s@vcdzDr_n! zU$x7v`FY^(0!EPBk_t0J5_x|+5iTlE_w#R!!C?~mrsJE~o2_1SAj|mH?Q<&E^W3Gy zR3YY!Fv+ZwZ9eWL!vTqwQWG~m+ykp`>H4{B$0EYxhuyzLu_es}x?hA@A6 zIY#Qf(K$D zo`RxBgR08T3u~p^-Bx^6o}s9&npPYv&rR96b9xGhO?Xj&83OM(kDsg=jr!$)9zk~i zs>?d8@PQEZ+ixzi{h~aNpb~TJ$Pye9@m9V{`V?JdoEYp-!XHWcxQ;K6`x81 z8&AhJa1<&XWnXELp`)IZgbT`xj$@q~@gEE}ov?^C>Zuy`^OC7VD5y;Ci$AUmNJIY{ zb2+$#;Q+;#GXCnEIUhm(9a%W~s;81OsFL3N3ps^T-lyjeA$CjCk6hx|} zy=BBgkqg@VGS4Il^RvQ(HeI|J55n$5zCAVGiNXqMMw(u6T8>s=B`JBSHP*%?jf zAHPsfPVC#jejKmng7?bF)!Xvuc za!`Eb*_Lw+DeWi1a*+uzICknECWfF>0 zgh<;|+6}rml-~FuF{>SKvG~6XHn2mSThwJ`2uIsHL-U69#e;||? zDw>G+bGf=S<$0T7W4eL$(pUCmQNP8X>ocBcpdzZ4q|2?=py#Rhzis#=8>b+@M3hN> zY4}=!r3WW?t&)W6J0=}-wO>$C+OCVGVl^yB>0VaTqJj5Ys04h|Uxl<|&_=J#s2wLg zb*}Fgv^cHZJ}s@Eknfkha3EYen5JIlrW+arGKoYPY|>=TKfNdwiq-NelfTm|YuXJb zkJx8ef2>mH>@ej9Mlpa1CxreHRJGSDEE(^SqE;=i1=V>#F-%dzk;RctnJqdj$gHKN z8d$zkW(Qpiu6JCw%p}RsQ7aN3se6cYr7tvkpDb($J~pdY9$WW>Rn;Q8Lo^vwMbzebI@b#;h+I)kH6Jg`kkaph)VzK#P?`R_{f!LSFu~G$>p=5RP z{94alq|^gmMp-h4k~>k*D|DsVp6S*VVqsE#vxV^T|GTeWFpU&=STWtuK{vc~P*WmI0C zIq#y`>CHBl0S(w@>>?{<^z!C#+xgq#T=2b}D26?rPGNsa+ul?92!}I-ge>Exj(7dU zK?ptkgt5*sGjva!L&24dKYLJ)`Gv0<^!rPL=*sk)Ee+ZPbKV_r9oU>5mIGT5OEc6ChJ|HK=d-?vM0LJvjJ4!$J zv7j|^d98E3mF4IqwueKr#Qn^~$~0~zGLvlt)(RnlK?X_mC?&JY;wXYyduJ_*2?)kC z1!Z5@dWmK7L#d5~ks*TJgvBc>&M1dI-psDP;eGMWNv$WRazYaa$7|^i%G}oPGYM*z zH_(}bg}%;TtZ8H4O5mz~g1b8Z-JH#!AlV>>;wv81`++>+V#nm`?ytO&{lIQB`5i3m zaJ+@37cbu<&nO5j{b#08Ka`ity14L2<_Gw+b zrQ@E@&o@}Bkw(${<3EugPn}O!&p0pEqQRdroy@qWZ?uT=*2AAbj*fiWIzv~eWMy6u zZAA@qw?Gk7b$2PI6#FOEb!!)uiTV(-q~zv4KAN20?L6orqsnqHjdE#DJ6hT$<7Y>@-Mtk^)yTrH z$<{KA57fI13^uIU{%wL1sy}xdk@ali`c>pmBv!1kP_YKdD-|PTS3c)nwzp>pN3qj9 zmySA`gXY-a3aVGgk1fA|Uk&nq%;H;;*Otv=k#7!?suu%Vp+8SnP z_8`1p%=T&}83?l1Lyprhl#|dR@B(P7o;9($$4EW*CqyqIT8-$%SD)SLB<6{}eChe1 zp3*K9VO1?s=ts(qyv4@c$>uTQ(eaERf)w64udAuOIq{1-FU}}IDzLTI2Yd0ki^j9g z8$G3n=ADH@bC`Rfczr}1aDICEPRD1WL}{NJ27(7R1r$~u_{X83Dt}KvWKWb(E1|f7 z&uuJcbNA9t#467cSFR%9R&A{6IB~nZE1UE9KxvLvy%B5cX(b-*&EQ^O zot~gG)qIT&dk}q@7`h^Rx>z!ldYWs# z@H&CwQ5&Bx##Z#Bj(5I#nO}LUZN{hxj&IY9<`&1p`$nnCgjmpw)p~S1)4(y-lII%t z9o5^X)g5Nzs3_NgsE;U5)DO#08B}HHZ-+b$j=}QO6)q!~%@7ngxKSf&!|;%3(gq*; zW;}OsjKUWR7{qa}rcfPdbD*MqM0#-Nd`(B~>Gf-ifwmAoOwT$qEgE{W>EzEOK#0)G z3rvaS!LNV~-L}@R6pxaXIeV}ApJN(I6d-KtQ@SptOC-=w)xDJ>e*NKFTaK(2 zg=QRLje2NV=z{0IkRY*@vD528cgRMSwT%eJ(Ad|O;>0^F?r}O(y4{3ZY!bx~R%5FY zw-&ffk5>tkf18g$Dj^?lf8NlQyh)wZCZiC*u_GlRacG-3vHPowlEY0c!CZl)>pdE~ z(TiT%#ybZg*V(t$zZPXQ<6n38XfQFpvoJUNh@bz<7E1X>jun24mTxF*i-S|&2w1(d zK%ZN%%`quOBIHeZcGY&C7PlDJNFe);W82ew7_j0?`XV{iZ?JoG%hZfl-|ToSt;{hb ztCT|g%%}E&N~Xxpf%ymLLvyrH@jbVW3PQsvGwIsIPi(Uh0~YghuBoU>bZRor#K zcx;#+{*>Q0;T}~g{1cNO-L^w4o#d%IE(2;A2j?5cyD(Rxn5FoCD5*F-C>sYU?E#}t z-Ex~DN2`U}$`!+Vll6A?;?P7w{$VR`++#$^xo_j7t|;~9p0>#P361)|8z{56n3-Ae zGQJZ8N&u*fT$^j;3jAzH&aO9L16aCTf!Y=silDG)t+P8V+^PW0ik+(N4jnFItDh?U zm@V(z7s;eYY<5nNEv-#p0%dc%qa;IRSdhA%I=VhGxOr~CsjS8LGmZlAE~2PiR#yWr zMNgel@#&r%Dk+P9qMAsbiM-@z!ASbvvB~W-Wa!u<NoqDIJ?y_Kig_Mu0LId>?Z z!-}P5m?uIJE*`)Y(jakvNcWFj3d>JTyB-w{4+7BxrfG3pch|3R21%@@MQCcczmT<1 zWkc1YT$|8s{wS7V4T^w7#TxYlbJAZKp6?<3&@VOVYO^zxSOUBlZvN5Y^3vdV)uXE` zt2al=e{zY*Uv^Xp+=!r2_*!YGsTY>;)^-a0as+$ZYC%3B=o_XH4nvtkT|*T*cFS)b zMW<5cfvUWcFEQ#@dUsTs%q`ewb@&VlEClde+m=S@$Yb7y`Lz?FC#-^ONW&DF#bH6u z%9s52l2!Y9zP=gp`QAl@L>Zg*8LM>hTPZGz1@D1vXJ0G(+{*W^k=Lp^K^+(EqG9bV z?U=z>Pl#06J1@3l{Z&_SF~a9u8!53O;R;#fE%jxvMh&=-m{d8C#-I-R(Q~s?!5{!E z%2Pt@M$rdTfpAH=!FU{`TVpHUUwWgzWqCU1XtOS zG5+kI?%}1Mf*E|bh7c*vgzXK(QL_HJQEfJ-na2j@t~6Ul-qi|Jz- zF@;ZtZg^hJD}3N60zJD&@Wm+3>E{irO|dpfk|L9Gn>lo1=-|O=)q7GUh0rKpJ#{@d zk+Vg%^Ri|i6)nOO6SY+w_Ti!kh!x8v?XavcPFNgqZGBMMvhXk{_dzqpGfgSVpZNi4&7iiml>`nMTOr3R9m0Q>L zHwaSF-QBIyB^@H&9U_7>NH?4A?nYAS?nW8}1f&}gl$PdO+xL0j=NrQ@&iUiu(Y^0` ztvRoG&FlBm;%AN>dU$Vi14iGuqyas1L;FXW4*$$fA(7v$kxE2p0Or)US(V}L^Ev@~ z@VBv`tPY7Un;Tt#m$tQU?vu04U{UXer4Wf(%Kh2QO0%8S+XtyXYIyYHwIMO*D~-eD zG$jfZkpe${{bWGHWwvLk)0 z%l}Qv!tm17;-9qnth1JRdizv8W0uD`SZc$@qu)oQxW;$^?&V$#@f77_lRBY%r2f1u z?#hLH*drhiPdS7xStQ%J0(~-W=W-i3Lb&ySg(Lma^_(d%31}=|tthZk3x=uk!2RF` zk(Ov1$lD%Tnpybd_;zRUO`6VT|6jtWo_z<4O%16h9WmihFzT#GRg`EP4+~0hu9Tn{qsepX;FN}Mw7*|G03UtPsTy6`8>fN0Tpx@GS1oqy6X9fEIs3;2-KBsh;~WDXGv9UwN(|9K*VRj_O@;|!)*MXU zD-$Du_*L0^{8LJf|2q#$KAst z?-NL|*7o`)?vUAKns-#um_(5q=QG#kX)& zam7XzF2Qc0XgfeidnPECfRM`VPhpJpwPthhk5Zph_$D@%U3kci%H`FzDH0^~4mz0PVr%1XbwC(Q&K zAXf#&+?8@f$l(#Uth#9yG(&lw9x6Mp-rzHfH#@%Et$tNX=&r~%8lx*#*`L4dB5YwD zetM_;$EK&<@yCd|L6Z0Nx01Hxh(7@c8%yB9ZF?kM(_PIR{WGB45bRfgcI)DPH`4wZ z+KwilFtvrr9BB3Y;D_*)i|iAnER8G`NYW4u;dl77QJR9>>u9kg81Jzq>E{$lZO%IW%@HH)<-B<#eB=RxDD#wGyZ}m^4-z2`fZ#8`5Gr@BMyF>cyKR8ab zE3l;-5x70p60x@#%-d89nQLcHeOLqiH_=2iT2V@-aspj*hjX zKBKLD-{D$yG4jTYcc05ggQGZhL>g3Z7!9CLg&rU4ubHe=zI*df2`*j7+qi)*QW&50 zCvw8+XGW_{?O(tutWJjA%cZr$-kx`aLLm`HuUc{Bwj9LR9f8Y!%Y{5l2WxMWMn za!3@8MduC~mt{Cc5}pO@VTYZWOM~MTym?=rlX5jK1nh}cPKQD9J$S)!{4A;*%RDFs zY0?qPo#S`I>e7BMI7wig_>VoWrM`?*cX%hoomkHI4Na$ZSFK4C+@Oc+3 zg>AS3#IuVL9ULvb7*`yIG^g-WjEDq0?QRXfVxuk_mQxLZ7Xr{TwepujIa^#_&v3g6 z_h7XmzlEOFU4V2?z)aL0cznUxnmUi2NIR80S| z+gFf>qCxws<0b~_gy`0Env92E5VP}-RLjIxVzwluNvP;KL;esng zUV@5F0_&UemHQLw=rb#|blkl$(r8?b31ObvL@q3xt=2+>oW3j;0<&ijN+115q}2rk zst#|HTCZsBV5T*fK^NhRX$)AGbrFga0gZRq>KD4Zs35HYFX6k`;h}s2z8vGa*Td?2 z)kJx*iIxeNp>(4I`jBoLtmEwTorvZX^A~0soo}#FY#Bs+t4$Q$4dTtg>=wLs>->2$ zTd|AtFDhJNZ@1%NGrV_8g*@@RBW^?KuY)GfJ8!||6+tPxJN zQV9)L*k)e>$@~a&5I-6x96Bv6H#_;MbkF$ETB2kq7MFpbM-T6oxQsxvaDXu;sQ9#| zH4YtuXQ6+*k3Y0A-PB|P^e2#-olz^tM43w`|Ni6cCMp!-aFc~9459`g{u!-G-l{Xe z7NSx8p&WHXSxVdOkS9jgwgM>`@dV9z7`~u_5fT)mOhR$@GbRN3$CRfGFJ5&^%V+f& zlG>Fs2L-k2Z2Qn=bexaTK&5K=>&$annnTYk+5gp}p}q6BdgduV-7z$!F%(&x&Ah{( zVAR36M#8-d!mw!hE{-1;_5e`pr$-Dx4=G&P3R)@9CUZ=n>zTuS6*`2JJ3?*=6jj?7 zA$|U#PZN#&K?^@bsi>zR5iOgnKSqdUs=%G0N6x{1O!h8zq7PQ2jZJ&VH_f03yJKL{ z6F4pFFEq&Gr(8+Dp-im^rG0E){9>h_q?6zA)mMh99!`IyfzN4+?$_eqRA(ts3f=(?AH z=KBuIv2bngpL7jW>#y*9Lp7I_zUq;n&=j*<0`0s+Ba3$nP**`QPc!tcA0;uuYlX1; z^!rG!$8&f9J#t;?sPYbOXT#;te+j2lfjzKDU_o*C;ajW8tg z1Ur~O1c?wQztIle7;N#&38!q3xOr3~OrYDsg96oK$qoCCvcKH;2I}GbU+l%GL0|bL1s&}) zcscYIZ6=}Cd^P?1%7&s}N-A`4GIhd`aN*5g)eq?7Pe`}SMvc8uGteF z4697SZ+hqiF|C~~G0DE4vq+C7_W+?NW0NhsH!u+eT9!r{ItrK0@qd2PelFT$GK$VS ze($ML7HD5F2*VG|>Xq@}790m@3d1vvFKuftKs$itSGwf-lbpdFv z06iEYAS*`I=|2M(BUIXXTU5dnZfl#Fp5N6Q?!kTqZnnvb%S(}BH z@y5(}`GV887VT_^u%Kl$QX-U?9hCmP&j~4vgS#Hw+D?CzwdSm#pdt->cT|0vf0#p% zEMVx@d(*xgbON3Fju#{UrhSs^V)@__DcR-8^yeRHk$yfkm_UsaZZXS0^>w~Lpi)q) zLM&_#Fj2tIiY3@r{Lk33A31NzbTG;Z$*Fd8@AI$Yt0cM;>BT3;8!1EvQJ!$Fbrgmo zic~N6-N9h<>+@f-P}12v+$0AbV->|TXJr`-o)p&Tn;8e3k*}t;WO22x%9YhN_(|%- zLy`RIAw6+n#S;{>ZZ2CBgx}#zsQdhUEhCFNXb<1BjW!g#9E#AMm+9xo?(Q8V74iZP zlEdm9_toz#RGrOjICh6L(7e!qD|mS^NTNjRoKHqTx^B#Kj8s4FTxYfg13zAc$KC}# zHjYSw%YcW#1U^rmla3vW+Ksxc1QaE&Q~3aGbb1?O-gshiI83WA~X}#dfqIk|AYvnLJCGDEFtU z`E@z+L!|U;-Nv`Xp?sRPkbBdTEQ4%)MVuz)U!jI8zTVC{qzt5!8($r}!Bv8;ST7+;+%>(i#Ro``EJJxR>;+0d(-kvngkW~G5I5MajNGNtF*7V{%S0G|)`J9#xD?ucacGNQ=~$lQ#(vzu`rCSGcW8gmnNxrl-QFTv(Shr zUiuxdJ%LgWcv7%Yq~+C?UoUFWTKBMElx)7zpcM*M`%?T&iiNC%24`d+fr4H@D`{KD z@#YYZLpm#p)h*Mbr2d%Ucn)Q7pU!<>OS7Ja+^#qW>8NR(;EETPpUYKsQQ#+yv_5lBQMD`W8oN3WAM?tcVa?%v5x8HW0B9Nftt z%bn25UiULgCl6i-fp598IF^QbmTk}O6SiF7khbN%mU6a{xm#~$x%-2(Lr9==WUuHAE6vS0v6;peA6)|i_05+gcW1VWR)c<^^upA=)g zZ#>lTM1IR5N+B=haq`Y*$F-j}CoTiEY!~YpRO*-Z`!T(p4ljO{A2G^?kNJvk^p!gBK!CwHSN?2x6CYW>S0={5VI8CR#5gFm zO(eo`Auor%5PVO_9>3l~OTPC3UD1%N=3jhw43HPy9%Cow`*4@)zJ0odheQV%qeuk) z3_{REIvVbnqaJ~w5uz~_6x~bTgl~wD$TE6D9d#0{pMg8f)J(UL=4sbo9@1ZRl=A>_ zN-SV(Z2U8x|E9MP<%9pj#nB2DJql>V;4L3uSY(cat~k; zlzR85WYFXv(kU9^#gQ8oAV^0gSiS!f!pRi)B-AF8Ta}RO01>>v*#z`t&2005 zoNhclN9e%Sw2mp35gT#G_J*k9M|w&rQJA7p3JSQA!8xju^wNWHM#NQ`?pgS{fatqN zC$WRx7iIh?g?-U8j@0Gx!Q{YUcMS*8=Ew;H{UbwA9zQ=&B#q)agJQ zdc&lK8eS@}usP`0CGc67K>%1;5TW1+EX&To@90FEZYI1ZL z?wNy2HiaJ2>i7*%28_$OOI3{_N8#2g@cx#p3vdkGDvUTfVYB-(*BEN ztQa(T?fYhyNAfy9#<=M;I~HC|TRyJlhJ?XKRXZU8K6Am}lkU{Rwsuv;G%vfeDt215 zfJjJh8>&b}*ZT`oqG}5$o%t1vj-~D_U$_1M0c(4N98|m;cU$EU*l79Rao5Y5=r#xF~xDiI~k9RWo&)ZM78 z;MwsE2n7r8N0_}w=$qPmlQ1DdhpQ8UZz?W>-AyNtJv}>$3p&B`jZKPxyi!NSr+xQD zB-USC4^V33N`8o;NnSA1D3#~p*Ag)W`0SqR(rwf3$+@Zy4`o_f#x$SBR!CHgR)j&V z1F-xPhLpeDf}0@bZy!d-K0F!d&!{!+1Rz%?XQo-AZ8TP9Erc4$l>yuMemuRp2_eEm zreM6evoMP%I0kNhRmUdvwjb}JMvgQ~)wLt+e$Vb=WEsUr;xZDlR`K8$CZ2oD%zS`@ zfGcGpQ;Z~Wq!zdF%ZIb~EE;}|h2px}`}Oa$GQjZ1?EAj(4w12L8&dB4OP~P@8#s>i zm*yaG;F?s75+?q|g@7B$ku_r!KdCVM0b6452z@I1rT|L7l?Te)oJL0gmW0`6Rxr#wh6{$% z4tly5GnKy~K$qkH;8-fZ6^N|t*kcOW-Rzx;SYMyP0>_19_3}Vr?TL8(Yivlu zY+bnN(Zti`t64qo>T4DLE{pB7NBb3Do8|ey`Obw%-c;@3YbFI8*Eu&9nrn`^hU>LQ zyP?M~k~@3#Z#R}os4wpG-CB8_ZO zi~YCKi@p%tIVZPE$yPUj_CNUYuMC>vf=}zI%3g! zcD`-ps83e^;6|h||Nkou#NF@3U%GN((=J&%YO%L$b2Jd zp<0Fbb*;jdV?lJ{PJsqxop2Y;P|sE`?%-xxx%b`S=bz0#qpUJ6z^9!t5Bi=WNte>v z8$VffM%+u_AH3TbAD6#f^lg^(c<93&+U%E?CQ7zfPa)R|cCT=bV<7nhhUekB-&aD* zI}_+kl_gVgCPKLZ!= zA%kC_e%mS<9=Q6wtja&T*|T`Ya&qvaBYcQ3SbXS3iIFbvoj8Ni1Ib;g+y#-!dD;`ODKHOu9y9GMBY=;nv$vlZ933{Tag)5eqlLhUaIz zEze+>)^jj&H-<{OZ^c`Dr(Jp{)wWauT`R`fCCRp458N7phuC>sIs?gg91~fF91|2( zx`nuV>QRKdKFxE{HPRT~^`8iUQVHsYs5T!iy3MM`l^Adit@B@At7VuE#42Y7-adAy zCK5m(V0_-^_AZH0xUrR2x%cDqPJ}Xo)gKNJAlK<&=gl43x(wWPt$LP<6lW{y+9jNx z^ga8Z+y`*5n1RL^C!^JSgwVQWlSTX){nKnnj=AoJx2kC-1*P-xKB5u!!R!*8e^=_r z*XIXW-enq^nuXc|giE~zxxmBQ?acKPao?Rm=Tb_**FXd=ECC@PmLDlu6p?};c^%8} zqYgpj+YLs|k{sdh?@h%;A0GHowop8Pn)gXSYkLp{_YKtBhMIeG?4Tl1MMl(aQQqps zPt)0xDHLH}$5orVn_hE=2S?2*Gj_EpA(nZr1JT1kzMtOh{?bT!_^ zYqnC!@M(n)=|?pXYvHL?1ww#QxC*SlcJ^GmSB4^R-Z&-hQHu3wToltA2Tq7Pz&L~l zz=HmKT|cT&(zQbAU=~sN48D*2{~x6A>*}D=+mDz^a;kE88z772j0~nZ`N=be1@sop z=Pyx0=P75K;oJH=9&Cyijui`YLOFqD#H<;JPDUS%c{~G=Qht_I^o+BC22e%3q590)ZE**Hb3SgK#l46ht*h`AVhh1)9RDESD*Ns zTd8aD#~ML33(uXV2B7VNN&Z&L98Y!n#n&1VbF${=!%mo;z|XK7Gad>AT;L6=P_06V z{2i^__(6K`m0Z-dDqY6@!B?l%87Z5EnW+0{!{%Q|&fg}61sDG76=0^3|EVdb*kliE z=vlVJj>aTmg=mT5|2j`Hd`9^Mn(ERs?2&5DY4NKkWi946n?*CfW3{tl7I}kYG@3G8 zz~S@(fx1epv_94qn`J2&Ee+JJA@6JC%2qRtnrj4;1ep}3&EIWd*75U zLyvUU$n0)@zXX4#femk8C#J@X{a(!zsT-;Ka)}IM-HLH^{<5)fR;BbXzt5FD1!eS1 zqYVRLP~F`Hp9U{)cjtTGPa};aK}y}+=4Um&ZgsVvTPkFSKc*vp%#nm#0W)l3qy0|> zj^344iEd%5^iVbVia{Xgip{!9^HH`}=dQjpC`xr5Je%B2bHsqiLFLYoDU zqBl6#%7jpEwd$n?^Kg3+<@ov8^o9s3+Q}yJ_c43lPBG{EgRz1HSo?Nl8n5XkL>@qX zahm#I5;)QuuClj-79s^sNXNrZzrL$Ssz3SpGQ}{t_H`|%PldF4M>it|b!=VQp1>ki z_|HNohaaDv@C4Y2^lh{!;;D{dz~{FhpuX((N4h$b!w4v42gg z&|Q4ru_vJz8WOWjlb-=bpvvCW=)f-0E6o@cy|mlFd1>aC$Z9$YK1s3XQ@@Gl)3v_%+W3Da8GdfmQv*&F_L)eiWqRItgTB<(B6*#!|f+LxTezm3DSh zC=oF(}vej%+qe)q!X&jcf>ra?$YWy zL@y>C7uIG!Ii?Z{vcuezpN<9<897S^O8Zto`UOn-|4-D>zxvEt!M%ijdDJhzo48rVKV1Fa2PL3 zgnLDM$v~#cxd#qxjgGiI+w+IQq0}fal>=(d_SS3v8SDOU={qFxTol?_&#^JvYnb18 zfd22d$0YEIOls%kwJUuzu!eIqsCklm^j^hCmJps;>8>a4R5I85_rvk-Zqd@!T|N{) z*5Vs8UQ*a^ex->G|Ag%SJV-#hB^ld@+sdhaqV!N;|3(e34~xLU^Hu`=yviY5@j15) zMTr8#V)NajgRhO*R|T@J9P81ougSWX23}-z!0s!tBI*bPFXGoI#J|eS?wL zbN7yN9CGE74s&} z+Oe#<{kGD+bm&7-sx3PFpXkcYxD0`*R)@Dj*eX+M@6|XkeP z85kCEnyJ+D#grs;>7Iqq$t4tp!M&68tdzNEp_F6H=Vl3(@T-znb1NN8FS#e3(ExyQ z`$9WPTAcD-qx!At)QxjnAF)bFVQq_6He-GYF)=p=rEFD#pRDEM%`~#Jz5&weuhq!R zw>>h2cx{LwVD>YGXEpuH-P^Ax4SM}Zpv0`y+y$--J|DhD{~_|V%-|QhbcAqo$)9nl zZQ=H@P`Tx$10a^W$eL|pGuN#>eM zHEQZUf)VbwrTNaYT17B=lPAw~bjD#=$n}m$=Oz<5YhnjYZEnH-U+cv)B-xN zACgQ-I^y)DBL>&RbfRu(o1RS+9^z-*@S73erO|u>5k!Hr{px5e`M%Cp20XLhzRG{isfY~fAEhn*tz5Om zBLKk_L@3P8^}*m%>g=4n5B#O+uUPAtO*`8%rQP1zZ8`4BeevJU65 zyz^Gs(nX^&mG8x-)Y5?MVyya{TL3q~YaF<>QbO23+@Y_wEKNUot;7fZ@ZM$5qweJ> zLyF+-va#{LD*j&wJ@eqlhvwk9^S`o)JYtGiuH3)W(?W=P9OFkOYhV2!WTfs!sLNwQFHfxQ3bcMcTae|AY%R{YYAPKGo!&Uy z@!*A-M(2(~KU2z}{qVs+`yR#DnEGIVIG&^+e)ZNoObkQ5Fw2RXoWN3FHEYG|5xAdW zLz`ddf^c13E~3CQ*jFEsWh9w>8{zq^;V2$!chiOyqoElN^4HM)nVr{fG|+RtT}mrd zXqA>3m7qQC%*ah`@V+dTSF-w;j z>Wmj&eoy|(h@Muy^A)>1T0*L!+>ijnMbWP?6~SD25sc z)ohC@QyvNvhqkVHc+zy!k#e>iX530PDptn%t(=O8Mu5fV=SQ0kQ_$ht^X0d&`qmHt zCv#T1`+D>nl=)azSD@tMLSR#LVB6~RGNx2u!eL;)dI9{zM7v*_R(mIXFJA!8$C7;) zIe_BKIbeqW&tvu#LI<`7eMA7e&BMbZ*U&$Kj+*G1M9ECNoZK@3u(Ao#qd>SZ#D8?9!8#=UQ^|1H+d9R;UT1~#(1 z4*@Z5KB2qLGeNME)JomTagY_{;hOSmVk;7Vt{@unN6Jc!8EiU@q z3nq7=zyKhA;iaE!QTyWh_H1Y!l#U>$V<>x6FhG)Pf*a3LANFo0&s(%K7TFDke^LUY z2|2GrBR}L9vU?j{nV5_=h4UxXBp;7U5j1&DeCg7GOTylkAFIiyU~()U42B?px5c;S z{Q_Wx5JC1hEOi~A7o_d%?Ea9#BK^THSsw7;tgNgsF*64YnEiAX14fU&=*6+}cwQc! z{JcDWlr;}t2eQ;!FQv*N5@o!2I{C6vYh^Wm6S@M?htBu!5AGs;hGm&(kA7nQLT+Z<9qlV z-3DBUN_s9LsP3J(Psxj+@W%3u_(*+2T$YMPxkJY=@vg?L2`oPVD4XY#fsuqR?IfMq$%wBA(*09 zCe5MJ|5IhXh2&%1UaXlAHB0Rwa2tG;aHHz(>?U!M&@*+AA-^pk&E~^^fDw5Z&S(jr zN(+1_YGtmp^WsY%eHY&0+4Y zKhh8XCHzxY3gnNHa;Fy%Z?_4iW#KS{;{J#YPSXO5R)ss zew{r3>C>muFJH(b^8feQL*9O%jX}5(ilxPmY0&%t$8^0|J9ZQ0-Szc%L+1`%lucnJ ziBEmLz(+?VTF_Uz=)Lqf0K3u_0p$4d&=it|_c>c_)BCeF-fy97skS zv(m}2?4wNK?6&6=)xsuazjs(yVv96x*a0?sR5@SF1IJ$vdYt_Y#`p*W1hk zVC~Fd=^)^OqnnpLgNLhUyd%rbJzF1CE3b{w+Bhdse`lB9TWy2 z1Oee}8#gdYMf1%;h2!;3>Jvzg`zA{J{pFH72r)kf92_!wdU}#Q6I)w$K*&k07R#MP z%H>SZ&I5U*5UL2XE|wqkl6caeHp#``qQ0~bSA_1e$A_O=u6{FMjKKpJDT8uO6iDpH z04bmbxwkC<&^9$ajm;;>Jd+M`KWrG2NR*9_~oy zsJ)s_bubUu<7%A}ost~kpOsisX_YVh%i1;r#Y26AA{-jWG?3lYXBrlro5(d_+vf+O z3bYxxOEU;8+}Xf-F$NG-TbHX_W15>vJk*a1_lPM=Bj*y0R@Go z_z0x+DjT>=OY`3z*72C_!BS*FYJCZe2Zwawmqq4us4jqFDDt_yW+g!v@%8NSJu)8y zQHK;M=Bn?Uk^}^-gSa2cET}n@B=6`=FCNLdYQ(f|(5Nc)YJ_Qff8yK@amHkz&EcB> z=N}W;td&;OW}`HxI`{}Jr(vkXoV~RJ`_QEvVy=NGBLS7#|- zrQWaNq|5p3ei@#G}FCYKT#4=(i8B5W`_$krBzk2zYtct{KQ~vzbyC7aPD8Q3#p>A6&3t6 z#{~aY6dYw?6!&5CX{h8SnEi|RUJA1YcroL~^_9*^%Z0rpW=M{Ub;3av#;o9OP&fq* zgtwPBHDiuLvOOR3p8%`~w^E%o`-uf0=#{ zU9p=NhU}`4jRE{uP$#D6lzATIOZ%Vk;g}gs=U@9}{dUK0q4$%l{{~%nE=3 zNs7n)9?mFXI4W?5d>tL_YuU@G_D!R9byE&>DwmU0lx&gcYr z#pvUyr+{6E^ultY9{>x2BO*`%Uf>RJU}Tu^djI%6dVwk%R5TD6+%17(CtM8Odj z;@ku&_8%U+_CB? z0Wh@77U0AU0^Sf$OPe1IefBBUs24`IuvzOQ2lU8Cw9GGmQH!iE4;O>n)_e$IAB&Oa zPkI=a1w7v`9IlQ{KO1$Bz@>;_a0t^;TY%gG;vL{B+fPTJLoe)Cg<%*OlzB`9Pf+Z! z0RvuhuVysv&Z|*8gq)zcr1a!;S=1l!G<`{l4v^b%)JPYQ6T0Kkn0Dv#1l?I&CMIvn zs%Jm&v(cLVJ^J&BBWWm^uv(iQZ=w-4^5~h(*QU&*4~g7Sag)g#e={(iKMjI|VWz-G zpKOdEt=bR%#Sy^zF#oLo$0k9WFtoN70pgIXY-#1$qN28%Mg6pi)ZY(%d|gts`~Y6B ze)?1ccnJSd{mc}I3x~y91C9~IA4ZsHKJEqll4i@XG&l$hzXOO%fdEa>rBnMBWyz21 zubw^V-vG=KRaaLxP1xf(0lOX);E*-}sR?fKERD+)p2B?{4hG@`L<*Z_Q4BKvI4;GT z6n#5bxHlXG1_1#!c$&`!EozP%h8{b~j^s&@(Zy>R$PVx@VKi7ch^D3{j4}N3$gcG# zgZCe=HFcv$7`z8Yw?pG>$AToCh@F0=1+J;55CG&70&oyH=pq?5>E-X0NM*5mJn)uQ zbv+1zu>F6`P{09#0es;gxxP2dIo_x8Hb3f_O>-eCYHER!p%~(L1LDp&&SXeIF<_)v zP$A_1N)7?3TLGiJFsH}q{GxSD{09hpDP^g}w8|L)##uAjCY;Lh*FD)wYcDH+jaUjB zoS>fe!cwrbh!DdOYsc?JgwJdF)r?R*C_)2iCFoq*Ru($Enod0dI0FXYb#ii2QB{Rq zLBud{*K&(9H+YbQ!C&gwU09*Pd=ptV!QTXejL_?M@6QD9DPqC}_mnnIh-MY_UwapY z1%hQjtdeb?NK_GwfeNFzfL9X3f9Gc>fRP@9H;IL21F&8l0Kar-#TNv90|p6zR5^7G zs_N>NIYI~s2*9u++Fw&g2XI}Bp+4v5=X7IL0w7=gjTnT>_D6lv&1$EQkhdr$AdF<4 z6YWl9R|x+(IFJB!zarByzz)U&ekf~|Y&1!7y)uu}>NJ3wh!3ZOgZTj)1~fSVr3@nI zLScL`0HW=p=H%qGG6tNq0;b-*Gb`xVfWbUM5LdI09El1Z_bq>#7`2y(QqR}qk)Qcc zlXi3$vhj%wUA~%H`&%W5m$t_r#>|vjmh*!)KW$3xDHe+5lQ$WwYoIhQm@{04$1jmawl#CscE`&_ZWZUo7~c&Ow*q3dQm zt@qrS1U)H{OB42Z09aJ?6Rd6Vim&V3_o~)?Hfg z?dj?1F$_FoSn~Td0ct@i{eh4s_PF&MPE(QY-e^ql{5#x-GvCrh3GS0n2F8sz0uEI%N>TA)GZ2$QsVIb}g|Ws*8Zt>epF5M1Y9Z0cj#Fzhd#1o$ z=mJB%c`ZI-CyKwabXyyGnZTr6w?Ay+Bvnd+pAAPe7PB89B+i2l_7 zZE~bL#?;S6NqdMp#qlj%S88~8KGxKhDJF5W1Cj6k{%#0Nw+UO7XWZ5Xg%9|+_Fc*} zmk%+)mkrQSWE7tJ)G%(&^k*S(r5=BK&A9=Pw*%)tXKI{wMw+qt!P6MT6rt1M*??pB znL$j95H~I3KYzBo-^x8&d4dd zexnH>y)l!&_TxIBF#}nN)z8eRk!YNWutTa$$t0~R@u^i6mXfvO?Sqy`{reL1?7C|W zSb?vDp3IHEIum{W5)rQ7=x=AMoO4|ikm4Qm(Ah8x`}=$Id#!l#`d%FG&Atp2rIekJ zLza+v1}926HG_WM9AJkzb@a4d#D9&@6}4PS;^It&=?)n_AA*a&b0JE(wED;V7eB>1 zA_I|A)+HQ=fxV$g)lOF+1X*zR`Rva{WAVwG_292g9n^Q)nETHTt9Dzc%cDI>=Zrf7 zjuQWU7pD23KL<`y*#WPew^IKe6I03*OguvV({ z_wR3|u)db%e1uig(2bt%yb+WXi()fM1oLJmoYNp8yJgaAS6r}P9Ea=r?SkHBJ`Xp} z{Pn$8{$gIxVxGEG@2a9^ka&u2YzW3>d)1%E{V&f1;lRq9f}iVXH=*Q64e|7DOse(7Xe@`+vkM zK)TLy`6jQh1f&0f(z8TOs<5B{9{d|4go=?73ABa6u;%&h$lk9Hz^q_^w{-x1CO+O8 z9v>e7m3Sl!NcDHk@6rGAe$@B@M=%=$R$QrQYli~S)xhhhLB0uawtNEt{qye7QcMzn zXJE}bP$6+af%a>L2o%!~0J`?CHv;!rd}d}O0LoH6o{2psYKb*8H0ZRsvIA<>wP-Xg z`8j;mHu>mz`Cvb!0Q*a~}yMy*7b*l%$~Ky*E4 zAu~-Nv4+P}be4EUyuMhKtUbq+VT=J2EUX~x`GRF7oIZ>}6FN4*nKdJ_NvK3ltk1+5 zw&f=_gYsbc4!8uR*c=n&ccHjhdQr1w!ScGxQJ!|Skne2WHpJA9DE-rcpnbKrmJk>3 z$J=C=7nPH~j0u3|#s|BAzb2o5W>YNgIQ*G_c}RN#sJ~ zaIsDnWthHx+4uJ7y}kXA@#D=r)b?BUPiAMvnHSaQZ?Zt$N%dELKNV*{+by(?UesKzM5LzL4-uHDjmtb@OT8hUa4TB&^bLR=Y+0~i*|)1ynvM)j>vJA0j3_UgLURMk4xXr(_;r8hvhFqIaq z<>A{!ITdASZ#$%?bBd0gnQ7#tALr`BZisI%N#AuufF3dFxwz94^RoRpaPzFz8a3AX9w#sr)U^-~ z-g!GXVThzl+2^F~pfql%SvkM-O)7>ANlU%f&-%bAgpEI+e`Ak@Jgg+Gt3w7=w&yOw z$L5Ju@jiKu2Z8T#__$yFiaA_wzUYS)b7a;?P)AB2To{~~yDGx`{Qqn1E2E<7!nOx# z=?-a-5)}+OBoqk|36C6zdXNxFkx&E?5Tq1o=@g|qMY@sv?t}5>uXlaV57s=h za5!_$-gjKFZ+?3PtRbBRfv;o2?`m(g-^kLXSuFo{Do&rfq!*@T;p%tb@bv_(@O-*t zB2e%tak}Y4U!N>9wS-CaSAAod?8{20{Z2-%Zn*S0Opfh86)=Rej@~9G5Awl@YDikA zH9YrepZM_{??wb16-f|)J%kQ-N#gsn^y$YRs!-QE)*2 zJ4O+ae!i5~99QeEyCx}0CtQ)sA;gNi*xlQP)vWJBRj^vBaj5vJxya4C>xAXQ74Hkt z99K>*J7loYjHuPrZjwEJ^)U!7D)s8(^Wc#Y>ncm^kmkNo7ZJX9nyn`I`4)3O_5Cud z8*?qnJL>B1JGHHa^tq{F4QTa!OxoY@j(JKzuUY65i&IrAqoH41@4c)*KA6%}6VQLi zCKf^@OL0#280n47;#kWhgyVI^>EvJLMn45;n)t0jUtNyJCwz%RQp?wotP+|x7CuRf zYHsh|cJ0fc0gt`ktnd#k75pORrm(OxKuX9`Q{fU^y%L~){aa9I&_4EuXGs)fH}OtB zMy1fODxesMtu#K|FMV(PzEYfq7*`M_>Wi;}@oc@*5PLxd#W1#HOqCZca(;Re|M8t% zaYhOU4~d2GMT7b8U-YpUDU#hj#$SJ(`{rz5>|5;nKEbi(&Zj#BFv4Z)&hio@r@mBN zc;@31QMNB`z2*Ee@^L?@4+^zGxG!0dMCKbd%Wg1yh2xRWLYm{uwd`lOn97X=IDKWr zk5TDl{F?onf`QDUIvlgF`kS$o6^zG61#wlKeefF@d~#J@z6;)*Yc$yV(C@n-E=c{^ zH~jtk*zi1i@x%xnMm`2{u9V>HE+OKyMurf3=AR-%19+u6${Qa{uJXQ1Y|TlzW79mO zgA(qMzl7mk=PMc)WO%dk$F)5A`UeavSx@Gwuf{|NUV+cY9Y2Y@ zyC#e~CiDwRv>SPiQEDmT$$m{NRSy)cX?YnQJU6`r)L2~?Rs%-TzO-QDF9R$L{&-eR zOJ&+y1SAC%s)T0iIM=!t4cb4vu-+;cQQklKj91*>^5VYclX)`OB9pOyk1#@$GM+YTs+7>Yrk0E_f3;LJK8Sbr)FRP6`O%JL zynu@dkv5ln9j5rJ+YOCom7dJT@r#->@H<`jw8TK-SbQO}a0_KI+lV7N*uOq>LeO+- zK8alS*zW^CL{0=FKNO z=H4`Fobcj=qY-`f(^foIGG~ejcYKSf-&HNQ8sy&c`Fvzj{cRZg!=-|41uwN_&tvk( z4l9*-k8wbPIb21CO=4|v4&Mp*XgE~DxN}@pDMzW3mP=VAAxBrP4&gEVSe`H?pDVkl zR!{B}(^T_}E9rS>!k4=0hq&xED4gAk>Y1h!D(Y}>*Mjo54_yMO7j$(MFZ+>FyWX*W z=fnIkJ2Xh}8oR2-fKt#dcd?E;0Y&D5N^xn~3w$p(302jao-(n1*9(DJv&1@6niTGTY9BhOUi$%-OfOT7-ZmDL9P0}z3UnSWp zC%ma<>#W$>v+A6-zdn;SWbwHW3nb;?`sD>OI`cO6@`^>Ywdo6Mk603gei!9jX-n#; zyRSLaayCr+qDZeL3+fWXiRNpC^F3c_@dvlI_^S)viSRXF#vBxSPdaZcq^gD;mQo)e zgf?2Qwd_7=$N27kptS|N>Yes_me52d8uvE>wDCXsCk=X4oM>7^M7}#{^xr!*&qW&a zqYe8@=BFMD{w4O zcqC0Q%8M~H@KAkRd?4|$=BhF%O;>jP^X2b43jT1}*;Au+aAIZiP@{p9`g2P&T>FHSpK5q1s&?VYG!}h8ZAqLl{d+*zp|$BShV7QvOh+N`!<6}ddoE)4_i*Id&X_+x?EdH zaPO-S(if*z>$*4JvUgZHan3Y-pCC|-+v_rZP-A3zN{p*rd#v=da@>qdarqKT)%j^~ zWh$wIApKd(7bkK?bMpOr^|9)e@$TIx5n$EJG@ZRbX_)ppGc-A`-D&0=U!I}i>GO3Y z4a7U~EWBa?CenAJ`qr38=I>GV7eO8tk318Dxgb2C)qnVH8KWqaMt@z1))=i;5*oa- z=0Wl#f+tzK7DGzRYj^7yi?yR`PYPJ8oJ~JFQArv4~&~`)W!b{NOI*$2w_DG=rIZUxgjw zG#E}^pwU&rq(}*Ao)D{FKsT-@d2mY3OkJ$TM_mgKA37Vt>coH{#oG3{IsrqKr=fTSt+?w9Q$;nxAxzHhNW{hde zmF3z2xt3@Me!x;bML~M_=9yXYM~@1opH0S1UZ#~awcmSEYsrXW7-60(6q0AFs#(%X z?BC;zsCk2oHfLi>;H{PA>uK6!GDuLaud-tRbpl{&WZ|jLR}5J(hvJY9@aIo+5c9ET zwBLo)963iD7AeC~@GvfDLA;0a{*^HKSlRgr#Ycn}3dGbn2NdGNhEB|IQ($q5UK6|J z>idp_CWQ=>O;tdBp5d8;TJt$Mw5d^C`Tz+t@zE0g?3pok^-;t;a0LtAvuO% zFJ=lEY(|Pg(ZQN8VOTx7Hu^G2+yzeuyC~v6bwqvHHMhYbUNljuW+HLu-*gQ8@|JV) zSLdm|A_r%IUcF1A5)`7?j1J!(xyNapXBt`BX}MiJNaXS9g@o}*MYgMc*_;^B4+Ne+ zw4-y#ZVqCzaMr&kY?^f{&Pmx)r8Bm^?_g>4nTzrx+#-jHEmYz~bkfu>Iw6=6D!6B5Ej_^xPV?eyT^UDL|P;kT(34@WxY=zxMVb;pT zb^l4W)aIolVQ`R#cEcR{CCbo}%$i=5B}**sH&$$~3C8Vw8vhM72jDzJ$KLa~vd9TgYarH`^d zX&9jp73agLkfY}==&_?!_S4k3jKYUDvNY!TR=J| z{aN_iA|CBduB3~XTB>)UsIgZm7A)YO^+|bWV}ax3mCk4r{nNvv5G|4R-n5W(PGS%U z{Gub1GUo6HJvSp@hRK#CCyVt_f(TY-D^9&K;jHK*0oM1gFyh?FKXKx2Kir;OFF3^V z)xot965CqPb_(|ivbY0QnS%MzJ2X6^nE6jPd{T5~e~7$pC*l#K$2Y_(v>lg0NTmqz zH9XWy0A+-R9Vq|{e;reN1YpX0m}mccdLi+E@T6V8$@A3obWEG_)%vGP&sl=5XHTS3 zD^Df4Wss4Z+ZTrY_(}ZoVZdEFrgMMhepl!_KS?&x&Y4%DrrgAd)yjLtI&lp%u9IW3 z@j;pzeAwPhJhJ#qF0{UK&(C;Xq9tld&}NZH_Iq9MwY6e+M`01?gaM78*Z%gh_vySpP`HN!k9P zt@y`*$v_+DPrgOW=Fb<0s}H?)+>HBUH-lPc$E|D*83{%Y7ciY(JYZ;%NyB(|@%`s# z5@}gd8J>%a!IXS!0m+FKOq?9et|FT_EMuWk=lp0nplP`v4s^9Zi}T{ZW8e{qYlC*L zeb5kLupB9;tK1u2UYW~GNqhAQ8$>LnTLTt$li@o?V5_WgpD5Aooy4m=QX(|gi7{aU;6A#2&39uuu_QWs)%_9hlNtWY&LXn|hjO@!@_ z;p2$ydD@KXC%<@#w)SlTztypcpr^qDS!e88C}S_kCtg+>ylKtq(DztIIcII}1M5h6 z6)&^%hXMhp4^NzKXW+d4({X`VYRt5uufHCPUs~Uwlc*@CBke+ujhBD}S@SdNlPz&^ zuc?_cJ3<%q$e|Xa2IU`i-;SBFwqYrE$V<$biTdg$L&+^a8Ie9%32#h#8@|@`=O^fE6Dh%B1f=ouM_wCXVp~0c(g>N%-(i@T8 zDq=*yrF?jx>E)H=@m}thpk<#Aw94W~`|I52&o>MdS=hB*?uFyVkXh&n#bVA&0mDpL zUv*VU?R-lBdxVwR0LUOBYG4D17f3c4C#S8aj0Dnzf+sp;34=R{?7Rd+)o_(GSSZui z=_%yTuD(vy&JrGc5T@fA@MoGCLsMPLG9JIdO-47yn?aWn*w{4eupN4{92D)fGnR*B zwx06v8W}XNxuxU3d1Kfyon4Q8nh&ciKjw4n8H@Q%<)^J@^&{Tj_US$!#Hi;*Mj^Gm zY>kx~dp z8SL|UB~F}F_nBT$(tK!tn~}gqFe}Q_Sx$tMWV!p}`NUIwdU_GXC{k%c{%h%KlZMg3 zG-&tk8$t#Eb??BPz;z@{@!XfJ+93ZW44G*r3)b>)E*42Hr1uPS_&%S^nf>fO7q z7WPQIcQ6t79Rw!`xa%?%DgUxJ^L(qG-508aUZeW3YLFwKy^r=r1=tLvrKM|De&RB@ zpa2}FA9n(gh*?A+%e4z|;1}4OM91+!sb`D@c&eO2h?ZqWrZXtXie7G?(_Po zqZg{Np1nQr5ZS|vFI=w%X=gG^6P&qPMFN=?mU0%W#xwROp4VTYuw z>ZYz%kvUaJXy|+M@4N`tOrvd!@t{2^_-$SVJ<4{msCVMmY|2ehrxG!h)lB~6GCkC?G_>pk6OWDu3ehU7yoI zsa0k-mdM5Fv1R@`wy)7M;P&0Q8{$B~&Xziy_0FD%|NXoCfX6~wi{nW_zP6DuS2B_> zKEDfQ7d0yjgEF>L#1~nYdPXls-inIr#7BizwCp-KW7vFdTvOgHugX0;$2_=XwXnRE z$KKNZ@e;ZD@FLC&_n2wP{jTtZ(x1cy43TmVRy(*1L&q|FSj4RM6t7a*ju-I=c?0&Y zUwAb{5x5h=yoz1((d|*(rHVc4B)6YgC?C@lPihkr6Bk&(c89-a(?5KmEEysv+p;Mg z$0D@A2GwzE2Ud9SU#z1}E70!NyZ z-)=%K$*!JGqs;ylTjMa06$zf1wGeEM1tq|=smD^$&`biQRs|^Y`1-Gv?aCuYpt9v;wD7n)+C5Ed8#!V!fmvXgT3Dw(7Jyw>NZ&G-1VCK>)1!MW&+{g$q?{hpY}o-*VS zn|@x6SibP?d~5fd?^i|YrNN$>nplh~^U$Dx^!jT{Js~aZ*z&^Xw$7=!LGkyf_=BxlgM%>AUGszWk#xxJz-nMlZt^^BpO02 zIFt8I-6E>Tzt#_%AZAuXL?itEmJkO3H_8*sq&GMq~?fvL_+p);m|MGn)Jgz^C9h zkje1rhPm2-67E}L@UE+DZYwzJmRJ9<=?wR)?iry?HQ8rcer)74Up)Wt@s(8O)SEc${PATzPqpxpTq-=dEkm#Kd;YeA`|o;6pM|-;2;`ud#HUada|jH= zW%eLNWF1VdQ)J4Cf<)?XsgR0i-^=^!V(SBkJWqLoAch9B5W0=iLoT52&?t8i$Ss?p zA9~cQ^BHXb8oHg0?wikgLB$55go4*?84A%WdEN(adyi+frd9qisn&H}EjOlU2|)99xptQuvG@TYp%;NzeTP{};C6_>HU?_t@WMwB)B!R&TQh}%Xzjh=^Z z$Vtu_$c5>kqKf9caiS_DA<1*AzjP`}3B{kObK^$%wd)sY3E{CPNM_zlK_i>V_sYu- zBNd?`7ohZor(_1s@=Fanl10i@iQX)lMIq(uf__>=q{ z&x0c1D(=@Dk?GgeP@`(PfhVnH7DqF2$9?H)YvBV7KoxSkEp!4an5fehTCuUdpQXnS zsN=V7=E)qZ#$~Q^=#U7+CZHaJ@YVn;3KmBW#%M1hcN!Yfrlh8xpsC!!2HM-UU1S!8 zf|2Zjy}Is17QMMwoty*@A+Dvpf3L<)uCA^=0Tf={;lZ9kVgJ+UXn)WrFo6av#cupD z%>-I($)|R_3b{!Tn9XFoq9avGSCJJA*P%c6^;PMwnZ&NnD&80ZKc9V%SozIJd8XRK2mrebOvAG$|D}l_l(05ICBUb+3QQh z2sZeGoU!j=Lxo7ww>X`N??w~4rhkBed>q*;TwZb6)SfbX^{p>aROw$Y_RTNBow zLPN@p_BR;p_;fx}R_;%qM<~tlI3u)g%XCeI*G9T@fA6x_w%Cx%WH^#_=;`TKQjYd< zaWE;ymO<9k`a%BKPit_~@AC>oci#{PSaTO71BF`NW&ncd&9F0ghSTKzfw4z7;1ybKK zd+o3SrJM6XWG2pNWp_Sr647IQ%euWfuXjX`v=Qejb|$3hgpeBb&3MKSV|Kabzy35; zJXl3AXc9Q9shLV<5cmM!OG!!Tb90)2KspSC2q5v=v<$1}9%6e)a+2FKbbhU32*P3S z961@+dgY#chNmvs%U=1DS;n&kCtDSMbd)b5Q=!IbzwO1b`~GJN?2_{93Na~|_QlU& z-5M#ZOh_@;Zn|5kV9W+$;0idoAEtg`0(Iq-@%b~iLSJ{)$-%o*{Vyqm@p#%R58M0i zEN^oUl`;OEF*d#kBX%6_bgYIFog zn;DvloP&1rfIC4b=&xs8U`TG3aS9ulCF~oHj-P&?=_4>C{O%e|_|drGc~i$g0KKsH zF;YVG7+VfXf3`m{GI$L0OOwj_RXMzA?4VyVAmsgbil#i$U*r!ij=3`R_xEcQng4US z7}j0+lWlB8g?wc@@<1d6Iy@4sM-_&s6b;KVLz_NSO z*;xp6C2!fi{Val6T>N)xg1`b&1HukqL5g!|WXcbB=Y>IYamfl!a9~a&^%LDZ+>Y%K~FjG!Ak;Er(gHnojM!1iisb-f9<)3eXB0wh*luCbmAl;*9z4!5Ruz~SS$^snkas#ld$}&yx@v_N-LP8!W9w{^2D7MqF){VzuPOY z08(|rme{jOrH2vT`|P<;v>;Q_Q9h7APuNr*2)=Op$+HjT2Nd?81e=otjUNUIvWM`& zM)!NW$@&1c)Q(yIj@jF~&W|2pASO8O5P`)HZ8X9AInKM9*O-u>??>Yd~_Mnr`m#Ik-Np^MMKnS zr4@))nco5%CVj!C-`wmsxLA*bHQ>&r4EFYxE15w9Vbi(>B2|~qBO4@a*l;!Q%PN`d zkxv_G<<#U%O}RJ}(p;5{2@yqodZk0pwEe4Z=zmW~uKVj(KUMC4ni`q5HTVGLX);#K zV*HzX>c~1^Dda;;j~P4_evLM~K5kmPgXAZTB6QO zLf3i5|0dh^IQrUacuLl{TE8ku#33JqL`e?qBB(xrJnb^_UOn%Fr(x2PQjV4Kr1$>> zy5CQ2(f)?vX!0n`0xKX1wtj_K%>5`Wzf60Z(C1qR(S5^n{{(eO-;Q7 z#RXDw#-7(h1886fWeL)Ljjg)Gm^b)e{XJD-e0vY0^ML7io5w5JY`Z62it-k0Cxzz- zZkmFE0+1}#I5jXh5P~N*YcYs(PqF4dDta z7&^q)2~lH+=F*=;G6-~X2;$_u+nr*&@V!Ov<6vj`qL}Q~o$gVq0Es5QpS-eurxFC|)e87Y~O2P`sf%d7O zJbCS-a{r=H-ybD;o*9KY*;Z!l^%s&F+%wt+9ZOPO2c6*?CGw;c&3(vpbvM4!D-g@R zxNiD#_tjA%dm_8l$k(H`bj*Yj%-#llj}nzvi`U}{DXo)O9Unca9=Cm2Shyba>y^Lo zS}N>4B!PjOWib*%N1l!1-#bQcX81(#FJghV9nm~pOSdZxlp*Dj{c9}1Ya3+8{D_w0 zl==Hky=t4LJvsTY%JWa6sZ%5VcU;l&_Q0Fq4*a{lf4|`;-25Ly9)52XwEe%?J-^=w zN@pO)k03>)Vajp*7@*pzMO2#OOvKU|b%+FLVZqV^fC1*ER8qM-22n>mq^p8TMF1qB zGwH$6n+Iyo&TKA5H&RFON7C zqq%O@XWxAba4^4VJ424nC+f1YvRa8te%G6!O(T5X@))Umq1YL)B?)ASx&=r|ef_bj zo`{GDDXYxQ&Gn1!2f9DRihQcdjRmHn&@U$<8vFr0nSgdY%{_oHU}-x@IbdlsNZB8G zWz4d?-KgM#l@C5O<@gKO;84r&x6~-Mq?6nk^B-rTmfmQFH5U2o+2x^HmY^8!bLZ-z z^n!y0uU-Xjf~=O-;0kdFiyQ1yXjxG{kPSE#UXBJ{q(HtL*?;+(?2)!u`48IYlU!%w z!EK4rUKY386F6tn)JyIG^BEFyZzAlL)@NuvaBB+`KBtrZFbs9Y#%2r+1)c z#z4VU)j~_5s-;CxG8m3XaG@G!%w?sJQ-qs-4bC^yREV+#0DU~>wn{bB%%a+|r|}-& znXxi5V%I)LgMv*?0sT)k^wCdD%H<4ycttc``k@=qD2Kd~d$Q|wv-gOV!e2p2i2xGm zNdkh)Ed&Gv2>y)nQOT^8C6NdKc`~@dhnxnj?#!WS)o;a@pz@)_L2fZyFbP@>T!R0>!TYzW{$m5%PLiqU>M~(T2Pug5|A$S4BM`i2W}QjLd0d{@@occN z{|w(BH(Kh85)N{-Eg=pj0F(aD&fMc|0jvRR3QgHJ!!(c_`+E_&x=&r{jcUB~u>H8b zfED%ARcXAQ9R*Kn_>=RQyltrY;Cn-jg-b<6g;XW)p%q4A5As;qBgByq03D_Wt^5s! zMWm`AN+?7k=~km}?&*KmC&zgqZN@6aoQ(pr8KMveN+4po<@p(_L#Y3h;De*8^`C-S zk;@ZIBeg>Meov80#64c;|MLl`r)= z&A-I(++JxHTh>21JV305C>Aj>Er*;m?uo;_$=)RSV$+WD_Y8}j;Q z3UPE(z_0aLBj{GlbQ%8?zp~rzNI@z$iaZHCw$|OvdAk9#%yEcm@^DN7ljlZPX$xu$ zfs+#w5)gvLu=WX2oP@g|qU$a$7DQFS84KY6>XT@TA;!$N@pI)C>0|5%JYJ z3WT-rg7R_#F_iuM{`RuxsN4KI#E+Hlb>IdD%t^#?&O-X5<<;-KMN0`W7J2;F{p3I! z9fJU23Zz&~=0)aAh}ADf5Xy$)5AbfAQ&$UO32U-0QPR!3*PWZ4t++dSbfBjY#bH+M zhy4}0+$3AR^lRZ!OkO(%Z3S>w?%NA9G|$6B)_|VwmhLiK?+F&G(Fz?%TD!2W&UzvS zh7G$Povipi9~@@eOZSomtr8qTIbEky0-ON8&$p40T)uU;&rx3tBYSdn-kP}s)~!gR z;~4qi6M}O!@AmYlH(3fQov+-t->3LJm`4B)w!`o;8CI7wH*ye;JXI+5*<-tdW^8nH zv^sO5kj<09#cS5qxer`#-XsAB9VQB~3_+9j(<^)v9#P%dk|DqI6+0)^XFD3e59I@> zx6wphaI-LS9zZR*33o8k?ZizZxVP=_quy~S@{#;#JK{QPu)8sb-;fXOi=(si z
5zw>SH9l&FbebE2kAn6x90>3@|8#EtB)AuBht%w99hPxW?)WaCXiuF88hpj0A zkcZv^t;sbTM;8?E4T>lPxm*FWqv7HHFT{>#2iIU$ZwEZpgfA{rP=9f0sjzhloB~3H zpaLbMUZp;^z5xZ(gJe+*;p2!>cines8`cF&P=z!C zr^qEB7#q}yU=e(ilM^=h0KOb>2dB?f?yr^36X9UWq^joaecYJ{J;v_RLeJHVe0E15 zNb~wt00;>Huj!n-btDYtq=uqi-Oyu;#E=~>;#h*qu=c6eN1@tELF`TlRXiYe6Clc0 z8CX7scSBC1uDH94L4nF|_BA7>;rDqG0=FRS?I2-~py|1Ccy0q2+sQ% zyBd71jPKwYK6aaYRp1&2jHm(FpJO<<$`=!&D;A1RyOb12Cf*T`^&3b-nweyO5-FWG z>_$De4IsFJGfKwG%d5r{3a4?17N2mlhfNFFx0 z!`mTLXW%h$BsmS3(MoxVAFe&GM4*VR1bjsisNWDK78B)l0Fzt6zIGZ}ktUyz z^_eao(X(;zY4m^?F)<>w4fM=VLma7jz59LU7cK9ffc4{45{JT9TM{IkH) Date: Wed, 15 Jul 2026 18:16:41 +0200 Subject: [PATCH 4/4] Add the measurement side of the batchsize accumulation aswell as the corresponding unit tests - Implemented tests for the independent measure_dataloader and measure_batch_size path. - Added validation tests for measure_batch_size and measure_dataloader parameters. - Enhanced logging behavior for single-measure and multi-measure scenarios. - Included tests for deterministic sampling with measure_subset_seed. - Verified that measure-side backward passes do not corrupt training gradients. - Ensured proper handling of partial last batches in measure loaders. --- PHASE1_batch_accumulation_review.md | 44 + PHASE2_full_project_review.md | 73 ++ examples/measurement_batch_sweep.ipynb | 725 +++++++++++++++++ examples/measurement_batch_sweep.md | 314 ++++++++ perspic/analyzer.py | 489 +++++++++--- tests/integration/test_analyzer_deployment.py | 272 +++++++ tests/unit/test_analyzer.py | 751 +++++++++++++++--- 7 files changed, 2471 insertions(+), 197 deletions(-) create mode 100644 PHASE1_batch_accumulation_review.md create mode 100644 PHASE2_full_project_review.md create mode 100644 examples/measurement_batch_sweep.ipynb create mode 100644 examples/measurement_batch_sweep.md diff --git a/PHASE1_batch_accumulation_review.md b/PHASE1_batch_accumulation_review.md new file mode 100644 index 0000000..deefabe --- /dev/null +++ b/PHASE1_batch_accumulation_review.md @@ -0,0 +1,44 @@ +# PHASE 1 — Batch-Accumulation Feature Audit + +## Data-flow trace + +Per micro-batch, [analyzer.py:262-365](perspic/analyzer.py#L262-L365) executes: `opt.zero_grad()` only when `_accumulation_count == 0` → analysis before-hook → wrapped `training_step` → `manual_backward(output / K)` → increment count → when `count >= K`: `opt.step()`, increment `_optimizer_step_count`, reset count, step per-step schedulers. + +The analysis path ([_analyze_accumulated_step](perspic/analyzer.py#L503-L584)) decides `_analysis_active` on micro-batch 0 using `effective_step`, saves/restores `p.grad` clones around the grad-clobbering analysis passes, accumulates chi metrics and unscaled linearizer gradients per micro-batch, and finalizes on micro-batch K−1 — **before** the optimizer step, consistent with the single-step path, which also measures pre-step state. Exactly K entries accumulate per cycle; **no off-by-one in the accumulation or flush ordering itself**. + +Math verified end-to-end: +- Gradient: `Σₖ ∇(Lₖ/K) = ∇(mean loss over N=K·B)` ✓ (assuming mean-reduced criterion, equal micro-batch sizes) +- `grad_norm_squared = ‖Σgₖ‖²/K² = ‖∇L_full‖²` ✓ — confirmed numerically by [test G](tests/unit/test_analyzer.py#L1248-L1307) +- `chi_net_eff = chi_loss_eff = mean over K` ✓ per the derivation in [batch_accumulation_notes.md](examples/batch_accumulation_notes.md). Note: **the old `K·sum` chi_loss bug (a K² error, 16× at K=4) is fixed only in the uncommitted working tree** — the committed branch tip still contains it. That fix needs to be committed. + +## Verdict: ⚠️ Minor issues — core math correct; one label misalignment and several edge cases + +## Correctness issues + +1. **`effective_step` is off by one against everything it's compared with** — [analyzer.py:329-334](perspic/analyzer.py#L329-L334). Cycle N (0-indexed) tags its micro-batches `effective_step = N+1`, but the same cycle's analysis metrics log `analysis_step = N` ([analyzer.py:765](perspic/analyzer.py#L765)), and the full-batch comparison run's `train_loss` lands at Lightning `step = N`. The notebook plots loss by `effective_step` and chi by `analysis_step` on shared axes, so the accumulation loss curve is shifted +1 — visible at early steps on log axes. + **Fix**: `self.log("effective_step", float(self._optimizer_step_count), ...)` — drop the `+1`. + +2. **`_accum_step_losses` is dead code** — initialized at [analyzer.py:251](perspic/analyzer.py#L251), appended at [323](perspic/analyzer.py#L323), cleared at [345-346](perspic/analyzer.py#L345-L346) and [735](perspic/analyzer.py#L735), never read. Retains detached GPU scalars for nothing. **Fix**: delete all four sites. + +3. **No guard against ragged micro-batches** — `output / K` assumes every micro-batch has exactly `micro_batch_size` samples. With `drop_last=False`, a smaller final batch makes the accumulated gradient a weighted (wrong) mean, and [analyzer.py:628](perspic/analyzer.py#L628) uses the *last* micro-batch's `x.shape[0]` for the logged `batch_size`/`effective_batch_size`. **Fix**: warn once in `training_step` when `batch[0].shape[0] != micro_batch_size`; document `drop_last=True`. + +4. **Interrupted / partial cycles**: `_accumulation_count` isn't reset at epoch end, so cycles span epoch boundaries (self-consistent — gradients and analysis both span — but undocumented); a trailing partial cycle at end of training silently drops its gradients; `on_train_epoch_end` steps epoch-interval schedulers even mid-cycle. Additionally `_optimizer_step_count`/`_accumulation_count` are **not checkpointed**, so a resumed run resets `effective_step` to 0 and misaligns `analysis_schedule`. **Fix**: `on_save_checkpoint`/`on_load_checkpoint` for both counters; document epoch-spanning behavior. + +5. **Question — mean-reduction assumption**: loss/K scaling *and* the mean-aggregation of `chi_loss` are only valid for a mean-reduced criterion (your notes acknowledge this). Nothing checks `criterion.reduction`; a sum-reduced criterion silently yields wrong gradients and a K²-wrong `chi_loss_eff`. Suggest a best-effort warning via `getattr(criterion, "reduction", "mean")`. + +6. **Question**: `effective_step` is logged even when `log_metrics=False` — intentional (training metadata, not an analysis metric)? It's the only unconditional `self.log` in the class. + +No race conditions: single-process Lightning, no threading/async anywhere in the accumulation flow. + +## Efficiency issues + +1. **Train-side linearizer passes are redundant** — [_accumulate_linearizer_grads](perspic/analyzer.py#L586-L623) adds one extra forward+backward per micro-batch, but at cycle end `p.grad` already holds `Σ∇(Lₖ)/K`, whose squared norm equals exactly the `‖Σgₖ‖²/K²` that [finalize](perspic/analyzer.py#L644-L647) computes. Reading `p.grad` just before `opt.step()` eliminates **K forward+backward passes per analysis cycle plus one full-model gradient buffer**. (Matches your existing hook-based-refactor design notes; measure side genuinely needs its own passes.) ~15% of analysis cost at 10 output dims; much more for low-output-dim models. +2. **`BatchStatSnapshot` pays a full dummy forward even with zero BatchNorm layers** — [utils.py:145-146](perspic/utils.py#L145-L146). For BN-free models: one wasted full-batch forward per analysis micro-batch. Early-out (but keep the `model.eval()` switch so dropout behavior is unchanged) — cheapest win in the feature. +3. **One GPU sync per parameter** — `.item()` inside per-parameter generator sums at [analyzer.py:644-647](perspic/analyzer.py#L644-L647), [675-682](perspic/analyzer.py#L675-L682), and [linearizer.py:78-79](perspic/calculator/linearizer.py#L78-L79), [103-107](perspic/calculator/linearizer.py#L103-L107). Sum on-device, call `.item()` once. +4. The loop itself is **O(K)** with in-place `acc.add_()` — no quadratic behavior, no unneeded copies beyond the required `saved_grads` clone (necessary because `sample_calc.compute` calls `model.zero_grad()`). Peak: ~3 concurrent full-gradient buffers (`saved_grads`, `_accum_grad_train`, `_accum_grad_measure`) — acceptable, worth a docstring note. `itertools`/`deque`/vectorization don't apply here; the buffers are parameter-shaped tensors handled correctly. + +## Best-practice suggestions + +- The train/measure branches of `_accumulate_linearizer_grads` are copy-paste duplicates; unify. +- Several tests drive `_before_training_step` by poking `_accumulation_count` directly — brittle coupling to private state; the `training_step`-driven style of `test_analysis_does_not_corrupt_training_gradients` is more robust. +- Typo "the the" in the manual-optimization warning ([analyzer.py:185](perspic/analyzer.py#L185)). diff --git a/PHASE2_full_project_review.md b/PHASE2_full_project_review.md new file mode 100644 index 0000000..ec79d9c --- /dev/null +++ b/PHASE2_full_project_review.md @@ -0,0 +1,73 @@ +# PHASE 2 — Full Project Review + +**[SEVERITY: HIGH]** [samplewise_opacus.py:287-340](perspic/calculator/samplewise_opacus.py#L287-L340) +- **Issue**: `gs_model.remove_hooks()` and `_cleanup_opacus_leftovers()` are not in the `finally` block (only `_restore_inplace_ops` is). +- **Impact**: any exception mid-computation (CUDA OOM, the `NotImplementedError` for tied params) leaves ghost-clipping hooks attached — every subsequent training forward accumulates activations and per-sample state, silently corrupting training and growing memory. +- **Fix**: +```python +gs_model = None +try: + ... + gs_model = _GhostNormFastGradientClipping(...) + ... + return total_sq_norms.sum() if reduce else total_sq_norms +finally: + if gs_model is not None: + gs_model.remove_hooks() + _cleanup_opacus_leftovers(model) + _restore_inplace_ops(inplace_states) +``` + +**[SEVERITY: MEDIUM]** [samplewise_opacus.py:150-153](perspic/calculator/samplewise_opacus.py#L150-L153) +- **Issue**: `_cleanup_opacus_leftovers` deletes `param.norm_sample`, but opacus 1.5.4 (verified against installed source of `get_norm_sample`) stores `param._norm_sample`; the current check is dead. +- **Impact**: after the final output-dim iteration, `_norm_sample` tensors stay attached to every parameter indefinitely. +- **Fix**: add `if hasattr(param, "_norm_sample"): delattr(param, "_norm_sample")`. + +**[SEVERITY: MEDIUM]** [utils.py:106-207](perspic/utils.py#L106-L207) +- **Issue**: `BatchStatSnapshot.__enter__` is not exception-safe — if the dummy forward raises, `__exit__` never runs. +- **Impact**: model left with `momentum=1.0`, BN layers in train mode, running stats clobbered, forward hooks still registered. +- **Fix**: wrap the body in `try/except`, restore saved state and remove hooks before re-raising (and add the no-BN early-out from Phase 1). + +**[SEVERITY: MEDIUM]** [samplewise_opacus.py:366-367](perspic/calculator/samplewise_opacus.py#L366-L367) +- **Issue**: `_compute_per_sample_gradient_norm_loss` runs `model(inputs)` with grad enabled, then immediately detaches the output. +- **Impact**: builds and stores the full parameter autograd graph (all activations) that is never used — wasted memory every analysis step. +- **Fix**: +```python +with torch.no_grad(): + outputs = model(inputs) +outputs = outputs.requires_grad_(True) +``` + +**[SEVERITY: MEDIUM]** [coupling.py:33](perspic/calculator/coupling.py#L33) +- **Issue**: unguarded division `grad_norm_squared / (chi_loss * chi_net)`. +- **Impact**: zero gradients → `ZeroDivisionError` (floats) or silently logged `inf`/`nan` (tensors). +- **Fix**: return `float("nan")` when the denominator is 0. + +**[SEVERITY: MEDIUM]** [logger.py:134-137](perspic/logger.py#L134-L137) +- **Issue**: every window whose tail exceeds `max_steps` is skipped — for `max_steps < base_window` even the step-0 window is dropped. +- **Impact**: `logarithmic_windows(max_steps=4)` returns an empty schedule; analysis silently never runs. +- **Fix**: warn when the resulting schedule is empty (or clamp the step-0 window). Related: overlapping windows keep their full step lists in `windows` even when `step_to_window` reassigns steps to a later window, so `window_width` overcounts. + +**[SEVERITY: MEDIUM]** [analyzer.py:239-240](perspic/analyzer.py#L239-L240) +- **Issue**: accumulation/step counters not persisted across checkpoints (see Phase 1 #4). +- **Impact**: schedule misalignment and `zero_grad` boundary desync on resume. +- **Fix**: save/restore `_optimizer_step_count` and `_accumulation_count` in `on_save_checkpoint`/`on_load_checkpoint`. + +**[SEVERITY: LOW]** — briefly: +- [analyzer.py:349-357](perspic/analyzer.py#L349-L357): scheduler stepping ignores `config.frequency`; relies on private `self._trainer`. +- [analyzer.py:315-317](perspic/analyzer.py#L315-L317): `output / K` breaks if the wrapped `training_step` returns Lightning's `{"loss": ...}` dict (pre-existing). +- [analyzer.py:415-501](perspic/analyzer.py#L415-L501): full analysis compute runs even when `log_metrics=False`, results discarded. +- [samplewise_opacus.py:178](perspic/calculator/samplewise_opacus.py#L178): Rademacher vectors hardcoded `float32` — mismatches float64/bf16 models. +- [samplewise_opacus.py:17-19](perspic/calculator/samplewise_opacus.py#L17-L19): import-time mutation of Opacus's global sampler registry affects any other Opacus user in the process. +- [mlps.py:103](examples/models/mlps.py#L103): mutable default argument `hidden_sizes=[1024, 512, 256]`. +- [lightning_modules.py:173](examples/models/lightning_modules.py#L173): `AdvancedClassificationModule` has no `criterion` attribute → incompatible with `analyzer()` (and its label smoothing wouldn't be visible to analysis regardless). +- [analyzer.py:17](perspic/analyzer.py#L17): `Optional[str]` on `sample_wise_engine` is misleading (`None` is rejected); the dynamically created `Analyzer` class also can't be unpickled / `load_from_checkpoint`-ed — worth a docstring note. +- Repo hygiene: `examples/cifar-10-python.tar.gz` (~170 MB) is untracked and **not** covered by the new `.gitignore` entries (only `cifar-10-batches-py/*` is); same for `examples/MNIST/` and `first_wrong_test_batch_accumulation.png` — one careless `git add .` away from a 170 MB commit. + +## Top 5 prioritized actions + +1. **Make the Opacus wrapper exception-safe** (`remove_hooks` + cleanup in `finally`) and fix the `_norm_sample` cleanup name — prevents silent training corruption after any analysis failure. +2. **Drop the `+1` from `effective_step`** and remove dead `_accum_step_losses` — restores x-axis alignment across runs and metrics. Also **commit the pending chi_loss `sum/K` fix**, which currently exists only in the working tree. +3. **Warn on ragged micro-batches and checkpoint the accumulation counters** — closes the remaining accumulation edge cases. +4. **`BatchStatSnapshot`: skip the dummy forward for BN-free models and make `__enter__` exception-safe** — cheapest meaningful speed + robustness win. +5. **Replace train-side `_accumulate_linearizer_grads` with a read of `p.grad` at cycle end** — K fewer forward+backward passes per analysis cycle and one fewer full-gradient buffer; first step of the hook-based refactor already sketched in your design notes. diff --git a/examples/measurement_batch_sweep.ipynb b/examples/measurement_batch_sweep.ipynb new file mode 100644 index 0000000..f58ccd1 --- /dev/null +++ b/examples/measurement_batch_sweep.ipynb @@ -0,0 +1,725 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e6c70d6a", + "metadata": {}, + "source": [ + "# Measurement Batch-Size Sweep\n", + "\n", + "Demonstrates `measure_dataloader` / `measure_batch_size` / `measure_subset_seed`: an\n", + "independent, sweepable batch size for the cross-response measurement side, decoupled\n", + "from the training batch/accumulation. See `measurement_batch_sweep.md` for the full\n", + "design.\n", + "\n", + "`measure_dataloader.batch_size` is the GPU's max single-pass capacity. Swept sizes at\n", + "or below it are measured with a single direct pass; sizes above it are gradient-\n", + "accumulated. One sweep, one training run, one plot." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ee8fd900", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:50:56.606900Z", + "iopub.status.busy": "2026-07-15T15:50:56.606823Z", + "iopub.status.idle": "2026-07-15T15:51:03.639421Z", + "shell.execute_reply": "2026-07-15T15:51:03.639180Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 7\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import pytorch_lightning as pl\n", + "import torch\n", + "import torch.nn as nn\n", + "from pytorch_lightning.loggers import CSVLogger\n", + "from torch.utils.data import DataLoader, random_split\n", + "from torchvision.datasets import MNIST\n", + "from torchvision.transforms import ToTensor\n", + "\n", + "from examples.models import ClassificationModule\n", + "from perspic import analyzer, logarithmic_windows\n", + "\n", + "pl.seed_everything(7)\n", + "\n", + "PATH_DATASETS = os.environ.get(\"PATH_DATASETS\", \".\")\n", + "NUM_WORKERS = int(os.cpu_count() / 2)" + ] + }, + { + "cell_type": "markdown", + "id": "b3ae66f4", + "metadata": {}, + "source": [ + "## Data\n", + "\n", + "A train split and a held-out measurement split, both from MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ca0205ce", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:03.640310Z", + "iopub.status.busy": "2026-07-15T15:51:03.640134Z", + "iopub.status.idle": "2026-07-15T15:51:03.679401Z", + "shell.execute_reply": "2026-07-15T15:51:03.679040Z" + } + }, + "outputs": [], + "source": [ + "mnist_full = MNIST(PATH_DATASETS, train=True, download=True, transform=ToTensor())\n", + "\n", + "# Train / measurement split -- the measurement set stays fixed and disjoint from training\n", + "generator = torch.Generator().manual_seed(42)\n", + "train_set, measurement_set = random_split(mnist_full, [55000, 5000], generator=generator)\n", + "\n", + "TRAIN_BATCH_SIZE = 64\n", + "MEASURE_MAX_SINGLE_PASS = 16 # measure_dataloader.batch_size = GPU max-capacity chunk\n", + "\n", + "train_dataloader = DataLoader(\n", + " train_set, batch_size=TRAIN_BATCH_SIZE, shuffle=True,\n", + " num_workers=NUM_WORKERS, drop_last=True,\n", + ")\n", + "measure_dataloader = DataLoader(\n", + " measurement_set, batch_size=MEASURE_MAX_SINGLE_PASS, shuffle=True,\n", + " num_workers=NUM_WORKERS, drop_last=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ce6cf252", + "metadata": {}, + "source": [ + "## Model + analyzer\n", + "\n", + "Sweep measurement batch sizes both below and above the max single-pass size (16): 4\n", + "and 8 run as a single direct pass, 32/64/128 are gradient-accumulated. A sparse\n", + "`analysis_schedule` keeps the sweep cheap -- recommended whenever sweeping more than\n", + "one size (`analyzer()` warns if you sweep without one)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "acd4baf5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:03.680423Z", + "iopub.status.busy": "2026-07-15T15:51:03.680311Z", + "iopub.status.idle": "2026-07-15T15:51:03.686139Z", + "shell.execute_reply": "2026-07-15T15:51:03.685880Z" + } + }, + "outputs": [], + "source": [ + "model = nn.Sequential(\n", + " nn.Flatten(),\n", + " nn.Linear(28 * 28, 128),\n", + " nn.ReLU(),\n", + " nn.Linear(128, 10),\n", + ")\n", + "\n", + "MEASURE_BATCH_SIZES = [4, 8, 16, 32, 64, 128]\n", + "MAX_STEPS = 200\n", + "\n", + "analyzed_model = analyzer(\n", + " lightning_module=ClassificationModule,\n", + " model=model,\n", + " lr=1e-3,\n", + " measure_dataloader=measure_dataloader,\n", + " measure_batch_size=MEASURE_BATCH_SIZES,\n", + " measure_subset_seed=0,\n", + " analysis_schedule=logarithmic_windows(\n", + " max_steps=MAX_STEPS, points_per_decade=3, base_window=1\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "73285f2a", + "metadata": {}, + "source": [ + "## Train" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "3f1a449b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:03.686909Z", + "iopub.status.busy": "2026-07-15T15:51:03.686829Z", + "iopub.status.idle": "2026-07-15T15:51:33.931293Z", + "shell.execute_reply": "2026-07-15T15:51:33.930993Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "💡 Tip: For seamless cloud uploads and versioning, try installing [litmodels](https://pypi.org/project/litmodels/) to enable LitModelCheckpoint, which syncs automatically with the Lightning model registry.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TPU available: False, using: 0 TPU cores\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "HPU available: False, using: 0 HPUs\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tikhome/jscheunemann/usr/miniconda3/envs/mast/lib/python3.13/site-packages/pytorch_lightning/trainer/configuration_validator.py:70: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.\n", + "You are using a CUDA device ('NVIDIA GeForce RTX 4090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + " | Name | Type | Params | Mode \n", + "---------------------------------------------\n", + "0 | model | Sequential | 101 K | train\n", + "---------------------------------------------\n", + "101 K Trainable params\n", + "0 Non-trainable params\n", + "101 K Total params\n", + "0.407 Total estimated model params size (MB)\n", + "5 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "83ba3e3aa3b84cc9811c806f9f4398ef", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00:0: UserWarning: Full backward hook is firing when gradients are computed with respect to module outputs since no inputs require gradients. See https://docs.pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.register_full_backward_hook for more details.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`Trainer.fit` stopped: `max_steps=200` reached.\n" + ] + } + ], + "source": [ + "trainer = pl.Trainer(\n", + " max_steps=MAX_STEPS,\n", + " accelerator=\"auto\",\n", + " devices=1,\n", + " logger=CSVLogger(save_dir=\"logs/\", name=\"measurement_batch_sweep\"),\n", + " log_every_n_steps=1,\n", + ")\n", + "trainer.fit(analyzed_model, train_dataloaders=train_dataloader)" + ] + }, + { + "cell_type": "markdown", + "id": "3404bf96", + "metadata": {}, + "source": [ + "## Sweep results\n", + "\n", + "Every analyzed step logs one `cross_*@bs{S}` metric set per swept size, in the same\n", + "row (no forward-fill needed). Take the last analyzed step and plot chi_net / chi_coup /\n", + "grad_dot_product against the swept batch size." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1604c946", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-15T15:51:33.932543Z", + "iopub.status.busy": "2026-07-15T15:51:33.932393Z", + "iopub.status.idle": "2026-07-15T15:51:34.302721Z", + "shell.execute_reply": "2026-07-15T15:51:34.302474Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABdEAAAGNCAYAAAD+VjiVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3XdYFFcXBvB3dylLRzooIgIizYaK2NCIYm+xJ9EYWyxRQ6KJsUeNSYwt1mhiixqNxsQkVuwF7BURG81GsQBKFfZ+f5Ddz5UiKLCU9/c8++jO3Jk5M+xyl7N3zpUIIQSIiIiIiIiIiIiIiCgXqaYDICIiIiIiIiIiIiIqq5hEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiIiIiIiIiIiIiCgfTKITEREREREREREREeWDSXQiIiIiIiIiIiIionwwiU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIqIyTSKRYMyYMZoOQ6VVq1bw9PQs0WPMmDEDEomkRI9BRERERESFwyQ6ERERlXvr1q2DRCKBRCLBiRMncq0XQsDe3h4SiQSdO3fWQIT0ss2bN2PRokUaO35YWBhmzJiBqKgojcVAZVN6ejpevHhRYvsPDg7GjBkzkJiYWGLHeJ39+/djyJAh8PT0hEwmQ40aNfJte/v2bfTq1QtVqlSBvr4+mjdvjsOHD+fZdunSpXBzc4Ouri6qVq2KwMBApKSklNBZEBEREZUuJtGJiIiowpDL5di8eXOu5UePHsW9e/egq6urgajoVWUhiT5z5swynUSfMmUK0tLSNB1GpXD27Fl88MEHsLa2hp6eHnR1dWFvb4+xY8fi9u3bxXqs4OBgzJw5U6NJ9M2bN2Pz5s0wMTGBnZ1dvu3u3r0LX19fnDhxAhMmTMDcuXPx/PlztGvXDseOHVNr+8UXX+CTTz6Bp6cnFi9ejHfffRdLlixBz549S/p0iIiIiEoFk+hERERUYXTs2BHbtm1DVlaW2vLNmzfD29sbNjY2Goqs+HBkZ+WgpaUFuVyu6TAqtKysLIwZMwY+Pj6Ijo7GxIkT8c8//2D79u0YNWoUjh8/Di8vLyxbtkzToRarb775BsnJyTh58iTq1q2bb7tvv/0WiYmJOHr0KL766iuMGzcOwcHBsLW1xaeffqpq9/DhQyxYsAAffPABtm3bho8//hg//vgjFi5ciP379+Off/4pjdMiIiIiKlFMohMREVGF0b9/fzx+/BhBQUGqZZmZmdi+fTsGDBiQ5zYKhQKLFi2Ch4cH5HI5rK2tMWLECDx9+lSt3c6dO9GpUyfY2dlBV1cXTk5OmDVrFrKzs9Xa3bp1C++++y5sbGwgl8tRrVo19OvXD0lJSQCAqKgoSCQSrFu3LlcsEokEM2bMUD1X1sUOCwvDgAEDUKVKFTRv3ly1fuPGjfD29oaenh7MzMzQr18/3L17V22fyvrdV65cgZ+fH/T19eHs7Izt27cDyBml7+PjAz09Pbi6uuLAgQO54rp//z4++ugjWFtbQ1dXFx4eHlizZo1amyNHjkAikeD333/HnDlzUK1aNcjlcrRp00ZtNG+rVq2wa9cuREdHq0rwFFRO4mWbNm2Cq6sr5HI5vL29c42GjY6OxqhRo+Dq6go9PT2Ym5ujd+/eaiPO161bh969ewMAWrdurYrhyJEjqjZ79uyBn58fjIyMYGxsjEaNGuV5h0NYWBhat24NfX19VK1aFd9//32hzuPFixeYOXMmXFxcIJfLYW5ujubNm6u9bl+tif7hhx+qYn318fJrJiMjA9OnT4ezs7NqRPXEiRORkZHx2rhe99rt2bMnGjRooLZNly5dIJFI8Pfff6uWnT59GhKJBHv27FEtS0xMxPjx42Fvbw9dXV04Ozvju+++g0KhUNtfYd+PNWrUQOfOnbF//37Uq1cPcrkc7u7u2LFjx2vPU2nw4MHYvHkzdu/ejWPHjuGzzz5D586d0bNnT0yaNAkXL17EypUr8fnnn2PlypWF2ueSJUvg4eEBfX19VKlSBQ0bNlS9dmbMmIEJEyYAABwdHVU/v5dfn0V5T58/fx5NmzaFnp4eHB0dCx2jnZ0dtLW1X9vu+PHjqF+/PlxdXVXL9PX10bVrV1y4cAG3bt0CAISEhCArKwv9+vVT2175fMuWLYWKi4iIiKgs09J0AERERETFpUaNGvD19cVvv/2GDh06AMhJiCYlJaFfv3748ccfc20zYsQIrFu3DoMHD8bYsWMRGRmJpUuX4uLFizh58qQq2bRu3ToYGhoiMDAQhoaGOHToEKZNm4bk5GTMmzcPQE7CPiAgABkZGfjkk09gY2OD+/fv499//0ViYiJMTEze6Lx69+4NFxcXfPPNNxBCAADmzJmDqVOnok+fPhg6dCgSEhKwZMkStGzZEhcvXoSpqalq+6dPn6Jz587o168fevfujRUrVqBfv37YtGkTxo8fj48//hgDBgzAvHnz0KtXL9y9exdGRkYAgLi4ODRp0kQ1uaelpSX27NmDIUOGIDk5GePHj1eL9dtvv4VUKsXnn3+OpKQkfP/993jvvfdw+vRpAMDkyZORlJSEe/fuYeHChQAAQ0PD116Do0ePYuvWrRg7dix0dXWxfPlytG/fHmfOnFFN8nn27FkEBwejX79+qFatGqKiorBixQq0atUKYWFh0NfXR8uWLTF27Fj8+OOP+Oqrr+Dm5gYAqn/XrVuHjz76CB4eHpg0aRJMTU1x8eJF7N27V+2LmKdPn6J9+/bo2bMn+vTpg+3bt+OLL76Al5eX6rWXnxkzZmDu3LkYOnQoGjdujOTkZJw7dw4XLlxA27Zt89xmxIgR8Pf3V1u2d+9ebNq0CVZWVgByEtBdu3bFiRMnMHz4cLi5ueHq1atYuHAhbt68ib/++ivfmArz2m3RogV27tyJ5ORkGBsbQwiBkydPQiqV4vjx4+jatSuAnOSrVCpFs2bNAACpqanw8/PD/fv3MWLECFSvXh3BwcGYNGkSHj58qFbap7DvRyAn6d+3b198/PHHGDRoENauXYvevXtj7969+V5HpV9//RV//vknTp8+DQ8PDwA5cyekpKSoXo+PHj3CBx98AAsLC/Tu3RsdOnSAg4NDvvtcvXo1xo4di169emHcuHFIT0/HlStXcPr0aQwYMAA9e/bEzZs38dtvv2HhwoWwsLAAAFhaWgIo+nu6Y8eO6NOnD/r374/ff/8dI0eOhI6ODj766KMCz72wMjIyUKVKlVzL9fX1AQDnz5+Hi4uL6gsaPT29fNsRERERlXuCiIiIqJxbu3atACDOnj0rli5dKoyMjERqaqoQQojevXuL1q1bCyGEcHBwEJ06dVJtd/z4cQFAbNq0SW1/e/fuzbVcub+XjRgxQujr64v09HQhhBAXL14UAMS2bdvyjTUyMlIAEGvXrs21DoCYPn266vn06dMFANG/f3+1dlFRUUImk4k5c+aoLb969arQ0tJSW+7n5ycAiM2bN6uWhYeHCwBCKpWKU6dOqZbv27cvV2xDhgwRtra24tGjR2rH6tevnzAxMVFdl8OHDwsAws3NTWRkZKjaLV68WAAQV69eVS3r1KmTcHBwyPca5XVdAIhz586plkVHRwu5XC569OihWpbXzygkJEQAEBs2bFAt27ZtmwAgDh8+rNY2MTFRGBkZCR8fH5GWlqa2TqFQqP6vvKYv7zMjI0PY2NiId99997XnU7duXbXXYV6UP/v83Lp1S5iYmIi2bduKrKwsIYQQv/76q5BKpeL48eNqbVeuXCkAiJMnT+a7v8K8ds+ePSsAiN27dwshhLhy5YoAIHr37i18fHxU7bp27Srq16+vej5r1ixhYGAgbt68qba/L7/8UshkMhETEyOEKNr70cHBQQAQf/zxh2pZUlKSsLW1VTt2XhQKhXB0dBSLFi1SLdu5c6ews7MTAET16tVV74XIyEghhBA9evQQX331VYH77datm/Dw8Ciwzbx589T2q/Qm7+n58+erlmVkZIh69eoJKysrkZmZWWAMLyvovdilSxdhamoqkpOT1Zb7+voKAOKHH34QQghx/vx5AUDMmjVLrZ3y52ZoaFjoeIiIiIjKKpZzISIiogqlT58+SEtLw7///otnz57h33//zbeUy7Zt22BiYoK2bdvi0aNHqoe3tzcMDQ1x+PBhVduXR1k+e/YMjx49QosWLZCamorw8HAAUI0037dvH1JTU4vtnD7++GO15zt27IBCoUCfPn3U4raxsYGLi4ta3EDOSO+XSy24urrC1NQUbm5u8PHxUS1X/j8iIgJAzsjcP/74A126dIEQQu1YAQEBSEpKwoULF9SONXjwYOjo6Kiet2jRQm2fb8rX1xfe3t6q59WrV0e3bt2wb98+VUmdl39GL168wOPHj+Hs7AxTU9NcceYlKCgIz549w5dffpmrHvnLpVWAnGv6/vvvq57r6OigcePGhTpPU1NTXLt2TVUOo6hSUlLQo0cPVKlSBb/99htkMhmAnNezm5sbateurfazeueddwAg1+viZYV57davXx+GhoaqMjrHjx9HtWrVMHDgQFy4cAGpqakQQuDEiROqn7syrhYtWqBKlSpqcfn7+yM7O1u1v6K8H4GcsiQ9evRQPTc2NsbAgQNx8eJFxMbG5nuu58+fR3x8PIYMGQIgp1xR//790bhxY/zxxx/49NNPc43m7t69u1rJn7yYmpri3r17OHv2bIHt8lLU97SWlhZGjBiheq6jo4MRI0YgPj6+2EZ+jxw5EomJiejbty8uXryImzdvYvz48Th37hwAqCa+bdCgAXx8fPDdd99h7dq1iIqKwp49ezBixAhoa2tzglwiIiKqEFjOhYiIiCoUS0tL+Pv7Y/PmzUhNTUV2djZ69eqVZ9tbt24hKSlJVQ7jVfHx8ar/X7t2DVOmTMGhQ4eQnJys1k5ZM9rR0RGBgYFYsGABNm3ahBYtWqBr1654//3337iUi3K/r8YthICLi0ue7V+td1ytWrVcSWATExPY29vnWgZAVX86ISEBiYmJWLVqFVatWpXnsV6+RkBOcvtlynIQr9a0Lqq8zrVWrVpITU1FQkICbGxskJaWhrlz52Lt2rW4f/++qvQN8P+fUUHu3LkDAKryMAXJ65pWqVIFV65cUT1/NZFrYmICPT09fP311+jWrRtq1aoFT09PtG/fHh988AHq1Knz2uMCwLBhw3Dnzh0EBwfD3NxctfzWrVu4fv26qjzIq179Wb2sMK9dmUwGX19fHD9+HEBOEr1FixZo3rw5srOzcerUKVhbW+PJkydqSfRbt27hypUrr42rKO9HAHB2ds71M6hVqxaAnLkH8ptI+Pz582jYsKGqbMumTZtQtWpVbN++XfWFhKmpKQYPHqzaxtraGgkJCXnuT+mLL77AgQMH0LhxYzg7O6Ndu3YYMGCAqqxNQYr6nrazs4OBgYHaspfPvUmTJq895ut06NABS5YswZdffqmqhe/s7Iw5c+Zg4sSJamWY/vjjD/Tt21f15YNMJkNgYCCOHj2KGzduvHUsRERERJrGJDoRERFVOAMGDMCwYcMQGxuLDh06qNUSfplCoYCVlRU2bdqU53pl0i8xMRF+fn4wNjbG119/DScnJ8jlcly4cAFffPGF2uSI8+fPx4cffoidO3di//79GDt2LObOnYtTp07lmXhVenWC0pe9WmtYoVCoJm5UJv1e9mqN8bzaFLRcmXxWntf777+PQYMG5dn21cTv6/ZZkj755BOsXbsW48ePh6+vL0xMTCCRSNCvX79cE1i+rcKcp62trdq6tWvX4sMPP0TLli1x584d1Wvk559/xsKFC7Fy5UoMHTq0wOMuXrwYv/32GzZu3Ih69eqprVMoFPDy8sKCBQvy3PbVL01e9brXLgA0b94cc+bMQXp6Oo4fP47JkyfD1NQUnp6eOH78OKytrQFALYmuUCjQtm1bTJw4Mc/jKpO/hX0/vq3Hjx/Dzs5O9TwqKgr169dX+5k2btxYbZu7d++qfWGRFzc3N9y4cQP//vsv9u7diz/++APLly/HtGnTMHPmzAK3Lep7urSMGTMGgwcPxpUrV6Cjo4N69erhl19+AfD/nxsAVK1aFSdOnMCtW7cQGxsLFxcX2NjYwM7OTq0dERERUXnFJDoRERFVOD169MCIESNw6tQpbN26Nd92Tk5OOHDgAJo1a5YrUf2yI0eO4PHjx9ixYwdatmypWh4ZGZlney8vL3h5eWHKlCkIDg5Gs2bNsHLlSsyePVs1MjsxMVFtm+jo6EKfn5OTE4QQcHR0LNEElaWlJYyMjJCdnZ1rUsu3kd8XCQXJq/TJzZs3oa+vr0qubt++HYMGDcL8+fNVbdLT03Nd6/yO7+TkBAAIDQ2Fs7NzkWN8VVBQkNpz5QSWAGBmZobBgwdj8ODBeP78OVq2bIkZM2YUmEQ/fvw4Pv/8c4wfPx7vvfdenvFfvnwZbdq0eaNrDBT82gVykuOZmZn47bffcP/+fVWyvGXLlqokeq1atVTJdGVcz58/f+1rqLDvR6Xbt29DCKF2rjdv3gSQM8lwfoyNjdXuTLCxscGZM2fU2rxclkcIgV9++aVQ7wEDAwP07dsXffv2RWZmJnr27Ik5c+Zg0qRJkMvlBb72ivKefvDgAVJSUtRGoxfm3N+EgYEBfH19Vc8PHDgAPT29PEfYu7i4qEbTh4WF4eHDh/jwww+LNR4iIiIiTWBNdCIiIqpwDA0NsWLFCsyYMQNdunTJt12fPn2QnZ2NWbNm5VqXlZWlSr4qR4a+PMo4MzMTy5cvV9smOTkZWVlZasu8vLwglUqRkZEBICeBZ2FhoaoDrfTqvgrSs2dPyGQyzJw5M9cIbyEEHj9+XOh9FUQmk+Hdd9/FH3/8gdDQ0FzrX1feIj8GBgaFKq/yspCQELW65nfv3sXOnTvRrl071c9HJpPluh5LlizJNcpfmXh8Nbnerl07GBkZYe7cuUhPT1db9yYj6f39/dUeypHpr/58DA0N4ezsrHqN5OXhw4fo06cPmjdvjnnz5uXZpk+fPrh//z5Wr16da11aWhpSUlLy3X9hXrtATt18bW1tfPfddzAzM1N9MdCiRQucOnUKR48eVRuFrowrJCQE+/bty3XcxMRE1XEL+35UevDgAf7880+1c9iwYQPq1auXbykXIGfE+NmzZ1V3J3Tr1g0XL17EtGnTEBERgePHj2PChAkAgIsXL+Ldd9/FvXv3MG7cuHz3CeT+uero6MDd3R1CCLx48QJA/q+9or6ns7Ky8NNPP6meZ2Zm4qeffoKlpaXa3AHFLTg4GDt27MCQIUMKLFGlUCgwceJE6Ovr55rTgYiIiKg84kh0IiIiqpDyKz/yMj8/P4wYMQJz587FpUuX0K5dO2hra+PWrVvYtm0bFi9ejF69eqFp06aoUqUKBg0ahLFjx0IikeDXX3/Nlew6dOgQxowZg969e6NWrVrIysrCr7/+qkpGKw0dOhTffvsthg4dioYNG+LYsWOqUaSF4eTkhNmzZ2PSpEmIiopC9+7dYWRkhMjISPz5558YPnw4Pv/888JfrAJ8++23OHz4MHx8fDBs2DC4u7vjyZMnuHDhAg4cOIAnT54UeZ/e3t7YunUrAgMD0ahRIxgaGhb4ZQeQU6c8ICAAY8eOha6urupLh5fLZHTu3Bm//vorTExM4O7ujpCQEBw4cCBXGY569epBJpPhu+++Q1JSEnR1dfHOO+/AysoKCxcuxNChQ9GoUSMMGDAAVapUweXLl5Gamor169cX+Vzz4u7ujlatWsHb2xtmZmY4d+4ctm/fjjFjxuS7zdixY5GQkICJEydiy5Ytauvq1KmDOnXq4IMPPsDvv/+Ojz/+GIcPH0azZs2QnZ2N8PBw/P7779i3bx8aNmyY5/4L+9rV19eHt7c3Tp06hS5duqhGVrds2RIpKSlISUnJlUSfMGEC/v77b3Tu3BkffvghvL29kZKSgqtXr2L79u2IioqChYVFod+PSrVq1cKQIUNw9uxZWFtbY82aNYiLi8PatWsLvP7NmzdHZmYm/v77b3Tv3h1169bF7NmzMWXKFMyaNQtaWlqYP38+xo0bh549e6Jdu3Y4duwYLCwsCtxvu3btYGNjg2bNmsHa2hrXr1/H0qVL0alTJxgZGQGAKsE9efJk9OvXD9ra2ujSpUuR39N2dnb47rvvEBUVhVq1amHr1q24dOkSVq1alat++quuXLmCv//+G0DOaP6kpCTVnQZ169ZVvRejo6PRp08fdO3aFTY2Nrh27RpWrlyJOnXq4JtvvlHb57hx45Ceno569erhxYsX2Lx5M86cOYP169fnmieBiIiIqFwSREREROXc2rVrBQBx9uzZAts5ODiITp065Vq+atUq4e3tLfT09ISRkZHw8vISEydOFA8ePFC1OXnypGjSpInQ09MTdnZ2YuLEiWLfvn0CgDh8+LAQQoiIiAjx0UcfCScnJyGXy4WZmZlo3bq1OHDggNrxUlNTxZAhQ4SJiYkwMjISffr0EfHx8QKAmD59uqrd9OnTBQCRkJCQ5/n88ccfonnz5sLAwEAYGBiI2rVri9GjR4sbN26o2vj5+QkPD49CXwsAYvTo0WrL4uLixOjRo4W9vb3Q1tYWNjY2ok2bNmLVqlWqNocPHxYAxLZt29S2jYyMFADE2rVrVcueP38uBgwYIExNTQUA4eDgkOf5vRrTxo0bhYuLi9DV1RX169dXXXelp0+fisGDBwsLCwthaGgoAgICRHh4uHBwcBCDBg1Sa7t69WpRs2ZNIZPJ1H6GQgjx999/i6ZNmwo9PT1hbGwsGjduLH777TfV+vyu6aBBg157LkIIMXv2bNG4cWNhamoq9PT0RO3atcWcOXNEZmamqo3yZ//yMQHk+Xj5NZOZmSm+++474eHhIXR1dUWVKlWEt7e3mDlzpkhKSso3psK+doUQYsKECQKA+O6779SWOzs7CwDizp07ubZ59uyZmDRpknB2dhY6OjrCwsJCNG3aVPzwww9q5y1E4d6Pytfvvn37RJ06dYSurq6oXbt2rtdffqZPny5q1qwpnjx5olp2//59cezYMREbGyuEEOLEiRMiPj6+UPsTQoiffvpJtGzZUpibmwtdXV3h5OQkJkyYkOu6z5o1S1StWlVIpVIBQERGRqrWFeU9fe7cOeHr6yvkcrlwcHAQS5cuLVScyt+XeT1efp88efJEdOvWTdjY2AgdHR3h6OgovvjiC5GcnJznPuvWrSsMDAyEkZGRaNOmjTh06FChrx0RERFRWScRohRmeSIiIiIiIiomNWrUgKenJ/7999832j49PR3NmjWDTCbDzp07c00Cq7R9+3b06NEj34lkNaFVq1Z49OhRniWWiIiIiKhksCY6ERERERFVKnK5HLt374ZEIoGrqyu++OILHDt2DNHR0QgPD8eGDRvg6+uLQYMGqdXiJyIiIqLKiTXRiYiIiIio0rG2tsbx48exdOlSLF26FN9//71qnVwuR48ePbBhwwa4uLhoMEoiIiIiKguYRCciIiIiokpJR0cHgYGBCAwMRFRUFO7fvw+5XA43Nzfo6+trOjwiIiIiKiNYE52IiIiIiIiIiIiIKB+siU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiIiIiIiIiIiIiCgfTKITEREREREREREREeWDSXQiIiIiIiIiIiIionwwiU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiIiIiIiIiIiIiCgfTKITEREREREREREREeWDSXQiIiIiIiIiIiIionwwiU5ERERERERERERElA8m0YmIiIiIiIiIiIiI8sEkOhERERERERERERFRPphEJyIiIiIiIiIiIiLKB5PoRERERERERERERET5YBKdiN5aq1at4Onp+dp2UVFRkEgkWLduXckHRURERLmwzyYiIipb1q1bB4lEgqioqLfe14wZMyCRSN4+qHKoRo0a+PDDDzUdBlVgTKITUaUTHByMGTNmIDExUdOhEBEREREREWnc8uXL+eV5ATZv3oxFixZpOgzSIC1NB0BElYeDgwPS0tKgra2t0TiCg4Mxc+ZMfPjhhzA1NdVoLERERGVRWemziYiIqHQsX74cFhYWHM2dj82bNyM0NBTjx4/XdCikIRyJTlRGpKSkaDqEEieRSCCXyyGTyTQdChER0Rtjn01ERFQ0FbHvrIjnpAm8jlReMIlOVELu37+PIUOGwM7ODrq6unB0dMTIkSORmZmpqnl29OhRjBo1ClZWVqhWrZpq2+XLl8PDwwO6urqws7PD6NGjc5UeuXXrFt59913Y2NhALpejWrVq6NevH5KSklRtgoKC0Lx5c5iamsLQ0BCurq746quvinwue/bsgZ+fH4yMjGBsbIxGjRph8+bNudqFhYWhdevW0NfXR9WqVfH999+rrX+T+qpHjhyBRCLB77//jjlz5qBatWqQy+Vo06YNbt++nav96dOn0b59e5iYmEBfXx9+fn44efKkav2MGTMwYcIEAICjoyMkEkmx1Z8jIqLyiX128fTZAJCeno4ZM2agVq1akMvlsLW1Rc+ePXHnzh1Vm5SUFHz22Wewt7eHrq4uXF1d8cMPP0AIUajjSyQSzJgxQ/VcWf81PDwcffr0gbGxMczNzTFu3Dikp6cXKX4iIiqcitJ3KhQKzJgxA3Z2dtDX10fr1q0RFhaWq752QecUHR2NUaNGwdXVFXp6ejA3N0fv3r3z/Bvz2rVreOedd6Cnp4dq1aph9uzZUCgURYpZ6cSJE2jUqBHkcjmcnJzw008/5dkuKysLs2bNgpOTE3R1dVGjRg189dVXyMjIULWpUaMGrl27hqNHj6r+Rm7VqlWhY5FIJBgzZgw2bdoEV1dXyOVyeHt749ixY2rtlH12WFgYBgwYgCpVqqB58+aFjhMAhBCYPXs2qlWrpvqZXbt2LVdM+dWHz68GfUGfoVq1aoVdu3YhOjpadX1q1KhR6OtDFQPLuRCVgAcPHqBx48ZITEzE8OHDUbt2bdy/fx/bt29Hamqqqt2oUaNgaWmJadOmqb59nTFjBmbOnAl/f3+MHDkSN27cwIoVK3D27FmcPHkS2trayMzMREBAADIyMvDJJ5/AxsYG9+/fx7///ovExESYmJjg2rVr6Ny5M+rUqYOvv/4aurq6uH37tlpCuTDWrVuHjz76CB4eHpg0aRJMTU1x8eJF7N27FwMGDFC1e/r0Kdq3b4+ePXuiT58+2L59O7744gt4eXmhQ4cOb31Nv/32W0ilUnz++edISkrC999/j/feew+nT59WtTl06BA6dOgAb29vTJ8+HVKpFGvXrsU777yD48ePo3HjxujZsydu3ryJ3377DQsXLoSFhQUAwNLS8q1jJCKi8od9dvH12dnZ2ejcuTMOHjyIfv36Ydy4cXj27BmCgoIQGhoKJycnCCHQtWtXHD58GEOGDEG9evWwb98+TJgwAffv38fChQvf+Ph9+vRBjRo1MHfuXJw6dQo//vgjnj59ig0bNrzxPomIKLeK1HdOmjQJ33//Pbp06YKAgABcvnwZAQEB+X4Jm9c5nT17FsHBwejXrx+qVauGqKgorFixAq1atUJYWBj09fUBALGxsWjdujWysrLw5ZdfwsDAAKtWrYKenl6RfwZXr15Fu3btYGlpiRkzZiArKwvTp0+HtbV1rrZDhw7F+vXr0atXL3z22Wc4ffo05s6di+vXr+PPP/8EACxatAiffPIJDA0NMXnyZADIc18FOXr0KLZu3YqxY8dCV1cXy5cvR/v27XHmzJlck5r37t0bLi4u+Oabb1RfohcmTgCYNm0aZs+ejY4dO6Jjx464cOEC2rVrh8zMzCLF+7LXfYaaPHkykpKScO/ePdVnFUNDwzc+HpVTgoiK3cCBA4VUKhVnz57NtU6hUIi1a9cKAKJ58+YiKytLtS4+Pl7o6OiIdu3aiezsbNXypUuXCgBizZo1QgghLl68KACIbdu25RvDwoULBQCRkJDwxueRmJgojIyMhI+Pj0hLS8t1Hkp+fn4CgNiwYYNqWUZGhrCxsRHvvvuuallkZKQAINauXVvoGA4fPiwACDc3N5GRkaFavnjxYgFAXL16VRWPi4uLCAgIUIstNTVVODo6irZt26qWzZs3TwAQkZGRhY6DiIgqJvbZxddnr1mzRgAQCxYsyLVOGcNff/0lAIjZs2erre/Vq5eQSCTi9u3brz0+ADF9+nTV8+nTpwsAomvXrmrtRo0aJQCIy5cvF/ociIjo9SpK3xkbGyu0tLRE9+7d1ZbPmDFDABCDBg1SLcvvnITI+ZvzVSEhIbn62/HjxwsA4vTp06pl8fHxwsTEpMh/n3bv3l3I5XIRHR2tWhYWFiZkMpl4OdV36dIlAUAMHTpUbfvPP/9cABCHDh1SLfPw8BB+fn6FjuFlAAQAce7cOdWy6OhoIZfLRY8ePVTLlH12//791bYvbJzK11CnTp3UPt989dVXuX5mymO9SvmzVF7vwn6G6tSpk3BwcCjcBaEKieVciIqZQqHAX3/9hS5duqBhw4a51r98O9GwYcPUao0eOHAAmZmZGD9+PKRSqVo7Y2Nj7Nq1CwBgYmICANi3b5/aN/0vU06YuXPnzje+PSwoKAjPnj3Dl19+Cblcnu95ADnfwr7//vuq5zo6OmjcuDEiIiLe6NivGjx4MHR0dFTPW7RoAQCq/V+6dAm3bt3CgAED8PjxYzx69AiPHj1CSkoK2rRpg2PHjr3xdSAiooqJfXaO4uqz//jjD1hYWOCTTz7JtU4Zw+7duyGTyTB27Fi19Z999hmEENizZ88bH3/06NFqz5Vx7N69+433SURE6ipS33nw4EFkZWVh1KhRasvz6sfyOycAaiPJX7x4gcePH8PZ2Rmmpqa4cOGCat3u3bvRpEkTNG7cWLXM0tIS7733XpHizs7Oxr59+9C9e3dUr15dtdzNzQ0BAQFqbZV9YGBgoNryzz77DABU17w4+Pr6wtvbW/W8evXq6NatG/bt24fs7Gy1th9//PEbxal8DX3yySdqr7W3meyzKJ+hqHJjEp2omCUkJCA5OTnX7Up5cXR0VHseHR0NAHB1dVVbrqOjg5o1a6rWOzo6IjAwED///DMsLCwQEBCAZcuWqdWH69u3L5o1a4ahQ4fC2toa/fr1w++//16kDxjK+qWFOZdq1arl6mCqVKmCp0+fFvp4BXn5w4Fy3wBU+7916xYAYNCgQbC0tFR7/Pzzz8jIyFC7PkREROyz/684+uw7d+7A1dUVWlr5V4yMjo6GnZ0djIyM1Ja7ubmp1r8pFxcXtedOTk6QSqWc94SIqBhVpL5TeTxnZ2e15WZmZqq/N193TgCQlpaGadOmqeb6sLCwgKWlJRITE9Vijo6OztVXAbmvx+skJCQgLS2tUPuKjo6GVCrNdY42NjYwNTV9q373VXnFU6tWLaSmpiIhIUFteV6vjcLEqfz31WNZWlrm+zN7naJ8hqLKjUl0Ig16k9pnSvPnz8eVK1fw1VdfIS0tDWPHjoWHhwfu3bun2vexY8dw4MABfPDBB7hy5Qr69u2Ltm3b5voWuDi8+m28knhpkrCS3L/yw9K8efMQFBSU54M1y4iI6E2xzy5d+Y38Ksr14OgxIiLNqkh9p1Je5/TJJ59gzpw56NOnD37//Xfs378fQUFBMDc3LzN3Q5e1PjG/10ZxxlkcnyWIXsYkOlExs7S0hLGxMUJDQ4u8rYODAwDgxo0basszMzMRGRmpWq/k5eWFKVOm4NixYzh+/Dju37+PlStXqtZLpVK0adMGCxYsQFhYGObMmYNDhw7h8OHDhYrHyckJAN7oXEqbMlZjY2P4+/vn+dDW1gZQ9j5AEBGRZrDPLl5OTk64ceMGXrx4kW8bBwcHPHjwAM+ePVNbHh4erloP/P+Os8TERLV2BY2YU96VpnT79m0oFArUqFGjsKdARESvUZH6TuXxbt++rbb88ePHRbo7a/v27Rg0aBDmz5+PXr16oW3btmjevHmuPszBwSFXXwXkvh6vY2lpCT09vULty8HBAQqFIlfbuLg4JCYmql3zt/07Oa94bt68CX19fVhaWha4bWHjVP77aruEhIRcP7PCfpYo7Gco5hGISXSiYiaVStG9e3f8888/OHfuXK71BY3y8vf3h46ODn788Ue1dr/88guSkpLQqVMnAEBycjKysrLUtvXy8oJUKkVGRgYA4MmTJ7n2X69ePQBQtXmddu3awcjICHPnzs01O3lZGq0GAN7e3nBycsIPP/yA58+f51r/8u1jBgYGAHJ3pkREVLmwzy5e7777Lh49eoSlS5fmWqeMoWPHjsjOzs7VZuHChZBIJOjQoQOAnC/FLSwscOzYMbV2y5cvz/f4y5YtU3u+ZMkSAFDtk4iI3l5F6jvbtGkDLS0trFixQm15Xv1YQWQyWa7zXrJkSa4Rzx07dsSpU6dw5swZ1bKEhARs2rSpyMcLCAjAX3/9hZiYGNXy69evY9++fbmOCQCLFi1SW75gwQIAUF1zIOfv5Lf5GzkkJEStBvzdu3exc+dOtGvXLt+74Ioap3Jw3JIlS9Su+avbAf9Pjr/8WSIlJQXr169Xa1fYz1AGBgYsEVvJ5V+wkIje2DfffIP9+/fDz88Pw4cPh5ubGx4+fIht27bhxIkT+W5naWmJSZMmYebMmWjfvj26du2KGzduYPny5WjUqJFqErBDhw5hzJgx6N27N2rVqoWsrCz8+uuvkMlkePfddwEAX3/9NY4dO4ZOnTrBwcEB8fHxWL58OapVq4bmzZsX6jyMjY2xcOFCDB06FI0aNcKAAQNQpUoVXL58Gampqbk6H02SSqX4+eef0aFDB3h4eGDw4MGoWrUq7t+/j8OHD8PY2Bj//PMPAKgmO5k8eTL69esHbW1tdOnSRZVcJyKiyoN9dvEZOHAgNmzYgMDAQJw5cwYtWrRASkoKDhw4gFGjRqFbt27o0qULWrdujcmTJyMqKgp169bF/v37sXPnTowfP171By8ADB06FN9++y2GDh2Khg0b4tixY7h582a+x4+MjETXrl3Rvn17hISEYOPGjRgwYADq1q1b4udORFSZVJS+09raGuPGjcP8+fNV/cfly5exZ88eWFhYFHrkcefOnfHrr7/CxMQE7u7uCAkJwYEDB2Bubq7WbuLEifj111/Rvn17jBs3DgYGBli1ahUcHBxw5cqVQh1LaebMmdi7dy9atGiBUaNGISsrC0uWLIGHh4favurWrYtBgwZh1apVSExMhJ+fH86cOYP169eje/fuaN26taqtt7c3VqxYgdmzZ8PZ2RlWVlZ45513Ch2Tp6cnAgICMHbsWOjq6qq++J45c+Zrty1snJaWlvj8888xd+5cdO7cGR07dsTFixdVP7OXtWvXDtWrV8eQIUMwYcIEyGQyrFmzBpaWlmpfPhT2M5S3tze2bt2KwMBANGrUCIaGhujSpUuhrw9VAIKISkR0dLQYOHCgsLS0FLq6uqJmzZpi9OjRIiMjQ6xdu1YAEGfPns1z26VLl4ratWsLbW1tYW1tLUaOHCmePn2qWh8RESE++ugj4eTkJORyuTAzMxOtW7cWBw4cULU5ePCg6Natm7CzsxM6OjrCzs5O9O/fX9y8ebPI5/L333+Lpk2bCj09PWFsbCwaN24sfvvtN9V6Pz8/4eHhkWu7QYMGCQcHB9XzyMhIAUCsXbu20Mc+fPiwACC2bdumtjy/fV28eFH07NlTmJubC11dXeHg4CD69OkjDh48qNZu1qxZomrVqkIqlQoAIjIystAxERFRxcI+u3j6bCGESE1NFZMnTxaOjo5CW1tb2NjYiF69eok7d+6o2jx79kx8+umnws7OTmhrawsXFxcxb948oVAocu1ryJAhwsTERBgZGYk+ffqI+Ph4AUBMnz5d1W769OkCgAgLCxO9evUSRkZGokqVKmLMmDEiLS2tSPETEVHhVJS+MysrS0ydOlXY2NgIPT098c4774jr168Lc3Nz8fHHH6vaFXROT58+FYMHDxYWFhbC0NBQBAQEiPDwcOHg4CAGDRqk1vbKlSvCz89PyOVyUbVqVTFr1izxyy+/vNHfpEePHhXe3t5CR0dH1KxZU6xcuVLVJ77sxYsXYubMmaq+2d7eXkyaNEmkp6ertYuNjRWdOnUSRkZGAoDw8/MrdCwAxOjRo8XGjRuFi4uL0NXVFfXr1xeHDx9Wa6eMLyEhIdc+Chtndna2mDlzprC1tRV6enqiVatWIjQ0NM/rff78eeHj4yN0dHRE9erVxYIFC1Q/y1ev9+s+Qz1//lwMGDBAmJqaCgBqn5uocpAIUcZqMhARERERERXSjBkzMHPmTCQkJOQahUZERFRUiYmJqFKlCmbPno3JkydrOpxyQSKRYPTo0UUuhUNUnrAmOhERERERERERVTppaWm5linra7dq1ap0gyGiMo010YkqqYSEhFwTnbxMR0cHZmZmJXb8zMzMPCeDeZmJiQn09PRKLAYiIqLygH02ERFR0RS279y6dSvWrVuHjh07wtDQECdOnMBvv/2Gdu3aoVmzZqUYcY7nz5/j+fPnBbaxtLR87USdxSU2NrbA9Xp6ejAxMSmVWIg0jUl0okqqUaNGiI6Ozne9n58fjhw5UmLHDw4OVpvEJC9r167Fhx9+WGIxEBERlQfss4mIiIqmsH1nnTp1oKWlhe+//x7JycmqyUZnz55ditH+3w8//PDaiTgjIyNRo0aNUonH1ta2wPWDBg3CunXrSiUWIk1jTXSiSurkyZN53rqmVKVKFXh7e5fY8Z8+fYrz588X2MbDw+O1nTYREVFFxz6biIioaDTdd76piIgIREREFNimefPmkMvlpRLPgQMHClxvZ2cHd3f3UomFSNOYRCciIiIiIiIiIiIiygcnFiUiIiIiIiIiIiIiykeFrYmuUCjw4MEDGBkZQSKRaDocIiIiNUIIPHv2DHZ2dpBK+Z12XtiXExFRWcf+/PXYnxMRUVlW2L68wibRHzx4AHt7e02HQUREVKC7d++iWrVqmg6jTGJfTkRE5QX78/yxPyciovLgdX15hU2iGxkZAci5AMbGxhqOhuj1srOzERwcDABo2rQpZDKZhiMiopKUnJwMe3t7VX9FubEvp/KI/TlR5cL+/PXYn1N5w76cqHIpbF9eYZPoytvEjI2N2VFTudGpUydNh0BEpYy3NeePfTmVV+zPiSof9uf5Y39O5RH7cqLK53V9OYu2ERERERERERERERHlg0l0IiIiIiIiIiIiIqJ8VNhyLkTljUKhwPXr1wEAbm5uBc4ITERERGUT+3MiIqLyjX05EeWFSXSiMkIIgYSEBABA7dq1NRxN8VIoFMjMzNR0GESlSltbu9QmIVq2bBnmzZuH2NhY1K1bF0uWLEHjxo3zbb9t2zZMnToVUVFRcHFxwXfffYeOHTuq1gshMH36dKxevRqJiYlo1qwZVqxYARcXF1WbOXPmYNeuXbh06RJ0dHSQmJiY7/EeP36MunXr4v79+3j69ClMTU2L47SJyqSK3J8TERFVBuzLiSgvTKK/RrZC4EzkE8Q/S4eVkRyNHc0gk3LSGKLCyszMRGRkJBQKhaZDISp1pqamsLGxKdHJxrZu3YrAwECsXLkSPj4+WLRoEQICAnDjxg1YWVnlah8cHIz+/ftj7ty56Ny5MzZv3ozu3bvjwoUL8PT0BAB8//33+PHHH7F+/Xo4Ojpi6tSpCAgIQFhYGORyOYCc93bv3r3h6+uLX375pcAYhwwZgjp16uD+/fvFfwGIqNzj520iIqLyj/05VXRMohdgb+hDzPwnDA+T0lXLbE3kmN7FHe09bTUYGVH5IITAw4cPIZPJYG9vz9vgqNIQQiA1NRXx8fEAAFvbkuszFixYgGHDhmHw4MEAgJUrV2LXrl1Ys2YNvvzyy1ztFy9ejPbt22PChAkAgFmzZiEoKAhLly7FypUrIYTAokWLMGXKFHTr1g0AsGHDBlhbW+Ovv/5Cv379AAAzZ84EAKxbt67A+FasWIHExERMmzYNe/bsKa7TJqIKgp+3iYiIyj/251QZMImej72hDzFy4wWIV5bHJqVj5MYLWPF+A/4iIHqNrKwspKamws7ODvr6+poOh6hU6enpAQDi4+NhZWVVIqVdMjMzcf78eUyaNEm1TCqVwt/fHyEhIXluExISgsDAQLVlAQEB+OuvvwAAkZGRiI2Nhb+/v2q9iYkJfHx8EBISokqiF0ZYWBi+/vprnD59GhEREUU4MyKqDPh5m4iIqPxjf06VBYeF5iFbITDzn7BcvwAAqJbN/CcM2Yq8WhCRUnZ2NgBAR0dHw5EQaYbyy6MXL16UyP4fPXqE7OxsWFtbqy23trZGbGxsntvExsYW2F75b1H2mZeMjAz0798f8+bNQ/Xq1Qu9TXJystqDiComft4mIiIq/9ifU2XCJHoezkQ+UbsF5VUCwMOkdJyJfFJ6QRGVYyVZD5qoLKvMr/1JkybBzc0N77//fqG3mTt3LkxMTFQPe3v7EoyQiDSJn7eJiIjKP/bnVJkwiZ6H+Gf5/wJ4k3ZEREQlwcLCAjKZDHFxcWrL4+LiYGNjk+c2NjY2BbZX/luUfebl0KFD2LZtG7S0tKClpYU2bdqoYp4+fXqe20yaNAlJSUmqx927dwt9PCIqX/h5m4iIqPxjf06VCZPoebAykhdrO6LCkEqlaNGiBVq0aMEJOKlQatSogUWLFhXrPtetWwdTU9Ni3WdZVxLXsbTo6OjA29sbBw8eVC1TKBQ4ePAgfH1989zG19dXrT0ABAUFqdo7OjrCxsZGrU1ycjJOnz6d7z7z8scff+Dy5cu4dOkSLl26hJ9//hkAcPz4cYwePTrPbXR1dWFsbKz2ICpv2J8XDj9vExFRWcW+vPDYn1NlwolF89DY0Qy2JnLEJqXnWddJAsDGRI7GjmalHRpVYBKJpEQmHqSK6+zZszAwMNB0GOVeeb+OgYGBGDRoEBo2bIjGjRtj0aJFSElJweDBgwEAAwcORNWqVTF37lwAwLhx4+Dn54f58+ejU6dO2LJlC86dO4dVq1YByPldNH78eMyePRsuLi5wdHTE1KlTYWdnh+7du6uOGxMTgydPniAmJgbZ2dm4dOkSAMDZ2RmGhoZwcnJSi/PRo0cAADc3t0r3RQ1VLuzPC4eft4mIqKxiX154jR3NYGMsR2xy/iPNbdmfUwXBJHoeZFIJpndxx8iNFyAB8vxgP72LO2TSylvrlog0z9LSUtMhVAjl/Tr27dsXCQkJmDZtGmJjY1GvXj3s3btXNTFoTEyM2giapk2bYvPmzZgyZQq++uoruLi44K+//oKnp6eqzcSJE5GSkoLhw4cjMTERzZs3x969eyGX/38EybRp07B+/XrV8/r16wMADh8+jFatWpXwWRNReffy5+388PM2ERFR2SaTSuBT0ww7Lz3It011M31kKwT7dCr3eF9KPtp72mLF+w1gY5L7lpOP/ZzQ3tNWA1FRRaZQKBAeHo7w8HAoFApNh1OptWrVCp988gnGjx+PKlWqwNraGqtXr1aN7jUyMoKzszP27Nmj2iY7OxtDhgyBo6Mj9PT04OrqisWLF6vWp6enw8PDA8OHD1ctu3PnDoyMjLBmzZo84xBCYMaMGahevTp0dXVhZ2eHsWPHqta/WoZEIpHg559/Ro8ePaCvrw8XFxf8/fffavv8+++/4eLiArlcjtatW2P9+vWQSCRITEzM93rs3LkTDRo0gFwuR82aNTFz5kxkZWXl2z4qKgoSiQRbtmxB06ZNIZfL4enpiaNHjxb6egHAkSNH0LhxYxgYGMDU1BTNmjVDdHQ0AODy5cto3bo1jIyMYGxsDG9vb5w7d+6tr+O6desgkUhyPWbMmKFq//PPP8PNzQ1yuRy1a9fG8uXL870WpWXMmDGIjo5GRkYGTp8+DR8fH9W6I0eOYN26dWrte/fujRs3biAjIwOhoaHo2LGj2nqJRIKvv/4asbGxSE9Px4EDB1CrVi21NuvWrYMQItcjvwR6q1atIITgKHSq8NifF157T1sEtq2Va7m+jgwr3m/Az9tERKQR7MsL7+6TVOy7FgsAMNHTVltnoqcNqQQ4HfkEH607i+T0F5oIkajYcCR6Adp72qKtuw3ORD5B/LN0HLgej38uP0BwxGMIISCR8Fs0Kj5CCMTG5nQ+Li4uGo6mZGVnZ+e7TiKRqI2aLagtALXb7PJr+ya34q1fvx4TJ07EmTNnsHXrVowcORJ//vknevToga+++goLFy7EBx98gJiYGOjr60OhUKBatWrYtm0bzM3NERwcjOHDh8PW1hZ9+vSBXC7Hpk2b4OPjg06dOqFz5854//330bZtW3z00Ud5xvDHH39g4cKF2LJlCzw8PBAbG4vLly8XGPfMmTPx/fffY968eViyZAnee+89REdHw8zMDJGRkejVqxfGjRuHoUOH4uLFi/j8888L3N/x48cxcOBA/Pjjj2jRogXu3Lmj+iIgv8khlSZMmIBFixbB3d0dCxYsQJcuXRAZGQlzc/PXXq+srCx0794dw4YNw2+//YbMzEycOXNG9Xv3vffeQ/369bFixQrIZDJcunQJ2traecZRlOvYt29ftG/fXvX8yJEj+OCDD9CsWTMAwKZNmzBt2jQsXboU9evXx8WLFzFs2DAYGBhg0KBBBV4PIqocKlN/Xhwep2QCAFrWsoCnnQmWH7kDuZYUbd0LP5ExERFRcWJfXnhf/xuG9BcK+DiaYdNQH5yNeor4Z+mwMsop4XLsVgJGb7qAE7cfoc/KEKwd3Ai2JnqaDpvojTCJ/hoyqQS+TuYAgKZOFth/LRaX7yYi5M5jNHW20HB0ROXT8ePH811nZmaGOnXqqJ6fPHky32//TU1NUa9ePdXzU6dO4cWL3N9uv0lpibp162LKlCkAgEmTJuHbb7+FhYUFhg0bBiCnlMWKFStw5coVNGnSBNra2pg5c6Zqe0dHR4SEhOD3339Hnz59AAD16tXD7NmzMXToUPTr1w/R0dH4999/840hJiYGNjY28Pf3h7a2NqpXr47GjRsXGPeHH36I/v37AwC++eYb/Pjjjzhz5gzat2+Pn376Ca6urpg3bx4AwNXVFaGhoZgzZ06++5s5cya+/PJLVYK4Zs2amDVrFiZOnPjaJPqYMWPw7rvvAgBWrFiBvXv34pdffsHEiRNfe72Sk5ORlJSEzp07q2pru7m5qV2bCRMmoHbt2gAK/nBblOuop6cHPb2cD3V37tzB6NGj8c0336Bt27YAcr44mD9/Pnr27KmKOywsDD/99BOT6ERERaRQCOwNzUlSDPKtgZa1LLH5TAyepL7Amcgnqs/gRFS8li1bhnnz5iE2NhZ169bFkiVLCvyMuWjRIqxYsQIxMTGwsLBAr169MHfuXLUyb0RU+RwIi0NQWBy0pBLM6u4JLZk0V9/d2tUKv4/wxeB1ZxEe+ww9lgVjzYeN4G5nrKGoid4cy7kUgaWRLvo2sgcALDtyW8PREFFJejmRL5PJYG5uDi8vL9UyZb3p+Ph41bJly5bB29sblpaWMDQ0xKpVqxATE6O2388++wy1atXC0qVLsWbNGpib558g6N27N9LS0lCzZk0MGzYMf/75Z4FlVF6N28DAAMbGxqoYb9y4gUaNGqm1f11S/vLly/j6669haGioegwbNgwPHz5EamoqPv74Y7V1L/P19VX9X0tLCw0bNsT169dVywq6XmZmZvjwww8REBCALl26YPHixXj48KFq28DAQAwdOhT+/v749ttvcefOnXzP4U2uozKB36lTJ0yYMAEAkJKSgjt37mDIkCFq5zx79uwCj09ERHm7eDcRscnpMNTVQnMXC2jLpGjnntO/7gl9+JqtiehNbN26FYGBgZg+fTouXLiAunXrIiAgQO0z7cs2b96ML7/8EtOnT8f169fxyy+/YOvWrfjqq69KOXIiKkvSMrMx459rAIAhzR1Ry9oo37aeVU3w56imcLYyRGxyOvr8FILjtxJKK1SiYsOR6EU0vGVNbD4dg5O3H+NizFPUr15F0yERlTstWrTId92rZZKUZTQKo0mTJm8c06teLQ0ikUjUlinjVI6S37JlCz7//HPMnz8fvr6+MDIywrx583D69Gm1/cTHx+PmzZuQyWS4deuWWumQV9nb2+PGjRs4cOAAgoKCMGrUKMybNw9Hjx7Nt3RJXnG/TR2/58+fY+bMmaqR1y+Ty+X4+uuvX1sSJi+FuV5r167F2LFjsXfvXmzduhVTpkxBUFAQmjRpghkzZmDAgAHYtWsX9uzZg+nTp2PLli3o0aNHrmMV9TpmZ2ejb9++MDY2xqpVq9SuBQCsXr1areY48GYlg4iIKrs9V3MS5f5uVtDVyvk92sHLFr+fu4c9obGY0cUDUk5CRlSsFixYgGHDhmHw4MEAgJUrV2LXrl1Ys2YNvvzyy1ztg4OD0axZMwwYMABAzlwy/fv3z/UZl4gql+VHbuPe0zTYmsgxts3rS95Uq6KPPz5uihEbz+FUxBMMXnsWc3t6oXdD+1KIlqh4cCR6EVWroo/u9asCAJYf4chDojchk8nyfbxcD/11bV9NXBamTUk5efIkmjZtilGjRqF+/fpwdnbOc3TyRx99BC8vL6xfvx5ffPGF2sjsvOjp6aFLly748ccfceTIEYSEhODq1atvFKOrq2uuyTfPnj1b4DYNGjTAjRs34OzsnOshlUphZWWltuxlp06dUv0/KysL58+fV5VkKez1ql+/PiZNmoTg4GB4enpi8+bNqnW1atXCp59+iv3796Nnz55Yu3ZtvudRlOv46aef4urVq/jrr7/UblO2traGnZ0dIiIicl0LR0fHAq8jERGpE0Jgz3+lXDp4/X8C0WZOFjCSayHhWQbOxzzVVHhEFVJmZibOnz8Pf39/1TKpVAp/f3+EhITkuU3Tpk1x/vx5nDlzBgAQERGB3bt355qU/GUZGRlITk5WexBRxRGR8Bw/HY0AAEzv4g4D3cKNzzXR18b6jxqjWz07ZCkEJmy/gkUHbkIIUZLhEhUbjkR/Ax/7OeGPC/cQFBaHG7HP4GqT/20rRFQ5uLi4YMOGDdi3bx8cHR3x66+/4uzZs2rJ1WXLliEkJARXrlyBvb09du3ahffeew+nTp2Cjo5Orn2uW7cO2dnZ8PHxgb6+PjZu3Ag9PT04ODi8UYwjRozAggUL8MUXX2DIkCG4dOkS1q1bByD3HQBK06ZNQ+fOnVG9enX06tULUqkUly9fRmhoKGbPnl3g8ZYtWwYXFxe4ublh4cKFePr0qWoS1dddr8jISKxatQpdu3aFnZ0dbty4gVu3bmHgwIFIS0vDhAkT0KtXLzg6OuLevXs4e/asqv7621zHtWvXYvny5fjzzz8hkUhUEwopS7fMnDkTY8eOhYmJCdq3b4+MjAycO3cOT58+RWBgYKF+DkREBFy5l4T7iWnQ15HBr5alarmOlhRt3a2x48J97L76EI1qmGkwSqKK5dGjR8jOzlaVJVSytrZGeHh4ntsMGDAAjx49QvPmzSGEQFZWFj7++OMCy7nMnTtXbe4bIqo4hBCYtvMaMrMV8KtliQCPok0Erqslw8I+9VDVVA/Lj9zBogO3cO9pGub29IK2jON8qWzjK/QNOFsZooNnzi+KFayNTkTISVD37NkTffv2hY+PDx4/foxRo0ap1oeHh2PChAlYvnw57O1zbllbvnw5Hj16hKlTp+a5T1NTU6xevRrNmjVDnTp1cODAAfzzzz8F1lEviKOjI7Zv344dO3agTp06WLFiBSZPngwA0NXVzXObgIAA/Pvvv9i/fz8aNWqEJk2aYOHChYVK5H/77bf49ttvUbduXZw4cQJ///03LCxyJmR+3fXS19dHeHg43n33XdSqVQvDhw/H6NGjMWLECMhkMjx+/BgDBw5ErVq10KdPH3To0CHfP9aKch2PHj2K7OxsdO3aFba2tqrHDz/8AAAYOnQofv75Z6xduxZeXl7w8/PDunXrOBKdiKiIdv9X8/yd2laQa6vfNdbRM2dk+t7QWCgUHJ1GpElHjhzBN998g+XLl+PChQvYsWMHdu3ahVmzZuW7zaRJk5CUlKR63L17txQjJqKStOvqQ5y4/Qg6WlLM7OqR72CsgkilEkxsXxtzenhCKgG2n7+Hj9adxbP0FyUQMVHxkYgKet9EcnIyTExMkJSUBGPj4p/1N/R+EjovOQGpBDjyeWtUN9cv9mNQ5SKEwIsXOZ2Gtrb2G3VGZU16ejoiIyPh6OioVhaDyo45c+Zg5cqVxfrHTVRUFBwdHXHx4kXUq1ev2PZbHhX0Hijpfqoi4DWi8qgi9ufFTQiBVj8cQfTjVCx/rwE6vlTOBQDSX2Sj4ewDeJ6RhT9HNeUcRFSmlae+KjMzE/r6+ti+fTu6d++uWj5o0CAkJiZi586dubZp0aIFmjRpgnnz5qmWbdy4EcOHD8fz589zlWLMS3m6RkQA+/L8PM/IQpv5RxCXnIHx/i4Y71/rrfd5KDwOozddRNqLbNS2McK6wY1hY8LcAZWuwvZTHIn+hjyrmsCvliUUAlh5jLXR6e1JJBLo6OhAR0eHnTSVmOXLl+Ps2bOIiIjAr7/+innz5mHQoEGaDouIqMJgf/56YQ+TEf04FXJtKVq5WuZaL9eW4Z3aVgCgqptORG9PR0cH3t7eOHjwoGqZQqHAwYMH4evrm+c2qampec5ZBIB1jKnCYl+et0VBNxGXnAEHc3187OdULPt8p7Y1to5oAgtDXYTHPkOP5ScRHst5FKhsYhL9LYxunTOJ3vZz9xCXnK7haIiIXu/WrVvo1q0b3N3dMWvWLHz22WeYMWOGpsMiIqJKZM/VnMR4q1pW0NfJe4qmjl45pRN3X33IRB1RMQoMDMTq1auxfv16XL9+HSNHjkRKSgoGDx4MABg4cCAmTZqkat+lSxesWLECW7ZsQWRkJIKCgjB16lR06dJFlUwnooovPDYZa4OjAAAzunrkKsX2NupUM8Wfo5rCydIAD5PS0XtFCE7eflRs+ycqLkVKoq9YsQJ16tSBsbExjI2N4evriz179qjWp6enY/To0TA3N4ehoSHeffddxMXFqe0jJiYGnTp1gr6+PqysrDBhwgRkZWWptTly5AgaNGgAXV1dODs7qya+K2saO5qhUY0qyMxW4OfjEZoOh8o5hUKBmzdv4ubNm1AoFJoOhyqohQsX4sGDB0hPT8fNmzcxdepUaGkV7xzTNWrUgBCi0pdyIaLKif15wYQQ2H01px56B6/8JyPzq2UFPW0Z7j1NQ+h9jkgjKi59+/bFDz/8gGnTpqFevXq4dOkS9u7dq5psNCYmBg8fPlS1nzJlCj777DNMmTIF7u7uGDJkCAICAvDTTz9p6hSIShz7cnUKhcCUP0ORrRBo72GD1q5WxX4MezN97BjZDI0dzfAsIwuD1pzBH+fvFftxiN5GkZLo1apVw7fffovz58/j3LlzeOedd9CtWzdcu3YNAPDpp5/in3/+wbZt23D06FE8ePAAPXv2VG2fnZ2NTp06ITMzE8HBwVi/fj3WrVuHadOmqdpERkaiU6dOaN26NS5duoTx48dj6NCh2LdvXzGdcvEa9d9o9E2nY/A0JVPD0VB5JoTAgwcP8ODBA464IiIiKqfYnxfsZtxzRDxKgY6WVFWyJS96Ov8v6aKchJSIiseYMWMQHR2NjIwMnD59Gj4+Pqp1R44cURvEpqWlhenTp+P27dtIS0tDTEwMli1bBlNT09IPnKiUsC9X98eFezgX/RT6OjJM6+JeYscx0dfGr0Mao0tdO2QpBD7bdhk/HrzFnwGVGUVKonfp0gUdO3aEi4sLatWqhTlz5sDQ0BCnTp1CUlISfvnlFyxYsADvvPMOvL29sXbtWgQHB+PUqVMAgP379yMsLAwbN25EvXr10KFDB8yaNQvLli1DZmZOAnrlypVwdHTE/Pnz4ebmhjFjxqBXr15YuHBh8Z99MWhVyxLutsZIzczGuv9ubSEiIiIiotyUo9BbuljCSK5dYFvlSPU9LOlCRESkEYmpmZi7JxwAMK6NC+xM9Ur0eLpaMizuW09Vc31B0E18+cdVvMjmHQGkeW9cEz07OxtbtmxBSkoKfH19cf78ebx48QL+/v6qNrVr10b16tUREhICAAgJCYGXl5fqVjEACAgIQHJysmo0e0hIiNo+lG2U+yhrJBKJqjb6uuAoPM/Ies0WRJUP//Clyoq3fxIRqdvz36jyjgWUclFq7WoFXS0poh6nIjz2WUmHRkRERK+Yt+8GnqRkwsXKEB81dyyVY0qlEnzZoTZmdfeEVAJsPXcXQ9afY76NNK7IhXCvXr0KX19fpKenw9DQEH/++Sfc3d1x6dIl6Ojo5Lqty9raGrGxOZMHxcbGqiXQleuV6wpqk5ycjLS0NOjp5f2tV0ZGBjIyMlTPk5NLr3Zie08b1LQwQMSjFGw+HY3hLYtnlmKi8k5bWxsSiQQJCQmwtLTkzOZUaQghkJmZiYSEBEilUujo6Gg6JCIijbsd/xw3455DWyZBGzfr17Y30NVCK1dL7LsWhz1XH8LN1rgUoiQiIiIAuHw3EZvPxAAAZnX3hLbsjcfhvpEPmjjAzkSOMZsv4tjNBPRZGYK1gxvB2lheqnEQKRU5ie7q6opLly4hKSkJ27dvx6BBg3D06NGSiK1I5s6di5kzZ2rk2DKpBB+3csLE7Vew+ngkBvrWKNaZionKK5lMhmrVquHevXuIiorSdDhEpU5fXx/Vq1eHVFq6HziJiMqivf+NQm/mbAETvYJLuSh18LTFvmtx2B0ai8B2riUZHhEREf0nWyEw5a9QCAH0rF8VTWqaaySONm7W2DK8CYasP4uwh8nosewk1g5uDFcbI43EQ5VbkZPoOjo6cHbOKV/i7e2Ns2fPYvHixejbty8yMzORmJioNho9Li4ONjY5t2va2NjgzJkzavuLi4tTrVP+q1z2chtjY+N8R6EDwKRJkxAYGKh6npycDHt7+6Ke3hvrXq8qFgXdxIOkdGw/fw/vN3EotWMTlWWGhoZwcXHBixcvNB0KUamSyWTQ0tLiHRhERP/ZfTXnztOOnraF3uYdNyvoyKS4Hf8ct+KewcWafzQTERGVtM2no3H1fhKM5FqY1NFNo7HUtTfFn6OaYdDaM4hISEGvlcH46X1vNHW20GhcVPkUOYn+KoVCgYyMDHh7e0NbWxsHDx7Eu+++CwC4ceMGYmJi4OvrCwDw9fXFnDlzEB8fDysrKwBAUFAQjI2N4e7urmqze/dutWMEBQWp9pEfXV1d6Orqvu3pvDEdLSmGt6yJGf+EYeXRO+jXyB5apXyrC1FZJZPJIJPx7gwiIqLKKupRCsIeJkMmlaCt++tLuSgZy7XRwsUCB8PjsftqLMYxiU5ERFSiEp5l4Pt9NwAAEwJcYWmkuVybkr2ZPnaMbIphG87hbNRTDFp7Bt/3qoMe9atpOjSqRIqU5Z00aRKOHTuGqKgoXL16FZMmTcKRI0fw3nvvwcTEBEOGDEFgYCAOHz6M8+fPY/DgwfD19UWTJk0AAO3atYO7uzs++OADXL58Gfv27cOUKVMwevRoVQL8448/RkREBCZOnIjw8HAsX74cv//+Oz799NPiP/ti1rdRdZgb6ODe0zT8ffmBpsOhckYqlaJJkyZo0qQJSz8QERGVU+zP87YnNGcUelMnc1QxKNo8ER28bP/bx8Nij4uIiOhVlb0vn7vnOp6lZ8GzqjHe8yk7VRZM9XXw6xAfdKpjixfZAp9uvYylh25BCKHp0KiSKNJvg/j4eAwcOBCurq5o06YNzp49i3379qFt27YAgIULF6Jz585499130bJlS9jY2GDHjh2q7WUyGf7991/IZDL4+vri/fffx8CBA/H111+r2jg6OmLXrl0ICgpC3bp1MX/+fPz8888ICAgoplMuOXo6MtVsxcuP3IFCwTcyFZ5EIoFcLodcLmf5ByIionKK/XnelAnwDkUo5aLU1s0aWlIJwmOfISLheXGHRkREpKYy9+WnIh5jx4X7kEiAWd08IZOWrfOXa8uwpF99jGhZEwDww/6b+OrPq8jKVmg4MqoMJKKCfmWTnJwMExMTJCUlwdjYuPSOm/4CzeYewrOMLKx83xvtPW1K7dhERFR+aKqfKk94jYgqhrtPUtHi+8OQSoAzk/1hYVj028IHrTmDozcTMCHAFaNbO5dAlERvhn3V6/EaEZUPL7IV6PTjcdyMe44BPtXxTQ8vTYdUoA0hUZjx9zUoBNDK1RJLBzSAoe5bV62mSqiw/VTluy+lhBnLtTGwac7tLsuP3OZtJVRoCoUCd+7cwZ07d6BQ8FtUIiKi8oj9eW77ruWUcvFxNH+jBDoAdPTKGZjCki5ERFTSKmtfvvZkJG7GPYeZgQ4mBrhqOpzXGuhbAyvf94ZcW4ojNxLQ96cQxCenazosqsCYRC8Bg5s5Qq4txZV7SThx+5Gmw6FyQgiBu3fv4u7du/zyhYiIqJxif57b7qs5iW9lIvxNtHW3gUwqQej9ZMQ8Ti2u0IiIiHKpjH35w6Q0LDpwCwDwZYfaMNUv2vwlmtLOwwZbhvvC3EAH1x4ko8fyYNyKe6bpsKiCYhK9BFgY6qJfo+oAgGWHb2s4GiIiIiIizXiYlIYLMYmQSIAAjzdPopsZ6KBJTTMAHI1ORERU3Gb9G4bUzGx4O1RBrwbVNB1OkdSzN8Wfo5qhpoUB7iemoeeKYITceazpsKgCYhK9hAxvWRNaUglORTzB+egnmg6HiIiIiKjU7Q3NKeXS0KEKrIzlb7Uv5aSku//bJxEREb29IzfisftqLGRSCWZ394S0jE0mWhjVzfXxx8im8HaogmfpWRi05gx2Xrqv6bCogmESvYTYmeqhZ4OqAIDlh+9oOBoiIiIiotK352pOwluZAH8bAR42kEiAy3cTce8pS7oQERG9rfQX2Zj+9zUAwIdNa8DNtvxO/lvFQAebhvqgo5cNMrMVGLflEucqpGLFJHoJ+tjPCRIJcDA8HmEPkjUdDhERERFRqYlPTsfZ/+7IbO/55qVclCyNdNG4Rk5Jl70cjU5ERPTWfjoagejHqbAy0sV4fxdNh/PW5NoyLO3fAEObOwIAvt97A5P/CkVWduWZIJZKDpPoJaimpSE6euWMullxlKPRiYiIiKjy2HctFkIA9aubws5Ur1j2qfxszSQ6ERHR24l+nIJlR3Lm8Zva2R1Gcm0NR1Q8pFIJpnR2x/Qu7pBIgM2nYzBswzmkZGRpOjQq55hEL2GjWjkBAHZdeYDIRykajoaIiIiIqHTs/q+US8diKOWipBzRfi76KWKT0ottv0RERJWJEAIz/r6GzCwFmjtboHOd4uury4rBzRyx8n1v6GpJcfhGAvqtOoX4Z/zsQG+OSfQS5mFngtaullAI4CeORqcCSKVSNGrUCI0aNYJUyrcmERFRecT+PMfj5xk4HfkYQPGUclGyNpbD26EKgJyR7kRERMWtMvTl+67F4fCNBGjLJJjZzQMSSfmbTLQwAjxs8NvwJjAz0MHV+0nosSwYt+OfaTosKqcq5m+DMmZ0a2cAwB8X7uFhUpqGo6GySiKRwMDAAAYGBhW2AyMiIqro2J/n2B8WB4UAvKqawN5Mv1j33eG/pPzuqw+Ldb9ERERAxe/LUzOz8PU/OZOJjmjpBCdLQw1HVLIaVK+CHSObooa5Pu4npqHn8mCcjnis6bCoHGISvRQ0rGGGxo5meJEtsPpYpKbDISIiIiIqUcoEdwev4huFrtThv7roZ6KeIOFZRrHvn4iIqCL78eBtPEhKR1VTPdWgz4quhoUBdoxqhgbVTZGcnoUPfjmDvy8/0HRYVM4wiV5KlL+YfjsTg8fP+WGfclMoFIiKikJUVBQUCs4cTUREVB6xPweepmQi+E7OCK8OxVgPXamqqR7q2ptCCJZ0ISKi4leR+/Jbcc/w8/EIAMDMrh7Q05FpOKLSY2agg83DmqC9hw0ysxUY+9tFrDx6B0IITYdG5QST6KWkpYsFPKsaI+1FNtYFR2k6HCqDhBCqjpq/xImIiMon9udA0PU4ZCsE3GyN4WhhUCLH6PhfSZc9oSzpQkRExaui9uVCCEzdGYoshYC/mzX83a01HVKpk2vLsOy9BviomSMA4Ns94TnXJLtifVlCJYNJ9FIikUgwulXOaPR1wVF4lv5CwxERERERERW/Pf+VculYjBOKvko5wv1UxBM8ScksseMQERFVFDsvPcCpiCeQa0sxvYu7psPRGJlUgmld3DG1szskEmDjqRiM+PU8UjOzNB0alXFMopeiAA8bOFka4Fl6FjaeitF0OERERERExSop7QVO3H4E4P+1y0tCdXN9eFY1RrZCICiMJV2IiIgKkpT2ArN3XQcAfPKOS7FP+l0eDWnuiOUDGkBXS4qD4fHot+oU51opJ7IVAiF3HmPnpfsIufMY2YrSuWOESfRSJJVKMPK/0ei/nIhA+otsDUdERERERFR8DoXH4UW2QC1rQzhbGZbosZSj0XdfZRKdiIioIAuDbuLR8wzUtDDA0BaOmg6nzOjgZYvNw3xQRV8bV+4locfyk7gd/1zTYVEB9oY+RPPvDqH/6lMYt+US+q8+hebfHcLeUijxxyR6KetWzw5VTfXw6Hkmfj93V9PhEBEREREVG2VCu30JTCj6qg7/lYs5efsRklJZKpGIiCgvofeTsCEkCgDwdTdP6GpVnslEC8PbwQw7RjWDg7k+7j1Nw7srgnE26ommw6I87A19iJEbL+BhUrra8tikdIzceKHEE+lMopcybZkUI/xqAgB+OhqBF5y8gIiIiIgqgOcZWTh6MwEA0NGr5OqhK9W0NERtGyNkKQSCrseV+PGIiIjKG4VCYMpfoVAIoEtdOzR3sdB0SGWSo4UBdoxsinr2pkhKe4H3fj6Nf6880HRY9JJshcDMf8KQV+EW5bKZ/4SVaGkXJtE1oE9De1gY6uJ+Yhp2XuKbkoiIiIjKv0Ph8cjMUqCmhQFcrY1K5ZjKki7KyUyJiIjo/7aeu4tLdxNhqKuFKZ3cNB1OmWZuqIvfhjVBO3drZGYpMGbzRaw6dgdClE69bcqfQiGw+UxMrhHoLxMAHial40xkyd1FwCS6Bsi1ZaoaVMuP3C61AvhUtkmlUjRo0AANGjSAVMq3JhERUXlUmftzZSK7g5cNJBJJqRxTOeL9+K1HSE5nSRciInp7FaUvf5KSie/2hgMAPm1bC9bGcg1HVPbp6ciw4n1vfNi0BgDgm93hmP73NebtNCD9RTYOhcdh0o6raDL3IKb+FVqo7eKf5Z9of1vl97dBOfeeT3UYy7UQkZCC/dc4GRIBEokExsbGMDY2LrU/PImoYli2bBlq1KgBuVwOHx8fnDlzpsD227ZtQ+3atSGXy+Hl5YXdu3errRdCYNq0abC1tYWenh78/f1x69YttTZz5sxB06ZNoa+vD1NT01zHePz4Mdq3bw87Ozvo6urC3t4eY8aMQXJy8lufL1FZVln789TMLBy+EQ/g/6PDS4OLtRGcrQyRma3AoevxpXZcIiKquCpKX/7dnnAkpr5AbRsjDPJ10HQ45YZMKsH0Lu6qkfsbQqIx4tfzSMvM1nBkFd+TlExsO3cXI349h/pfB+Gjdefw25kYxD/LgFy7cClsK6OS+7KISXQNMZJrq77ZWnbkNm8PISKiN7J161YEBgZi+vTpuHDhAurWrYuAgADEx+edTAoODkb//v0xZMgQXLx4Ed27d0f37t0RGvr/b/a///57/Pjjj1i5ciVOnz4NAwMDBAQEID39/9/qZ2Zmonfv3hg5cmSex5FKpejWrRv+/vtv3Lx5E+vWrcOBAwfw8ccfF+8FIKIy4ciNBKS/UKC6mT487IxL9dgd/5tgdE8JTyZFRERUXpyPfoKt5+4CAOb08ISWjOm/opBIJBjaoiaWv9cAOlpSHLgeh36rT+HR8wxNh1bhRCQ8x6pjd9B7ZTAazg7ChO1XsO9aHNJeZMPWRI4Pmjhgw0eNcWFqW9iayJHf11oSALYmcjR2NCuxWCWigmZvk5OTYWJigqSkJBgbl+4H+cJ6kpKJZt8eQtqLbKz/qDH8allqOiTSIIVCgXv37gEAqlWrVq5vGyOi1yuufsrHxweNGjXC0qVLAeT8LrG3t8cnn3yCL7/8Mlf7vn37IiUlBf/++69qWZMmTVCvXj2sXLkSQgjY2dnhs88+w+effw4ASEpKgrW1NdatW4d+/fqp7W/dunUYP348EhMTXxvrjz/+iHnz5uHu3buFOrfy0JcTvaqy9uef/HYR/1x+gBF+NTGpQ+nWXA17kIyOPx6HrpYUF6a2hYGuVqkenyo39lWvx2tE5U1578uzshXovOQEwmOfoU/Davi+V11Nh1SunYt6gqEbziEx9QWqm+lj3eBGqGlpqOmwyq1shcDFmKcIuh6HA2FxuJOQorbe3dYYbd2t0dbdGh526neD7A19iJEbLwCA2gSjyhYr3m+A9m9wR2Rh+6ny9ZuggjEz0MEAn+oAgGWHb2s4GtI0IQQiIiIQERHBOxOIqFAyMzNx/vx5+Pv7q5ZJpVL4+/sjJCQkz21CQkLU2gNAQECAqn1kZCRiY2PV2piYmMDHxyfffRbGgwcPsGPHDvj5+eXbJiMjA8nJyWoPovKmMvbn6S+yceh6HIDSLeWi5GZrhBrm+sjIUqhKyhAREb2p8t6XbwiJRnjsM5joaeOL9rU1HU6517CGGXaMbIrqZvqIeZKKniuCcS6q5CavrIjSMrOx/1osJmy7jMZzDqDXyhD8dDQCdxJSoC2ToIWLBb7u5oGTX76D3eNa4NO2teBZ1SRXOaX2nrZY8X4D2Jiol2yxMZG/cQK9KJhE17BhLWpCWybBmcgnOMs3IRERFcGjR4+QnZ0Na2trteXW1taIjc17vo3Y2NgC2yv/Lco+C9K/f3/o6+ujatWqMDY2xs8//5xv27lz58LExET1sLe3L/LxiKj0HbuZgJTMbNiZyFG3mkmpH18ikaCDV84fTXuucq4hotcpylwqrVq1gkQiyfXo1KlTKUZMRIUVl5yOBUE3AQBftK8Nc0NdDUdUMdS0NMSOUU1Rt5oJElNfYMDPp7H7KsvIFST+WTq2nInB0PVnUe/r/Rj+63lsO38Pj1MyYSTXQrd6dljSvz7OT22LX4f4YKBvDVQ11Xvtftt72uLEF+/gt2FNsLhfPfw2rAlOfPFOiSfQASbRNc7GRI5e3tUAAMs5Gp2IiCqYhQsX4sKFC9i5cyfu3LmDwMDAfNtOmjQJSUlJqkdhy74QkWbtCc1JXHfwstXYBGwd//vD6VB4PCf+IipAUedS2bFjBx4+fKh6hIaGQiaToXfv3qUcOREVxpxd1/E8Iwt17U3RrxEHpBQnC0Nd/Da8CfzdrJGZpcDozRfw8/HyebdCSRBC4FbcMyw7fBs9lp+EzzcH8eWOqzhwPR4ZWQpUNdXDh01rYPNQH1yY2haL+9VHl7p2MJZrF/lYMqkEvk7m6FavKnydzCGTls7nTxYMLANGtHTC1rN3cfhGAq49SIKHXemP4CEiovLHwsICMpkMcXFxasvj4uJgY2OT5zY2NjYFtlf+GxcXB1tbW7U29erVK3KMNjY2sLGxQe3atWFmZoYWLVpg6tSpavtW0tXVha4uR8sQlScZWdk4EJbzO6WjV96/d0qDZ1VjVKuih3tP03D0ZnypjEYiKo8WLFiAYcOGYfDgwQCAlStXYteuXVizZk2ec6mYmalP0LZlyxbo6+sziU5UBp28/Qh/X34AqQSY090T0lJKLFYm+jpa+OkDb8z85xo2hERj9q7ruPc0DVM7u5daIrcsycpW4Fz0UxwIi0PQ9ThEP05VW1+3mgn83azh726N2jZGGhtsUVw4Er0MqGFhgM517AAAy4/c0XA0RERUXujo6MDb2xsHDx5ULVMoFDh48CB8fX3z3MbX11etPQAEBQWp2js6OsLGxkatTXJyMk6fPp3vPgtLoVAAyKl9TkQVw8nbj/AsIwvWxrqob19FY3FIJBJ0/K+ky26WdCHK05vMpfKqX375Bf369YOBgUFJhUlEbyAjKxtTd4YCAD5o4gDPqhycWVJkUglmdvXA5I45E6mvC47CqE3nK82dcM8zsrD76kMEbr2EhnMOoN+qU/j5RCSiH6dCRyZFK1dLzOnhiVOT2mDnmOb4pI0L3GyNy30CHeBI9DJjZCsn/H35AXZffYiIhOec6ZeIiAolMDAQgwYNQsOGDdG4cWMsWrQIKSkpqhFmAwcORNWqVTF37lwAwLhx4+Dn54f58+ejU6dO2LJlC86dO4dVq1YByElEjR8/HrNnz4aLiwscHR0xdepU2NnZoXv37qrjxsTE4MmTJ4iJiUF2djYuXboEAHB2doahoSF2796NuLg4NGrUCIaGhrh27RomTJiAZs2aoUaNGqV5iYioBCkT1h08bTU+4q29pw1WHYvAofB4pL/IhlxbptF4iMqaguZSCQ8Pf+32Z86cQWhoKH755ZcC22VkZKh9Yc6JwolK3s/HIxGRkAILQ10EtnPVdDgVnkQiwbCWNWFrKkfg1svYdy0O/Vefwi+DGlbIOvSxSek4cD0OQWFxCLnzGJnZCtU6U31tvFPbCm3drNGiliUMdStuqrninlk542ZrDH83Kxy4Ho+VR+/g+151NR0SERGVA3379kVCQgKmTZuG2NhY1KtXD3v37lX9gRwTEwOp9P83njVt2hSbN2/GlClT8NVXX8HFxQV//fUXPD09VW0mTpyIlJQUDB8+HImJiWjevDn27t0Lufz/s6BPmzYN69evVz2vX78+AODw4cNo1aoV9PT0sHr1anz66afIyMiAvb09evbsmeet4kRUPr3IViDov1IuHTw1V8pFqV41U9iayPEwKR0nbj2Cv7v16zciokL75Zdf4OXlhcaNGxfYbu7cuZg5c2YpRUVEd5+kYsmhWwCAyZ1qw0Sv6DWm6c10rmMHKyM5hm04h0t3E9FzRTDWDW4MR4vyfbeOEALhsc8QFBaHA9fjcOVektp6B3N9tHWzRlt3a3g7VIGWrHIUOpGICloBPzk5GSYmJkhKSoKxsbGmwymUCzFP0XN5MLSkEhyd2LpQs9JSxSGEQFJSzi8mExOTCnGrCxHlrzz2U6WN14jKo8rUnx+7mYCBa87AwlAHp7/yLxO1QGf+cw1rT0ahZ4OqWNCnnqbDoUqgPPVVmZmZ0NfXx/bt29XuLhs0aBASExOxc+fOfLdNSUmBnZ0dvv76a4wbN67A4+Q1Et3e3r5cXCMioPz15cM2nENQWBx8HM2wZXiTMh9vRXQ7/jkGrzuDu0/SUEVfGz8PagRvB82VuXsTL7IVOBP5RJU4v/c0TbVOIgHq25vC390a7dyt4WRpWKFeZ4XtyzkSvQxpUL0KfGuaIyTiMVYfi8CMrh6aDolKkUQigampqabDICIiordQmfrzPaEPAQABHjZlIoEOAB29bLH2ZBSCwuKQmaWAjlblGBlFVBgvz6WiTKIr51IZM2ZMgdtu27YNGRkZeP/99197HE4UTuVdeerLD4TllNjQkkowu7tnhUpslifOVobYMbIZhqw/iyv3kjBg9Sks7levzE90npz+AkduJOBAWBwO34jHs/Qs1TpdLSlauFigrbs1Wte2gpWRvIA9VQ5Mopcxo1s7IyTiMbacjcGYd5xhUQFrKRERERFR+ZaVrcC+azmlXJQTepYF3tWrwMpIF/HPMnDyziO0drXSdEhEZUpR51JR+uWXX9C9e3eYm5trImwiykNaZjZm/HMNADCkhSNcrI00HFHlZmmkiy3Dm+CTzRdxMDweIzddwNRO7viouaOmQ1Nz72kqDl6PR1BYHE5FPEaW4v8FSswNdNDGzQr+btZo4WIJPR3OL/MyJtHLmGbO5qhbzQSX7yVh7clITAioremQqJQoFAo8fJgzosvW1lathjERERGVD5WlPz8T+QRPUjJRRV8bPo5mmg5HRSqVoL2nDTaERGPP1YdMohO9oqhzqQDAjRs3cOLECezfv18TIROVuvLSly8/chv3nqbBzkSOse+4aDocAqCvo4WfPvDGjH+uYeOpGHz9bxjuPU3D5E5uGrtrTwiB0PvJCLoehwNhcQh7qD7Zs5OlAdq626CtuxXq2VcpM3cXlkVMopcxEokEo1o7Y8Sv57EhOBoj/JxgLOekEJWBEAK3buVMBmJjo/nJuYiIiKjoKkt/vvulUi5lbTKpDp622BASjf1hcZiTrYB2GYuPSNPGjBmTb/mWI0eO5Frm6uqKCjqVGlGeykNfHpHwHD8djQAATOviDgNdpvfKCi2ZFLO6eaJaFX18uycca05G4kFiGhb1qwe5dumM7M7IysapiCcICovFgbB4xCanq9ZJJUBDBzP4u+eMOK9paVgqMVUEfJeVQW3drOFiZYhb8c/xa0g0Rrd21nRIREREREQAgGyFwN7QnFIuHcpQKRelxo5mMDfQweOUTJyOeILmLhaaDomIiKjYCCEwbec1ZGYr0MrVEgEeZTPRX5lJJBJ87OcEO1M9fP77Zey9FosBq0/h50GNYGagUyLHTEzNxOEb8TgQFo+jNxPwPOP/9c31tGVoWcsCbd1t0NrVEuYsHf1GmEQvg6RSCUa1dsKnWy9jzYlIfNTMkXWIiIiIiKhMOBf1BI+eZ8BETxtNncpefWSZVIJ2Hjb47UwMdoc+ZBKdiIgqlF1XH+LE7UfQ0ZJiZlcPTiZahnWtawdrI10M23AOF2IS0XP5Sawb3Bg1LAyKZf8xj1OxPywWB67H4WzUU2S/VN/c0kgX/m7WaOtuhaZOFqU2Cr4iYxK9jOpSxw7z99/Evadp2Ho2Bh82K1sTERARERFR5bQnNBYA4O9mXWZLpXT0ykmi7wuNxaxunqzvSUREFcLzjCzM+jcMADCqlRMczIsnGUslx6emOXaMaopBa84i6nEqeq4Ixs+DGqJB9SrIVgiciXyC+GfpsDKSo7GjWYGfWRQKgcv3EnHgehyCwuJwM+652npXayO0dbeGv7s16lQ1gZSff4oVk+hllJZMio/9nDDlr1D8dCwCA3wcoKNVNv9IISIiIqLKQaEQ2PtfEr2jV9m9fbxJTXOY6mvjcUomzkQ+gW8ZHDFPRERUVIuCbiIuOQMO5vr42M9J0+FQITlbGeHP0U3x0bqzCL2fjP6rTuHDpjXw9+UHeJj0/3rltiZyTO/ijvae/y+Xl/4iG8F3HiEoLA4Hrscj4VmGap1MKkHjGmbwd7dGWzdrVDfXL9XzqmyYRC/DenlXw+KDt/AwKR1/XbyPPo3sNR0SEREREVViF+8mIjY5HYa6WmW6TIq2TIp27tb4/dw97Al9yCQ6ERGVe9cfJmNtcBQAYGZXD5bnKGesjOTYOtwXYzZfwOEbCfjpWESuNrFJ6Ri58QK+71UHABAUFofjtx4h7UW2qo2hrhb8XC3R1s0arVwtYapfMjXWKTcm0cswubYMw1o44pvd4Vhx9A7e9a7GW1GJiIiISGP2XH0IAPB3s4KuVtn+472Dl+1/SfRYzOjiwVuaiYio3FIoBKb+FYpshUAHTxu0crXSdEj0Bgx0tbDyfW/UnxWE1MzsXOuVFc0nbL+ittzWRP5ffXNr+NQ0K/OfwSoqJtHLuAE+Dlh2+A4iH6VgT+hDdK5jp+mQqIRIpVJ4eXmp/k9ERETlT0Xuz4UQqnroHbxsX9Na85o5WcBIroWEZxk4H/MUjWqYaTokIiIqB8piX/7HhXs4F/0U+joyTO3srulw6C1ciEnMM4H+KgczfXSvXxVt3a3hYWfMCWTLgCL9Npg7dy4aNWoEIyMjWFlZoXv37rhx44Zamzt37qBHjx6wtLSEsbEx+vTpg7i4OLU2T548wXvvvQdjY2OYmppiyJAheP5cvRj+lStX0KJFC8jlctjb2+P7779/w1Ms3wx1tfBh0xoAgGWH70AIUfAGVG5JJBKYm5vD3NycvxyJiIjKqYrcn1+5l4T7iWnQ15HBr5alpsN5LR0tKdq6WQMA9lyN1XA0RERUXpS1vjwxNRNz94QDAMa1cYGdqZ6GI6K3Ef8s/fWNAAS2q4VP29aCZ1WTMvE6pCIm0Y8ePYrRo0fj1KlTCAoKwosXL9CuXTukpKQAAFJSUtCuXTtIJBIcOnQIJ0+eRGZmJrp06QKFQqHaz3vvvYdr164hKCgI//77L44dO4bhw4er1icnJ6Ndu3ZwcHDA+fPnMW/ePMyYMQOrVq0qptMuXz5sWgP6OjJcf5iMIzcSNB0OEREREVVCu0NzSrm8U9uq3NRhVY6Y3xP6EAoFB6MQEVH5M2/fDTxJyYSLlSE+au6o6XDoLVkZyYu1HZWeIpVz2bt3r9rzdevWwcrKCufPn0fLli1x8uRJREVF4eLFizA2NgYArF+/HlWqVMGhQ4fg7++P69evY+/evTh79iwaNmwIAFiyZAk6duyIH374AXZ2dti0aRMyMzOxZs0a6OjowMPDA5cuXcKCBQvUku2VRRUDHbznUx2rj0di6eHbaOVqyW+hKiCFQoH4+HgAgJWVVZm5bYyIiIgKr6L250II1WjuDp5lv5SLUgsXCxjoyPAwKR2X7yWifvUqmg6JiIjKuLLUl1+6m4jNZ2IAALO6e0JbVjE+V1RmjR3NYGsiR2xSOvL6el8CwMZEjsaOLENX1rzVuy8pKQkAYGaW84PNyMiARCKBrq6uqo1cLodUKsWJEycAACEhITA1NVUl0AHA398fUqkUp0+fVrVp2bIldHT+P8NsQEAAbty4gadPn75NyOXW0BY1oSOT4nz0U5yJfKLpcKgECCEQHh6O8PBwlu0hIiIqpypqfx72MBkxT1Ih15ailWvZL+WiJNeWoY2ypEsoS7oQEdHrlZW+PFshMOWvqxAC6Fm/KprUNNdYLFR8ZFIJpnfJqWv/6vBY5fPpXdwh44ToZc4bJ9EVCgXGjx+PZs2awdPTEwDQpEkTGBgY4IsvvkBqaipSUlLw+eefIzs7Gw8f5tz+GRsbCysr9VmEtbS0YGZmhtjYWFUba2trtTbK58o2r8rIyEBycrLaoyKxNpajV8NqAIBlR+5oOBoiIiIiqkyUo9Bb1bKCgW6RbmbVuI5eNgCA3VcfVqgvNoiIqGLbfDoaofeTYSTXwqSObpoOh4pRe09brHi/AWxM1Eu22JjIseL9Bmhfju76q0zeOIk+evRohIaGYsuWLapllpaW2LZtG/755x8YGhrCxMQEiYmJaNCgQYnf/jJ37lyYmJioHvb29iV6PE34uKUTpBLg2M0EXL2XpOlwiIiIiKgSEEJg99WcATEd/ktIlyd+taygpy3DvadpCL1fsQbaEBFRxZTwLAPf77sBAJgQ4ApLI93XbEHlTXtPW5z44h38NqwJFverh9+GNcGJL95hAr0Me6PM9pgxY/Dvv//i8OHDqFatmtq6du3a4c6dO4iPj8ejR4/w66+/4v79+6hZsyYAwMbGRlVbSikrKwtPnjyBjY2Nqk1cXJxaG+VzZZtXTZo0CUlJSarH3bt33+TUyrTq5vroWtcOALD8yG0NR0NERERElcHNuOeIeJQCHS0p3qlt9foNyhg9HZkqbuXkqERERGXZ3D3X8Sw9C55VjfGej4Omw6ESIpNK4Otkjm71qsLXyZwlXMq4IiXRhRAYM2YM/vzzTxw6dAiOjvnPCmxhYQFTU1McOnQI8fHx6Nq1KwDA19cXiYmJOH/+vKrtoUOHoFAo4OPjo2pz7NgxvHjxQtUmKCgIrq6uqFIl78mAdHV1YWxsrPaoiEa2cgYA7L0Wi9vxzzQcDRERERFVdMpR6C1dLGEk19ZwNG+mvWfOQJw9LOlCRERl3KmIx9hx4T4kEmB2dy8mVonKiCIl0UePHo2NGzdi8+bNMDIyQmxsLGJjY5GWlqZqs3btWpw6dQp37tzBxo0b0bt3b3z66adwdXUFALi5uaF9+/YYNmwYzpw5g5MnT2LMmDHo168f7OxyRlkPGDAAOjo6GDJkCK5du4atW7di8eLFCAwMLMZTL59cbYzQ1t0aQgArjkRoOhwiIiIiquD2/Dd6u2M5LOWi1Lq2FXS1pIh6nIrwWA5EISKisulFtgLTdoYCAPo3ro569qaaDYiIVIqURF+xYgWSkpLQqlUr2Nraqh5bt25Vtblx4wa6d+8ONzc3fP3115g8eTJ++OEHtf1s2rQJtWvXRps2bdCxY0c0b94cq1atUq03MTHB/v37ERkZCW9vb3z22WeYNm0ahg8f/panWzGMauUEAPjr0n3cfZKq4WiIiIiIqKK6Hf8cN+OeQ1smQRs3a02H88YMdbXgV8sSQM5odCIiorJo7clI3Ix7DjMDHUwMcNV0OET0Eq2iNC7MrY/ffvstvv322wLbmJmZYfPmzQW2qVOnDo4fP16U8CqN+tWroJmzOU7efozVxyPwdTdPTYdExUAqlcLd3V31fyIiIip/Klp/vve/UejNnC1golc+S7kodfSyxf6wOOwOjUVgOyYmiIgob5rqyx8mpWHRgVsAgC871Iapvk6pHZuIXq/8f7KvpEb/Vxt9y9m7iH+WruFoqDhIJBJYWVnBysoKEglrnhEREZVHFa0/3301FgDQ0dNWw5G8vXfcrKAjk+J2/HPcimNJFyIiypum+vJZ/4YhNTMbDR2qoFeDaqV2XCIqHCbRyylfJ3PUszdFZpYCa05EaTocIiIiIqpgoh6lIOxhMmRSCdq6l99SLkrGcm20cLEA8P8vB4iIiMqCIzfisftqLGRSCWZ194SUk4kSlTlMopdTEokEo1vnjEbfeCoaSakvNBwRvS0hBOLj4xEfH1+o0klERERU9lSk/nxPaE6iuamTOaoYVIxbyjt45YyoV06WSkRE9KrS7svTX2Rj+t/XAAAfNq0BN1vjEj8mERUdk+jlWJvaVnC1NsLzjCxsCInSdDj0lhQKBcLCwhAWFgaFQqHpcIiIiOgNVKT+XJlo7lABSrkotXWzhpZUgvDYZ4hIeK7pcIiIqAwq7b78p6MRiH6cCmtjXYz3dynx4xHRm2ESvRyTSiUY1doJALDmZCRSM7M0HBERERERVQR3n6Tiyr0kSCVAO4/yX8pFyURfG02dc0q6KEfaExERaUr04xQsO3IbADClkzuM5OV7Em+iioxJ9HKuk5ctHMz18TT1BX47c1fT4RARERFRBbD3vwRzY0czWBjqajia4tXR0wYAS7oQEZFmCSEw/e9ryMxSoLmzBTrXqTh3fhFVREyil3NaMik+9ssZjb76WAQysrI1HBERERERlXfKBHNHr4r3B307DxvIpBKE3k9GzONUTYdDRESV1L5rcThyIwE6Mim+7uYBiYSTiRKVZUyiVwA9G1SFtbEuYpPT8eeF+5oOh4iIiIjKsYdJabgQkwiJBAjwsNF0OMXOzEAHTWqaAeBodCIi0ozUzCx8/U/OZKLDW9ZETUtDDUdERK/DJHoFoKslw7AWNQEAK47eQVZ2+Z7EioiIiIg0R1nKpaFDFVgbyzUcTclQTpa6m3XRiYhIA348eBsPktJRrYoeRrd21nQ4RFQITKJXEP0bV0cVfW1EP07lHwNERERE9Mb2XM35LKlMNFdEAR42kEiAy3cTcT8xTdPhEBFRJXIr7hl+Ph4BAJjZ1QN6OjINR0REhcEkegVhoKuFwc0cAQDLD9+GEELDEVFRSSQS1K5dG7Vr12YtNCIionKqvPfn8cnpOBv9BADQ3rPilXJRsjTSRaMaOSVd9nIAChERvaQk+3IhBKb8FYoshYC/mzXauFkX6/6JqOQwiV6BDPKtAQMdGcJjn+FQeLymw6EikkqlsLGxgY2NDaRSvjWJiIjKo/Len++7FgshgPrVTWFnqqfpcEpUx/++JNhzlXXRiYjo/0qyL9956QFORz6BXFuK6V3ci3XfRFSyyt8ne8qXib423vd1AAAs5Wh0IiIiIiqi3f+VculYgUu5KLX/7xzPRT9FbFK6hqMhIqKKLintBWbvug4A+OQdF9ib6Ws4IiIqCibRK5ghzR2hoyXFxZhEnIp4oulwqAiEEHj8+DEeP37ML0CIiIjKqfLcnz9+noHTkY8BVOxSLko2JnJ4O1QBkDMCn4iICCi5vnxh0E08ep6BmpYGGNrCsdj2S0Slg0n0CsbKSI6+De0BAMuP3NZwNFQUCoUCV69exdWrV6FQKDQdDhEREb2B8tyf7w+Lg0IAXlVNKs3ouA7/fVmwmyVdiIjoPyXRl4feT8KGkCgAwKxuntDV4mSiROUNk+gV0PCWNSGTSnD81iNcvpuo6XCIiIiIqBxQJpI7eFX8UehKHbxySrqciXqChGcZGo6GiIgqIoUiZzJRhQC61LVDM2cLTYdERG+ASfQKyN5MH93q2QHgaHQiIiIier2nKZkIvpNTyqVDJaiHrlTVVA917U0hBEu6UOWxbNky1KhRA3K5HD4+Pjhz5kyB7RMTEzF69GjY2tpCV1cXtWrVwu7du0spWqLyb+u5u7h0NxGGulqY0slN0+EQ0RtiEr2CGtXKCRIJsO9aHG7FPdN0OERERERUhgVdj0O2QsDN1hiOFgaaDqdUKUu67A1lEp0qvq1btyIwMBDTp0/HhQsXULduXQQEBCA+Pj7P9pmZmWjbti2ioqKwfft23LhxA6tXr0bVqlVLOXKi8ulJSia+2xsOAPi0bS1YG8s1HBERvSkm0SsoZysjBLjn/EGw4sgdDUdDREQlqagjyrZt24batWtDLpfDy8sr12gyIQSmTZsGW1tb6Onpwd/fH7du3VJrM2fOHDRt2hT6+vowNTXNdYzLly+jf//+sLe3h56eHtzc3LB48eK3PlciKhl7/ivl0rESTCj6KmUSPSTiMZ6kZGo4GqKStWDBAgwbNgyDBw+Gu7s7Vq5cCX19faxZsybP9mvWrMGTJ0/w119/oVmzZqhRowb8/PxQt27dUo6cqHz6bk84ElNfoLaNEQb5Omg6HCJ6C0yiV2CjWjsBAHZefoC7T1I1HA0REZWEoo4oCw4ORv/+/TFkyBBcvHgR3bt3R/fu3REaGqpq8/333+PHH3/EypUrcfr0aRgYGCAgIADp6emqNpmZmejduzdGjhyZ53HOnz8PKysrbNy4EdeuXcPkyZMxadIkLF26tHgvABG9taS0Fzhx+xGA/9cIr0wczA3gYWeMbIVAUBhHo1PFlZmZifPnz8Pf31+1TCqVwt/fHyEhIXlu8/fff8PX1xejR4+GtbU1PD098c033yA7Ozvf42RkZCA5OVntQVQZnY9+gq3n7gIA5vTwhJaMKTii8ozv4AqsTjVTtHCxQLZC4KdjHI1ORFQRFXVE2eLFi9G+fXtMmDABbm5umDVrFho0aKBKbgshsGjRIkyZMgXdunVDnTp1sGHDBjx48AB//fWXaj8zZ87Ep59+Ci8vrzyP89FHH2Hx4sXw8/NDzZo18f7772Pw4MHYsWNHsV8DIno7B6/H4UW2gIuVIZytDDUdjkZ0/O/Lg91XmUSniuvRo0fIzs6GtbW12nJra2vExub92o+IiMD27duRnZ2N3bt3Y+rUqZg/fz5mz56d73Hmzp0LExMT1cPe3r5Yz4OoPMjKVmDynzmDVPo0rAZvBzMNR0REb4tJ9ApudGtnAMDv5+4hPjn9Na1JkyQSCVxcXODi4gKJRKLpcIioHHiTEWUhISFq7QEgICBA1T4yMhKxsbFqbUxMTODj45PvPgsrKSkJZmb8A4IqtvLYn+/5rxZ4ZRyFrqQs6XLy9iMkpb7QcDREZYdCoYCVlRVWrVoFb29v9O3bF5MnT8bKlSvz3WbSpElISkpSPe7evVuKERO9veLoyzeERCM89hlM9bXxZQdOJkpUEWhpOgAqWT6OZvB2qILz0U/xy4lITOrIX95llVQq5QQ9RFQkBY0oCw8Pz3Ob2NjYAkegKf8tyii1wggODsbWrVuxa9eufNtkZGQgIyND9Zy3f1N5VN768+cZWTh6MwEA0NGr8tVDV6ppaYjaNkYIj32GoOtx6OVdTdMhERU7CwsLyGQyxMXFqS2Pi4uDjU3e739bW1toa2tDJpOplrm5uSE2NhaZmZnQ0dHJtY2uri50dXWLN3iiUvS2fXlccjoWBN0EAHzRvjbMDHK/T4io/OFI9ApOIpFg9H+10TeeikZiKidLIiKi0hUaGopu3bph+vTpaNeuXb7tePs3Uek7FB6PzCwFaloYwNXaSNPhaFQHz5yR+MpJVokqGh0dHXh7e+PgwYOqZQqFAgcPHoSvr2+e2zRr1gy3b9+GQqFQLbt58yZsbW3zTKATETB713U8z8hCPXtT9G3Iz7NEFQWT6JVAa1cr1LYxQkpmNtYHR2s6HMqHEAKJiYlITEyEEELT4RBROfAmI8psbGwKbK/8tyj7LEhYWBjatGmD4cOHY8qUKQW25e3fVBGUt/5cmTDu4GVTbsrPlJQO/43EP37rEZ6ls6QLVUyBgYFYvXo11q9fj+vXr2PkyJFISUnB4MGDAQADBw7EpEmTVO1HjhyJJ0+eYNy4cbh58yZ27dqFb775BqNHj9bUKRCVuLfpy0/efoR/Lj+AVALM7u4JqbRy961EFQmT6JVAzmj0nNroa4MjkZKRpeGIKC8KhQKXLl3CpUuX1EZ6EBHl501GlPn6+qq1B4CgoCBVe0dHR9jY2Ki1SU5OxunTp/PdZ36uXbuG1q1bY9CgQZgzZ85r2+vq6sLY2FjtQVTelKf+PDUzC4dvxAP4/yjsyszFyhBOlgbIzFbgUHi8psMhKhF9+/bFDz/8gGnTpqFevXq4dOkS9u7dqyrjFhMTg4cP/383hr29Pfbt24ezZ8+iTp06GDt2LMaNG4cvv/xSU6dAVOLetC/PyMrG1J05k4l+0MQBnlVNSipEItIA1kSvJDp62WL+/huIepyK387EYGiLmpoOiYiIikFgYCAGDRqEhg0bonHjxli0aFGuEWVVq1bF3LlzAQDjxo2Dn58f5s+fj06dOmHLli04d+4cVq1aBSDni9fx48dj9uzZcHFxgaOjI6ZOnQo7Ozt0795dddyYmBg8efIEMTExyM7OxqVLlwAAzs7OMDQ0RGhoKN555x0EBAQgMDBQVU9dJpPB0tKy9C4QEeXryI0EpL9QoLqZPjzs+KWVRCJBRy9bLDl0G/9r787DoirbP4B/Z4Zl2BFkFWRxAwRBUBFzF9ey7Fdv5ppmWqaVWVmWuaRvvm22uqRlVmraamlC4r6LgguIoiAIKsMisi8DM+f3BzCJgjIjcGbg+7muuZThmXPueYC559zznPvsjMvAY0GG09ueSBuzZ8/G7Nmz6/ze/v3777ovLCwMx48fb+KoiAzfN4dScCW7GG0tTTF3WBexwyGiRsaV6K2ETCrBzIFVvdHXHryCsgqVyBEREVFj0HZFWZ8+fbB582asXbsWgYGB+PXXX7Ft2zb4+/trxsybNw8vvfQSZsyYgZ49e6KoqAiRkZGQy+WaMQsXLkT37t2xaNEiFBUVoXv37ujevTtOnToFAPj111+RnZ2NjRs3wsXFRXPr2bNnM80MEd3PzppWLv5s5VKjZkX+/sRsnr1JREQNlp5bgi/3XgYALHjYFzZmxiJHRESNTSIYQrNGHRQUFMDGxgb5+fk8HbyaslKNAR/tQ0Z+Gf77uD8mhHqIHRLdRqVS4dChQwCAfv36QSaTiRwRETUl5qn74xyRITKUfF5WoULI0igUK1XYNushBLnbih2SXhAEAYM+3o/UmyX4anx3PNLNVeyQSM8xV90f54gMjS65/LnvT2H3hUz09rbDT9N788NpIgPS0DzFleitiImRFNOr27isOZCMSpV+9+kkIiIioqZx8FI2ipUquNrIEejGnq01JBIJRgZUrUaPiFOIHA0RERmC3QmZ2H0hE0ZSCZY+5s8COlELxSJ6K/N0L3fYWZggPbcUO85l3P8BRERERNTiRMRXFYhHBrjwYP8Oo6pbuuy9mIVSJVsgEhFR/UqVKizefh4A8Fw/b3RyshI5IiJqKiyitzLmJkZ49iFPAMCq/UlQq1tkNx8iIiIiqkd5pQq7EzIBAKMCnEWORv/4t7OGWxszlFaocOBSttjhEBGRHlu1PwnXbpXC1UaOl4d0FDscImpCLKK3QpPCPGFpaoRLmUXYfSFT7HComkQigbe3N7y9vbkiTAsqtYBjyTfx55nrOJZ8Eyp+MERERCIyhHx+JCkHheWVcLI2RXf3NmKHo3ckEglG+ld9uBARzzM3iYham4bm8uTsInx94AoAYOHorjA3MWquEIlIBPwLb4VszIwxKcwDq/cnY+X+ZAz1c9Lbg7zWRCqVon379mKHYVAi4zOwZHsCMvLLNPe52MixaLQfRlSfik1ERNScDCGf76zu9T3S3wVSKd8D1mVkgAvWHUrBngtZKKtQQW6snxeIJSKixteQXC4IAhb9eR5KlRoDuzhgeFenZoqOiMTCleit1LMPecHUSIqz6Xk4mnxT7HCItBYZn4GZG2NrFdABQJFfhpkbYxHJlWNERER3UVaqset8VRF9hD9budQnyM0WLjZyFJVX4vDlHLHDISIiPfN3XAYOJ+XAxEiKJY925cJEolaARfRWysHKFE/3dAcArNyXJHI0BFR9kl1QUICCggIIAluS3ItKLWDJ9gTUNUs19y3ZnsDWLkRE1Oz0PZ8fu3ITBWWVaGtpgp6edmKHo7ekUonmQ4ad/GCeiKhVuV8uLyqvxNIdCQCAWQM7wsPeorlDJCIRsIjeik3v7w0jqQRHk28iNu2W2OG0emq1GrGxsYiNjYVarRY7HL0WnZJ71wr02wkAMvLLEJ2S23xBERERQf/zec2ZWsO7OkPGVi73NCqgqjVcVEImlJX697MkIqKmcb9c/lnUJWQWlMPD3hzPD/AWIUIiEgOL6K2YWxtzjOneDgCwal+yyNEQNVxWYf0FdF3GERERtQaVKjX+OV91UfmaAjHVL6R9GzhamaKwrBJHktnShYiIgAsZBfjuaCoAYMmjXXnNDKJWhEX0Vu6FAR0gkQC7L2TioqJA7HCIGsTRSt6gcbbmxk0cCRERkeGITslFbrESbcyNEerFVi73I5VKMLxrVUuXyOqLsRIRUeulVgt4d1s8VGoBI/2dMbCLo9ghEVEzYhG9levoaImR1f0eV+/nanQyDL287OBsc/9C+rxfzuGHY6koq1A1Q1RERET6bedtrVyMZDwMaIiRAVXvk/9JUKBCxZYuRESt2W+x13Dq6i2Ym8jw7iN+YodDRM2M754JLw7sCADYfvYGrt4sFjkaovuTSSUY5udU5/dqurvamhsjs7AcC/88j4Ef7cePx1JRXsliOhERtU4qtYDI+KpWLiPZyqXBennawd7CBHklFThxhddaISJqrfJKlFgecREAMCe8E1xtzUSOiIiaG4voBP92NhjQ2QFqAVhz4IrY4RDdV16JEjvOVa2ms5Ib1fqes40cayYG48TbQ7B0jD9cbORQFJThXRbTiYioFTuVmouconJYy40Q5m0vdjgGw0gmxbDqli41K/mJiKj1+eifROQWK9HZyRJTH/ISOxwiEoFWRfTly5ejZ8+esLKygqOjI8aMGYPExMRaYxQKBSZNmgRnZ2dYWFggODgYv/32W60xubm5mDBhAqytrWFra4tp06ahqKio1phz586hX79+kMvlcHd3x4cffqjjU6SGmD24ajX6bzHXoMjnxRhJv30Q+e8bmJPvhOOn6b3x+dNB+Gl6bxx+czBG+LvA1EiGSb09sP+NgVj6WFc4W8uRkc9iOhERtU4R8VU9vYf6OcPEiOtotDGqpqVLvAIqtSByNERE1NzOpOdhc3QaAGDpY/4wZks0olZJq7/8AwcOYNasWTh+/DiioqJQUVGBYcOGobj43xYgkydPRmJiIv766y/ExcXh//7v//DUU0/h9OnTmjETJkzA+fPnERUVhR07duDgwYOYMWOG5vsFBQUYNmwYPDw8EBMTg48++giLFy/G2rVrG+EpU116etqhl6cdlCo1vjnE1ehikEgk8PT0hKenJyQSyf0f0ErFpt3CT9VvYJaNCYDcWIawDvZ4LKgdwjrYQyatPXemRjJMCvPEgXn1FNOPX2UxnYiIGo0+5nO1WkBkdRG9piBMDdfb2x625sa4WaxEdApbuhARtXS353K1ACzYFgdBAP4vuB1CeTYXUaulVRE9MjISU6ZMQdeuXREYGIgNGzYgLS0NMTExmjFHjx7FSy+9hF69esHb2xsLFiyAra2tZsyFCxcQGRmJb775BqGhoejbty++/PJLbNmyBTdu3AAAbNq0CUqlEuvXr0fXrl3x9NNP4+WXX8aKFSsa8anTnV4c1AEAsOlEGm4VK0WOpvWRSqWaRC2V8pPtulSq1Hjnj3gAwJMhbujlZdfgx9ZbTN8Wz2I6ERE1Gn3M56fT86AoKIOlqRH6dmordjgGx1gmxVDfqmuxRLClCxFRiydAggyVFc7mGWN5xEXEXy+AldwI80f6ih0aEYnogd7Z5+fnAwDs7P4tZPXp0wdbt25Fbm4u1Go1tmzZgrKyMgwcOBAAcOzYMdja2qJHjx6ax4SHh0MqleLEiROaMf3794eJiYlmzPDhw5GYmIhbt27VGUt5eTkKCgpq3Ug7Azo7oKurNUorVPjuaKrY4RDd5YdjV3EhowA2ZsaYP9JHp23UFNP3vzEQ77GYTkRErUBEXFXhN9zXEaZGMpGjMUyjqi/GGhmvgJotXYiIWqzI+Az0/WAvxq07jle2nMH6I6kAgIcDXOBgZSpucEQkKp2L6Gq1GnPmzMFDDz0Ef39/zf0///wzKioqYG9vD1NTUzz//PP4448/0LFjVc9thUIBR0fHWtsyMjKCnZ0dFAqFZoyTk1OtMTVf14y50/Lly2FjY6O5ubu76/rUWi2JRIJZg6p+ThuOpKCovFLkiFoXQRBQXFyM4uJiCAIPzu6UWVCGFVGXAABvjvCBveWDvYGRG8swuZ5i+qCP9mMji+lERKQDfcvngiBo+qGPrC4Ek/b6dLSHldwIWYXliE2re1EPEREZtsj4DMzcGIuM/FLIUQE5KgBU5fKtJ9MRybORiFo1nYvos2bNQnx8PLZs2VLr/nfffRd5eXnYvXs3Tp06hblz5+Kpp55CXFzcAwd7L/Pnz0d+fr7mlp6e3qT7a6mGd3WGt4MFCsoqsen4VbHDaVXUajVOnjyJkydPQq1Wix2O3nlvRwKKyivRvb0tnu7ZeB+S3VlMd7I2xY38MixgMZ2IiHSgb/n83LV8XM8rhbmJDAM6O4gdjsEyNZJpWrrsjKt7UQ8RERkulVrAku0JEABIIcDHKAs+RlmQ4t8PxJdsT+AFpolaMZ2K6LNnz8aOHTuwb98+uLm5ae5PTk7GV199hfXr12PIkCEIDAzEokWL0KNHD6xcuRIA4OzsjKysrFrbq6ysRG5uLpydnTVjMjMza42p+bpmzJ1MTU1hbW1d60bak0klmDmgqjf6ukMpKKtg8ZDEd/BSNv4+lwGpBFg2xh9SaeNfqK2mmH7gjUFY8iiL6URE1DLsrF41N8jHEXJjtnJ5EDUr+SPiM9jShYiohYlOyUVGflm93xcAZOSX8QLTRK2YVkV0QRAwe/Zs/PHHH9i7dy+8vLxqfb+kpKRqo3dcREkmk2lW4oSFhSEvL6/WxUj37t0LtVqN0NBQzZiDBw+ioqJCMyYqKgpdunRBmzZttAmZdDCmezu0szVDTlE5fom5JnY41MqVVaiw8M+qi4k+08cTXV1tmnR/cmMZnulTfzF904mrUFaKv7KQiIjofgRBQET1qulR/mzl8qD6dWoLCxMZMvLLcPZantjhEBFRI0rLLW7QuKzC+gvtRNSyaVVEnzVrFjZu3IjNmzfDysoKCoUCCoUCpaWlAAAfHx907NgRzz//PKKjo5GcnIxPPvkEUVFRGDNmDADA19cXI0aMwPTp0xEdHY0jR45g9uzZePrpp+Hq6goAGD9+PExMTDBt2jScP38eW7duxeeff465c+c27rOnOhnLpJjR3xsA8PWBZFSoWDAk8aw5kIzUmyVwsjbF3KGdm22/9RXT3/kjHoM+ZjGdiIj0X0JGAdJySyA3lmJgF7ZyeVByYxmGVLd0qekzT0REhi23WIlPdiVi8V/nGzTe0UrexBERkb7Sqoi+evVq5OfnY+DAgXBxcdHctm7dCgAwNjbGzp074eDggNGjR6Nbt2744Ycf8P3332PUqFGa7WzatAk+Pj4YMmQIRo0ahb59+2Lt2rWa79vY2GDXrl1ISUlBSEgIXnvtNSxcuBAzZsxopKdN9zO2pzvaWprg2q1SbD97Q+xwqJVKzSnGqv3JAIB3H/GDldy42WO4vZi+eLQfHK1McT2vlMV0IiLSezWr0Ad2doSFqZHI0bQMI/2rWkvujMvQiwvHEhGRbhT5ZVi6IwEP/W8vvtybhNIKNWT3aBsqAeBiI0cvL7vmC5KI9IpW76Yb8kaxU6dO+O233+45xs7ODps3b77nmG7duuHQoUPahEeNSG4sw7N9vfBhZCJW7U/GmKB2TdKHmqg+giBg4V/noaxUo1+ntng4QNzT0OXGMkx5yAtP92qPLdFpWLU/WVNMX7UvGbMGdcSTIW4wMdL5es1ERESNRhAE7Iyr6oc+MqDuawqR9gZ2cYSZsQzXbpXi/I0C+Ldr2jZzRETUuK7eLMaaA1fwW8w1KKvPuvdvZ43ZgzpCrQZmbY7FnZWPmq8Xjfa7Z6GdiFo2VnuoXhN7e8BKboSkrCLsSsi8/wOIGtHOOAUOXsqGiZEU7z3mD4lEP96s1BTTD86rvTL97T/iMOjj/dh8Io0r04mISHSXMotwJacYJkZSDPZxFDucFsPMRIZBPlWtcWo+pCAiIv2XqCjEnC2nMejj/fgpOg1KlRq9vOzw/bO9sH12X4zwd8Gobi5YPTEYTta1W7Y428ixemIwRvD6IkStGs/rpHpZy43xTJgnvtqXhFX7kzC8q5PeFDJbIolEAnd3d83/W7PCsgq8t6OqJ93MAR3g1dZC5IjudvvK9J+i07C6emX623/EYeW+JK5MJyJqpfQln9cUePt3chClHVpLNtLfBTvjFNgZl4E3hndp9e/bDIlKLSA6JRdZhWVwtKpqy8BVpUQt29n0PHy1LwlRty0MHNjFAbMGdURPz7tbs4zwd8EQH0dEHjuHvFIl5nl5I7RDW75WEBGL6HRvUx/yxDeHr+DctXwcTspBv068KFVTkUql6NChg9hh6IXPdl9GZkE5POzNMXOgfs+J3FiGqQ95YVw9xfTZgzviiWAW04mIWgt9yecR8dWtXPzZyqWxDfJxhKmRFKk3S3BRUQhfF2uxQ6IGiIzPwJLtCcjIL9Pc52Ijx6LRflxdStTCCIKAY1duYtW+ZBxOygEASCRVOfHFgR3v24rL2EiG0f26N0eoRGRAWNWhe7K3NMW4Xu0BACv3JYkcDbUGCTcKsOFoKgDgvcf8ITeWiRtQA9UU0w/OG4RFo/3gUN3mZf7vcf+eMsg2L0RE1AySsgpxKbMIxjIJwn2dxA6nxbE0NcKAzlULSyLY0sUgRMZnYObG2FoFdKDqwoIzN8YiMp4/R6KWQBAE7LmQiSdWH8X4dSdwOCkHMqkETwS7IerVAVg1IYTXsiAinbGITvc1vZ83jGUSHL+Si5iruWKH02IJgoCysjKUlZU16CK+LZFaLWDBtjio1AIeDnDRHKAakppi+qF5g7DwERbTiYhaG33I5xFxCgDAQx3bwsacrVyawqjqC57vjFeIHAndj0otYMn2BNT111hz35LtCVCpW+f7b6KWQKUWsP3sDYz8/BCmfX8KsWl5MDGSYnKYB/a/PhCfPBWIjo6WDd6ePuRyItI/LKLTfbnamuH/ursBAFbtSxY5mpZLrVbj+PHjOH78ONTq1llk3XoqHbFpebAwkeHdR/zEDueByI1leLZv3cX0wZ/sx5boNFSoWufPmYioJdOHfB5RXdgdxRYVTWawryOMZRIkZRXhcmah2OHQPUSn5N61Av12AoCM/DJEp3CxEJGhUVaqsfVkGsJXHMBLP53GRUUhLExkeH6ANw6/OQjvPeYPdztzrberD7mciPQPe6JTg7wwsAN+iUnHnotZSLhRAD9X9n6kxnWzqBz/i7gIAHh1aGc428jv8wjDUFNMHx/aHptPpGH1gWRcu1WKt36Pw1f7kjB7UEc8EeIGYxk/0yQiogeXmlOMhIwCyKQSDPVjK5emYi03Rr9ODth7MQsR8Qp0crISOySqR1Zh/QV0XcYRkfhKlSpsOZmGtQevaD4kszU3xtQ+XpjSx5NnYRFRk2DVhhrEq62F5rTV1Qe4Gp0a3/8iLiK/tAK+LtaY0sdT7HAa3e0r0999xA9tLU01xfRBH3NlOhERNY6aVeh9OtijjYWJyNG0bDUXbd3Jvuh6zdGqYQszGjquJVi5ciU8PT0hl8sRGhqK6Ojoesdu2LABEomk1k0ubz1zRfqloKwCK/cloe8HezUXCna0MsWCh31x5M3BeCW8EwvoRNRkWESnBntxYEcAwN/nbiAlp1jkaKglOZmai19irgEAlo3xh1ELXpUtN5Zh2j2K6VtPsphORES6i6i+QOJItnJpckP9nGAkleCiohBXsovEDofq0cvLDjZm9RfVJABcbOTo5WXXfEGJaOvWrZg7dy4WLVqE2NhYBAYGYvjw4cjKyqr3MdbW1sjIyNDcrl692owRE1WdtfzxP4l46H978dE/ibhZrIS7nRn++7g/Ds4bhOf6ecPClI0WiKhptdxKFTU6P1drDPZxhFoAvuZqdGokFSo1FvwRDwB4uqc7QjzaiBxR8zAzqbuY/uZvLKYTEZFu0nNLcO5aPqQSYFhXtnJparbmJujTsS2Af88AIP1zIaMAxeWVdX5PUv3votF+kEkldY5paVasWIHp06dj6tSp8PPzw5o1a2Bubo7169fX+xiJRAJnZ2fNzcmJry/UPDLyS/He9gT0/WAfvtqXhMKySnRytMSnYwOx77WBmBDqAbmxTOwwiaiVYBGdtDJrUAcAwG+x15CRXypyNNQSfHckBYmZhbCzMMGbI3zEDqfZ3V5MX/CwL4vpRESks8jqQm4vLzu0tTQVOZrWYVR1S5eaMwBIv+SVKPHCxhhUqgX4t7OGs3XtNiTONnKsnhiMEa3kzA2lUomYmBiEh4dr7pNKpQgPD8exY8fqfVxRURE8PDzg7u6Oxx57DOfPn2+OcKkVS80pxlu/nUP/D/dh/ZEUlFaoENDOBmsmhuCfOf3xeHe3Fn32MhHpJ57vQloJ8bBDqJcdTqTkYt3BFCwc7Sd2SGTAbuSV4rPdlwEAb430adW9W81MZHiunzcmhHpg04mrWHPgiqaY/tW+JLw0qBMeD27HC5ASEVG9dlYXcmuuY0NNb1hXZ7yzLR7x1wuQdrME7e3NxQ6JqqnVAuZsPYNrt0rR3s4cm6b1hqXcCNEpucgqLIOjVVULl9ayAh0AcnJyoFKp7lpJ7uTkhIsXL9b5mC5dumD9+vXo1q0b8vPz8fHHH6NPnz44f/483Nzc6nxMeXk5ysvLNV8XFBQ03pOgFu2iogCr9iVjx7kbUAtV94V62WH24I7o27EtJJLW8/dKRPqHRXTS2qxBHXEiJRo/Radh1qAOsOdKp0YhkUjg6uqq+X9rsGT7eZQoVejp2QZPBtf9Jry1qauYnp5binm/ncOX+y6zmE5EpOfEyucZ+aU4nZYHiQQY3tW52fbb2tlZmCDUyw5Hk28iIj4Dzw/oIHZIVO2LvZexPzEbpkZSrJ4YrLnYYFgHe5EjMyxhYWEICwvTfN2nTx/4+vri66+/xtKlS+t8zPLly7FkyZLmCpFagNNpt7ByXzJ2X8jU3DeoiwNmDeqIHp7Nf72C1nhsTkT3xyoMaa1fp7YIaGeD0goVNhxNFTucFkMqlaJz587o3LkzpNKW/6e592Im/jmfCZlUgqVj/CFtRauAGqKmmP5vmxcTTTF98Cf78fPJdLZ5ISLSQ2Ll85pWLj082sDpjpYV1LRGVq/8Z190/bEvMQuf76k62/G/jwegq6uNyBHph7Zt20ImkyEzM7PW/ZmZmXB2btiHb8bGxujevTuSkpLqHTN//nzk5+drbunp6Q8UN7VMgiDgaFIOxq87jsdXHcXuC5mQSICHu7ng75f74rupvUQpoAOt79iciBqGrwakNYlEoumNvuFoKgrLKkSOiAxNqVKFRX9V9VKc1tcLPs7WIkekv/4tpg++q5g+5JMD+PkUi+lERARExFUVcEe2kt7O+mR4VydIJMCZ9Dxcz+M1g8SWnluCOVvOQBCA8aHt8WQIz3asYWJigpCQEOzZs0dzn1qtxp49e2qtNr8XlUqFuLg4uLjU/1pjamoKa2vrWjeiGmq1gKiETDy+6ijGf3MCR5NvwkgqwX9C3LB77gCsHB/MD76ISC+xiE46GebnjA4OFigsq8TG42lih9MiCIIApVIJpVIJQRDEDqdJrdyXhPTcUrjYyPHKkE5ih2MQ6iqmp+WWYN6vLKYTEekTMfJ5VkEZTl7NBQCM8Gcrl+bmaCVHz+rVkpFcjS6qsgoVZm6KQX5pBQLdbLCI12+6y9y5c7Fu3Tp8//33uHDhAmbOnIni4mJMnToVADB58mTMnz9fM/69997Drl27cOXKFcTGxmLixIm4evUqnnvuObGeAhkolVrAn2euY9QXhzD9h1M4k54HUyMpngnzwIF5g/DRfwLRwcFS7DABtK5jcyJqOPZEJ51IpRK8OLAjXvvlLL49fAVTH/KE3FgmdlgGTa1W4+jRowCAfv36QSZrmfOZlFWErw8mAwAWjfaDhSlfhrRRU0wfH9oem46n4euDyZpi+ld7kzB7cEc83p0904mIxCJGPv/nvAKCAAS528LV1qzJ90d3G+XvjOiUXETEZWBaXy+xw2m1Fv15HvHXC9DG3BirJobA1Khlvp9+EGPHjkV2djYWLlwIhUKBoKAgREZGai42mpaWVqt9xa1btzB9+nQoFAq0adMGISEhOHr0KPz8+AEFNUx5pQp/xF7HmgPJSL1ZAgCwNDXCpDAPPPuQFxys9O8aa63l2JyItMMqC+ns0SBXtLM1Q06REltPss8d3Z8gCFj4ZzwqVAIGdXHghc8egLmJEab398bBeYPwzqjaK9PDVxzAL6fSUcmV6a3GypUr4enpCblcjtDQUERHR99z/C+//AIfHx/I5XIEBARg586dtb4vCAIWLlwIFxcXmJmZITw8HJcvX6415r///S/69OkDc3Nz2Nra1rmfl19+GSEhITA1NUVQUNCDPEUiuoed1a1cRgUwr4plRHUbnVNXb0GRXyZyNK3Tlug0bD2VDqkE+HJcMNrxA6V6zZ49G1evXkV5eTlOnDiB0NBQzff279+PDRs2aL7+9NNPNWMVCgX+/vtvdO/eXYSoydCUKCux/nAKBny4H2/9HofUmyVoY26M14Z2xpE3B+PNET56WUAnIqoPi+ikM2OZFC8M8AYAfH0gGcpKFuzo3v46ewNHk2/C1EiKJY/680rnjeDOYrq9hQmu3izBG7+ew5B6iukqtYBjyTfx55nrOJZ8Eyo1T1E0ZFu3bsXcuXOxaNEixMbGIjAwEMOHD0dWVlad448ePYpx48Zh2rRpOH36NMaMGYMxY8YgPj5eM+bDDz/EF198gTVr1uDEiROwsLDA8OHDUVb2b2FIqVTiP//5D2bOnHnP+J599lmMHTu2cZ4sEd3lZlE5TqTcBMB+6GJytpEjxKMNgKozA6h5nbuWh4XV19t5bVgX9O3UVuSIiFqv/NIKrNyXhL4f7MN7OxKgKCiDk7UpFjzsi8NvDsZLQzrBxtxY7DCJiLTGPgr0QP7Twx2f70nCjfwy/HnmOv7Tw13skEhP5ZdWYOmOCwCAlwZ3RHt7c5EjallqiukTerfHxuNX8fWBK5pi+lf7kjB7UFWbl90XMrFkewIyblsl52Ijx6LRfppVdGRYVqxYgenTp2t6ma5ZswZ///031q9fj7feeuuu8Z9//jlGjBiBN954AwCwdOlSREVF4auvvsKaNWsgCAI+++wzLFiwAI899hgA4IcffoCTkxO2bduGp59+GgCwZMkSAKi1Wu1OX3zxBQAgOzsb586da7TnTET/2pWQCbUABLSzgbsdc6uYRvo7I+bqLeyMy8AzfTzFDqfVuFWsxMyNsVBWqhHu64SZAzqIHRJRq5RTVI71h1Pw47GrKCyvBAC0tzPHCwM64ImQdmyvREQGjyvR6YHIjWV4rl9V38fVB5K5opXqtWJXInKKyuHtYIHp/b3FDqfFMjcxwoz+HXDozUF4e5RPrZXpYcv34IWNsbUK6ACgyC/DzI2xiIzPEClq0pVSqURMTAzCw8M190mlUoSHh+PYsWN1PubYsWO1xgPA8OHDNeNTUlKgUChqjbGxsUFoaGi92yQi8eyMq3rtHslWLqKruajrydRcZBeWixxN66BSC3hl6xlczyuFh705PnkqEFIpz3Qkak438kqx+K/z6PvBXqzan4zC8kp0drLE508HYe9rAzA+tD0L6ETUIrCITg9sQmh7WMuNcCW7mKevUp3iruXjx+NXAQDLHvPnm6hmcGcx3c7cGNlFyjrH1nz0tWR7Aj8IMzA5OTlQqVSai4HVcHJygkJR9+uxQqG45/iaf7XZZmMpLy9HQUFBrRsR1e9WsRJHk9nKRV+4tTFHoJsN1AKwK4HviZvD57sv4eClbMiNpVgzMQQ2ZmwRQdRcUnKK8eav5zDgo33YcDQVZRVqBLrZYO2kEES+0h+PBbWDkYwlJyJqOfiKRg/MSm6MKdWnrK7clwRBYBGO/qVSC3hnWxzUAvBYkCv6dGSPyuZUU0xfMTbonuMEABn5ZYhOyW2WuIjqsnz5ctjY2Ghu7u5sEUZ0L1EXMqFSC/B1sYZXWwuxwyEAIwOqPsyIiGMRvantvZiJL/YmAQCW/18AfF2sRY6IqHW4kFGA2ZtjMeST/dh6Kh0VKgFh3vbYOC0U22Y9hGFdnXlGCBG1SCyiU6OY8pAXzIxlOH+jAAcuZYsdjkGSSCRwdnaGs7Nzi7rg5uYTV3HuWj6sTI3wzsO+YofTauWXVjRoXGR8BgrKGjaWxNe2bVvIZDJkZmbWuj8zMxPOznW3dnB2dr7n+Jp/tdlmY5k/fz7y8/M1t/T09CbdH1FTaM58HlHTysWfrVz0Rc3P4tiVm8gtrvsMMHpwaTdLMGfLGQDApN4eeLy7m7gBEbUCMVdvYdqGkxj5+SHsOJcBtQAM8XHEbzP74KcZvdG3U9sWcxzbUo/NiejBsIhOjcLOwgTjQ9sDAFbtSxY5GsMklUrh4+MDHx8fSKUt408zu7AcH/6TCAB4fXgXOFrJRY6o9Wro3H9/7CqC34vC+HXH8e3hFFy9WdzEkdGDMDExQUhICPbs2aO5T61WY8+ePQgLC6vzMWFhYbXGA0BUVJRmvJeXF5ydnWuNKSgowIkTJ+rdZmMxNTWFtbV1rRuRoWmufJ5fWoHDSTkAgFHsh643POwt0NXVGiq1gCi2dGkSZRUqvLAxBgVlleje3hbvPuIndkhELZYgCDh8OQfj1h7HE6uPYs/FLEgkwCPdXLDz5X74dkpPhHi0ETvMRtcSj82J6MEZiR0AtRzT+3njh2OpiE7NRXRKLnp52YkdEons/Z0XUFhWiYB2NpjY20PscFq1Xl52cLGRQ5FfhvoaLlmYyuBkZYorOSU4mnwTR5NvYumOBHRwsEC4rxOG+DohuL0texvqmblz5+KZZ55Bjx490KtXL3z22WcoLi7G1KlTAQCTJ09Gu3btsHz5cgDAK6+8ggEDBuCTTz7Bww8/jC1btuDUqVNYu3YtgKqVN3PmzMGyZcvQqVMneHl54d1334WrqyvGjBmj2W9aWhpyc3ORlpYGlUqFM2fOAAA6duwIS0tLAEBSUhKKioqgUChQWlqqGePn5wcTE5PmmSCiFmrPhUxUqAR0crRER0crscOh24wKcMH5GwXYGafA2J7txQ6nRREEAe/8EY+EjALYW5hg1YRgmBjxfQlRY1OrBey+kImV+5NxNj0PAGAsk+D/urvh+QHe8HawFDdAIiIRsIhOjcbZRo4nQ9zwU3Q6Vu67DJW6I7IKy+BoJUcvLzvI2BftngRBgFqtBlD1ybehnzZ2NDkHf5y+DokEWDbGnz9/kcmkEiwa7YeZG2MhAWoV0mt+Mp/8JxAj/F2QmlOMPRezsOdCJqJTcpGcXYzk7Cv4+uAV2JobY1AXRwzxdUT/zg6wlvMCXmIbO3YssrOzsXDhQigUCgQFBSEyMlJzYdC0tLRaK2j69OmDzZs3Y8GCBXj77bfRqVMnbNu2Df7+/pox8+bNQ3FxMWbMmIG8vDz07dsXkZGRkMv/PaNh4cKF+P777zVfd+/eHQCwb98+DBw4EADw3HPP4cCBA3eNSUlJgaenZ6PPBZE+aK58vrO653ZND27SHyP8nfHRP4k4kpSD/JIK2JgzVzaWzdFp+C32GqQS4Mtx3eFiYyZ2SEQtSqVKjb/jMrBqXzISMwsBAHJjKZ7u2R4z+nvD1bZ1/M21tGNzImocEqGFXgWyoKAANjY2yM/P5+ngzejqzWIM/Gj/XStdXWzkWDTaDyP8eaBXH5VKhUOHDgEA+vXrB5lMJnJEulNWqjHy84NIzi7GxN7tsWxMgNghUbXI+Aws2Z6AjPwyzX33+vvML63AwUvZ2HMhE/sSs2v1VjeSShDqbYfBPk4I93WEhz0vaqcN5qn74xyRIWqOfF5UXongpVFQVqoROacffJz596Fvhn96EImZhfjkP4F4IoT9uhvDmfQ8PLXmGJQqNd4c4YOZAzuIHRIA5qqG4Bzpv/JKFX6LuY41B5KRllsCALAyNcKkMA8829cLbS1NRY6webWkY3Miur+G5imuRKdGdSGjoM5WEYr8MszcGIvVE4NZSG8F1h26guTsYrS1NMEbw33EDoduM8LfBUP9nBGdktugM0VszIwxOtAVowNdUalSI+bqLey9mIXdFzKRnF2MI0k3cSSpqu1LR0dLDPF1RLivE4Lbt+HZB0RETWTvxSwoK9XwbmuBLk5s5aKPRgY4IzGzEBHxGSyiN4LcYiVe3BgDpUqNYX5OeGGAt9ghERkMlVqo971/ibISm0+kYd2hK8gsKAdQdb2zZx/yxKQwT9iY8UwaIqIaLKJTo1GpBSzZnlDn9wRUtYxYsj0BQ/2cWVxrwdJzS/Dl3ssAgHce9uUbLz0kk0oQ1sFe68cZyaQI9bZHqLc95o/yRWpOMXZfyMSeC1k4mZqLpKwiJGUV4esDV9Cmuu3LYLZ9ISJqdBFxGQCqCrU8xVw/jQpwwWe7L+PgpRwUllXAinlQZyq1gJd/Oo0b+WXwamuBj58K5O89UQPVdxbq68O74MatUqw/koJbJVVnmjpbyzGjvzee7uUOcxOWioiI7sRXRmo00Sm5tZLznQQAGflliE7J1amAR/pPEAQs/us8yirU6O1thzFB7cQOiZqQZ1sLPNfPG8/180Z+aQUOVLd92Z+YjVslFfj99HX8fvq6pu3LEB8nhPs6ob29udihExEZrBJlJfYlZgEARvLsPr3VydESHRwskJxdjL0Xs/AY3xPp7NOoSziclAMzYxnWTAzhB/NEDRQZn4GZG2PvOlM8I78Mr/18VvO1h705Zg7ogMeD28HUiG1LiIjqwyI6NZqswvoL6LqMI8MTlZCJPRezYCyTYNkYf64SakVszIzxaKArHr2t7cue6rYvV25r+/LejgR0crTEEN+qPurd2faFiEgr+xOzUVahhrudGbq6srewvpJIJBgV4IIv9yZhZ1wGi+g6ikrIxFf7kgAA/3siAF2c2b6IqCFqzhK/1wXwjKQSfPRkN4wOdIWRTHqPkUREBLCITo3I0UreqOPIsJQoKzXtfKb380ZHRx7ktFa3t315e5QvUnKKsae67Ut0ai4uZxXhclYR1hxI1rR9GeLrhP6d2/J0dyKi+9hZ3cpllL8LP6zWcyP9q4ro+xOzUVxeCQtTHnppIzWnGHN/PgMAmNLHkx9EEGnhfmeJA0ClWoCzjRkL6EREDcR3ctRoennZwcVGDkV+Wb2feLvYVF3IhFqez/dcxvW8UrSzNcNLgzuJHQ7pEa8Gtn0xlkkQ6mWvuTipux3bvhAR3a6sQoW9F6tbuQSwlYu+83Wxgoe9Oa7eLMG+xCw80s1V7JAMRqlShRc2xqCwrBIhHm3w9ihfsUMiMig8S5yIqPGxiE6NRiaVYNFoP8zcGAsJUGchPcjdlq0b6iGRSODg4KD5vyFJVBTi20MpAIAlj3aFmQl76VHd7mz7curqLc0q9Ss5xTiclIPDSTlYsj0BnZ0sMdiHbV+IyLA0ZT4/eCkbJUoVXG3kCHSzadRtU+OTSCQY6e+CNQeSERGvYBG9gQRBwNt/xOGiohBtLU2wcnwwTIy4UpZIGzxL/MEY8rE5ETUdFtGpUY3wd8HqicF3XQHcWm6EgrJKRMQr8OWey3hpCFcq30kqlaJr165ih6E1QRDw7rZ4VKoFDPVzQrifk9ghkYEwkknR29sevb3t8c7DfriSXYS91X3UT6bewqXMIlzKrGr7YmdhgoFdHBDu64R+ndj2hYj0V1Pm84h4BYCqVeg8qDcMowKcseZAMvZdzEKpUsWFBg2w8fhV/HH6OmRSCb4cFwxnGxb5iLSVfqvknt+XAHDmWeL1MtRjcyJqWiyiU6Mb4e+CoX7OiE7JRVZhGRytqpLzt4ev4P2dF/FJ1CWYGksxo38HsUOlRvBb7HVEp+bCzFiGRaP9xA6HDJi3gyW8HSyr2r6UVGD/pSzsuZCF/YlZyC1W4vfY6/g9lm1fiKh1Kq9UYXdCJoCqwiwZhoB2Nmhna4breaU4cCkbI/z5s7uX2LRbeG9H1TV23hzRBWEd7EWOiMjw/B57DW/+dk7z9Z1nidd8BLtotB/P9CQi0gKL6NQkZFLJXW96Z/TvgPIKNT6JuoT3d16EqZEMz/TxFCdAahR5JUq8v/MCAODlIZ3g1obFTGocNubGeCyoHR4LaocKlRqnUqvbvlzMQkodbV+G+Fa1fQlyZ9sXImqZjiTloLC8Ek7Wpuju3kbscKiBJBIJRgU4Y92hFETEZ7CIfg85ReWYtSkWFSoBI/2dMb2ft9ghERmcP05fw2u/nIUgABNC2+OhDm2x9O/aZ4k728ixaLQfRvjz2hpERNpgEZ2a1UtDOkGpUuPLvUlY9Nd5GMukGB/aXuyw9IJKpcKhQ4cAAP369YNMpv+n+374TyJyi5Xo5GiJaX29xA6HWihjmRRhHewR1sEeCx6pavuy50JV25dTV/9t+7J6P9u+EJH4miqf74yrauUyoqszpPyw0KCMDHDBukMp2HMhC2UVKsiN9f89XnOrVKnx8k+nkZFfBm8HC3z4ZDe2LCLS0rbT1/Haz1UF9HG92mPpY/6QSiUY7n/3WeJcdHJvhnhsTkRNj0V0anZzh3ZGeaUaaw9ewTvb4mBiJMWTIW5ih0VaOp12Cz9FpwEAlo3x5wWfqNnUtH2Z3t8beSVKHLiUXW/bl97e9hji44ghbPtCRAZMWanGrvP/9kMnwxLkZgsXGzky8stw+HIOrx9Th0+iLuFo8k2Ym8jw9cQQfghOpKU/z1zH3J/PQC0A43q5479j/DUfuNZ1ljgREWmPRXRqdhKJBPNH+kBZqcaGo6mY9+tZmBhJ8Wigq9ihUQNVqtR45494CALwRLAbQr35pozEYWtucs+2L4cu5+DQ5Rws3p6ALk5WGOLriCFs+0JEBubYlZsoKKtEW0sT9PTkReAMjVQqwfCuzthwNBU74zNYRL/DP+cVWL0/GQDwwRPd0MnJSuSIiAzLX2dv4NWtVQX0p3u6479jAnjGEhFRE2ARnUQhkUiwaLQfyivV+Ck6Da9uPQMTmYR92QzED8euIiGjADZmxpg/ykfscIgA3N32JTm7CHsuZGL3hSzEXL2FxMxCJGYWYtX+ZNhbmGBgF0eE+zqiX2cHWJrePx2q1AJPhSUiUUTGZwAAhnd15uuOgRoV4IINR1OxOyETyko1z+CrdiW7CK//fBYA8OxDXhjNRTVEWtl+9gbmbDkNtQA81cMN7z/OAjoRUVPR6t3b8uXL0bNnT1hZWcHR0RFjxoxBYmKi5vupqamQSCR13n755RfNuLS0NDz88MMwNzeHo6Mj3njjDVRWVtba1/79+xEcHAxTU1N07NgRGzZseLBnSnpHIpHgv2P88USwG1RqAS/9dBp7LmSKHRbdR2ZBGVZEXQIAzBvRBW0tTUWOiKhuHRwsMaN/B/z8fBhiFoTj86eDMDrQFVZyI9wsVuK32GuYuSkWwe9FYdK3J/D90VSk55bUua3I+Az0/WAvxq07jle2nMG4dcfR94O9msIWEVFTqVSp8c/5qvdHo9jKxWCFeLSBg5UpCsoqcTQ5R+xw9EKJshIzN8aisLwSPT3bcGEGkZZ2nLuBOdUr0P8T4ob//V83FtCJiJqQVkX0AwcOYNasWTh+/DiioqJQUVGBYcOGobi4GADg7u6OjIyMWrclS5bA0tISI0eOBFB1gYaHH34YSqUSR48exffff48NGzZg4cKFmv2kpKTg4YcfxqBBg3DmzBnMmTMHzz33HP75559GfOqkD6RSCT58shtGB7qiQiVg5sZYHLyULXZYdA9LdySgqLwSge62GNeTF4Ulw1DT9uXLcd0R++5QbJ4eiml9veBpbw6lSo1Dl3Ow6K/z6PfhPgz/9CA+jLyImKu3oFILiIzPwMyNscjIL6u1TUV+GWZujGUhnYiaVHRKLnKLlWhjboxQL7ZyMVQyqQQjujoDACKqLxLbmgmCgPm/xyExsxAOVqZYOT4YxjKuzidqqL/PZeCVLWegUgt4MsQNHzzBAjoRUVPTqp1LZGRkra83bNgAR0dHxMTEoH///pDJZHB2dq415o8//sBTTz0FS0tLAMCuXbuQkJCA3bt3w8nJCUFBQVi6dCnefPNNLF68GCYmJlizZg28vLzwySefAAB8fX1x+PBhfPrppxg+fPiDPF/SQzKpBCueCoSyUoV/zmdixo+n8N2UXrz4iR46dDkbO85lQCpBrYvVEBkSY5kUfTq0RZ8ObbHgYV8kZxdj78Wqti+nUnNrtX2xMzdGaYUaQh3bEQBIACzZnoChfmyxQERNY+dtrVyMWGQ0aCMDnPHj8av4J0GBZSr/Vl00/uHYVfx55gZkUglWjg+Go7Vc7JCIDEZEXAZe3nIaKrWA/wtuxwI6EVEzeaB3bvn5+QAAO7u6V8XExMTgzJkzmDZtmua+Y8eOISAgAE5O/15QZ/jw4SgoKMD58+c1Y8LDw2tta/jw4Th27Fi9sZSXl6OgoKDWjQyHsUyKL8cFY7CPI8oq1Jj2/UmcSs0VO6xmJZFIYGdnBzs7O0gk+vcmqKxChXe3xQMAJod5wr+djcgRET04iUSCjo7/tn2JfXcoPhsbhEe6ucBKboTckgqUVqjqfbwAICO/DNEprev1iojq15j5vOpsmKpWLiP8ne8zmvRdL0872FuYIK+kAieutN68EXM1F0t3JAAA5o/0QS+eYUHUYJHxGXjpp+oCevd2+OjJQC7kaAL6fmxOROLQuYiuVqsxZ84cPPTQQ/D3969zzLfffgtfX1/06dNHc59CoahVQAeg+VqhUNxzTEFBAUpLS+vc1/Lly2FjY6O5ubu76/rUSCQmRlKsmhCMfp3aokSpwpTvTuJMep7YYTUbqVSKbt26oVu3bpBK9W9l0tcHriD1ZgkcrUzx2rDOYodD1CRszU0wpns7fDU+GLHvDsXsQR0a9LiswrL7DyKiVqEx8/mp1FzkFJXDWm6EPh3aNlKEJBYjmRTDqlu67GylrcCyC8vx4qZYVKoFPNzNBdP6eokdEpHBiIxXYPbm06hUC3i8ezt89B8W0JuKvh+bE5E4dH41mDVrFuLj47Fly5Y6v19aWorNmzfXWoXelObPn4/8/HzNLT09vVn2S41LbizD2kk90NvbDkXllZj87Qmcv5EvdlitXmpOMVbuTwIAvPuIH6zkxiJHRNT0jGVSPNTRoUFjHa14GjoRNb6I+KoFJkP9nGFixIP4lmBk9RkF/8QroFLX1Sys5apUqfHST7HILChHR0dLfPBEN67wJGqgf84rMHtz1QdQjwW54mMW0ImImp1O78Znz56NHTt2YN++fXBzc6tzzK+//oqSkhJMnjy51v3Ozs7IzMysdV/N1zX91OsbY21tDTMzszr3Z2pqCmtr61o3MkxmJjJ8+0xPhHi0QUFZJSZ+cwKJikKxw2q1BEHAwr/OQ1mpRt+ObfFINxexQyJqNr287OBiI0d9hygSAC42cp6KTkSNTq0WEFG9WnlUAFu5tBRhHexhY2aMm8XKVtcK7KN/EnH8Si4sTGRYMzEElqZaXZ6LqNXadV6BWdVncDwa6IpPWEAnIhKFVkV0QRAwe/Zs/PHHH9i7dy+8vOo//e7bb7/Fo48+CgeH2qv4wsLCEBcXh6ysLM19UVFRsLa2hp+fn2bMnj17aj0uKioKYWFh2oRLBszC1AjfTe2JQDcb3CqpwIRvjiMpq0jssJqUSqXCwYMHcfDgQahU9fdgbm4R8QocvJQNE5kU7z3WlSuGqFWRSSVYNLoqN935m1/z9aLRfjyQISKNxsrnp9PzkFlQDktTI/TtxFYuLYWxTIphflVtKyNbUUuXiLgMfH3wCgDgo/8EoqOjpcgRERmG3QmZmFW9An10oCtWPBXIi0w3A309NicicWn16jtr1ixs3LgRmzdvhpWVFRQKBRQKxV19ypOSknDw4EE899xzd21j2LBh8PPzw6RJk3D27Fn8888/WLBgAWbNmgVTU1MAwAsvvIArV65g3rx5uHjxIlatWoWff/4Zr7766gM8VTI01nJj/PBsKPxcrJFTpMSEb47j6s1iscNqUmq1Gmq1WuwwNIrKK/He9qoLP70wsAO8HXjAQ63PCH8XrJ4YDGeb2i1bnG3kWD0xGCP8eXYGEdXWGPk8Iq6qwBru6whTI1ljhEV6YlRAVd6IiFdA3QpauiRnF+GNX88BAKb389I8fyK6tz0XMjFzUwwqVAIe6eaCT1lAb1b6dmxOROLT6hy61atXAwAGDhxY6/7vvvsOU6ZM0Xy9fv16uLm5YdiwYXdtQyaTYceOHZg5cybCwsJgYWGBZ555Bu+9955mjJeXF/7++2+8+uqr+Pzzz+Hm5oZvvvkGw4cP1yZcagFszI2x8blQPL32GC5lFmH8uhPY+nxvuLUxFzu0VuHTqEtQFJShvZ05XhzYsAssErVEI/xdMNTPGdEpucgqLIOjVVULF65AJ6KmIAiCph/6SBYcW5w+He1hJTdCVmE5YtNuoYdny20JVlxeiRd+jEFReSV6ednhzRE+YodEZBD2XszEzI2xqFBVXYT3s7FBLKATEYlM63Yudd1uL6ADwPvvv4+0tLR6r2Ls4eGBnTt3oqSkBNnZ2fj4449hZFS7nj9w4ECcPn0a5eXlSE5Ovmsf1HrYWZhg03O94e1gget5pRi/7gQy8kvv/0B6IAk3CrDhaCoA4L3HukJuzFVw1LrJpBKEdbDHY0HtENbBngV0Imoy567l43peKcxNZBjQuWEXOCbDYWokw1DfqpYuO+MUIkfTdARBwJu/ncPlrCI4Wpniq/HdWQTUEytXroSnpyfkcjlCQ0MRHR3doMdt2bIFEokEY8aMadoAW7l9F7Pwwo+xUKrUeDjABZ+zgE5EpBf4SkwGwcHKFJuf6432duZIyy3BhHUnkFVYJnZYLZZaLWDBtjio1AJGBThjYBdHsUMiIiJqNXZW98oe5OPID7FbqJGali4ZLbaly3dHUrHjXAaMpBKsmhAMRyv5/R9ETW7r1q2YO3cuFi1ahNjYWAQGBmL48OG1rllWl9TUVLz++uvo169fM0XaOu1LzMLzP8ZAqVJjpL8zPnuaBXQiIn3BV2MyGM42cmyeHop2tma4klOMCetO4GZRudhhtUg/n0pHbFoeLExkePcRP7HDISIiajUEQUBE9erkUbzmQovVr1NbWJjIkJFfhrPX8sQOp9GdTM3F+zsvAADeedi3RbesMTQrVqzA9OnTMXXqVPj5+WHNmjUwNzfH+vXr632MSqXChAkTsGTJEnh7ezdjtK3L/tsK6CO6OuOLcd1hzAI6EZHe4CsyGRS3NubYPD0UztZyXM4qwsRvo5FXohQ7rBYlt1iJ/0VeBAC8OrQzXGzMRI6IiIio9Th/owBpuSWQG0sxsAtbubRUcmMZBle3dKnpf99SZBWU4cVNsahUC3g00BVT+niKHRJVUyqViImJQXh4uOY+qVSK8PBwHDt2rN7Hvffee3B0dMS0adMatJ/y8nIUFBTUutG9HbiUjRk/xkBZqcbwrk74cjwL6ERE+oavymRwPOwtsGl6KNpamuJCRgEmr49GQVmF2GE1CltbW9ja2ooaw/8iLiCvpAI+zlY86CEiItLBg+TzyOqC6sDOjrAwNbrPaDJko/ydAVS1dBGEltHSpUKlxuzNp5FdWI7OTpZY/n8BkEh4DRF9kZOTA5VKBScnp1r3Ozk5QaGo+8Ocw4cP49tvv8W6desavJ/ly5fDxsZGc3N3d3+guFu6g5eyMf2HU1BWqjHMzwlfjgtmAV0P6MOxORHpF74yk0Hq4GCJzdNDYWdhgnPX8jFlfTSKyivFDuuByGQyBAUFISgoCDKZOP1PT6Xm4udT1wAA/33cn/33iIiItPQg+VwQBOyMq+qHPjLAuSnCIz0ysIsjzIxlSM8txfkbLWOl7gcRFxGdmgtLUyOsmRjCD4IMXGFhISZNmoR169ahbdu2DX7c/PnzkZ+fr7mlp6c3YZSG7fDlHE0BfaifE74aHwwTIx6DiU0fjs2JSP/w1ZkMVmcnK2ycFgobM2PEpuVh2oaTKFWqxA7LYFWo1Hjnj3gAwNge7gjxYO9KIiKi5nQpswhXcophYiTFYB9e1LulMzORYZBPVcuemg9PDNnf5zLwzeEUAMDH/+kGbwdLkSOiO7Vt2xYymQyZmZm17s/MzISz890f3CUnJyM1NRWjR4+GkZERjIyM8MMPP+Cvv/6CkZERkpOT69yPqakprK2ta93obkeScjDt+5Mor1Qj3NcRK1lAJyLSa3yFJoPm52qNH57tBStTI5xIycWMH0+hrIKFdF1sOJKKxMxCtDE3xlsjfcQOh4iIqNWpKaT279QWVnJjkaOh5jCy+uKxO+MMu6VLUlYh3vj1LADg+QHeGMGL4uolExMThISEYM+ePZr71Go19uzZg7CwsLvG+/j4IC4uDmfOnNHcHn30UQwaNAhnzpxhm5YHcPS2AvoQH0esnMACOhGRvuOrNBm8QHdbbHi2J8xNZDh0OQczN1ZdkMXQqFQqHDlyBEeOHIFK1bwfBNzIK8Wnuy8BAOaP9EUbC5Nm3T8REVFL8SD5PCK+upULC5CtxiAfR5gYSZF6swQXFYVih6OTovJKPP9jDEqUKoR52+ONYV3EDonuYe7cuVi3bh2+//57XLhwATNnzkRxcTGmTp0KAJg8eTLmz58PAJDL5fD39691s7W1hZWVFfz9/WFiwmMGXRxNzsGz359EWYUag30csWpiMEyN2DJEn4h5bE5E+otFdGoRQjzssH5KT8iNpdiXmI2XfopFhcrwCukVFRWoqGj+i6S+tz0BJUoVQjza4MkQt2bfPxERUUuiSz5PyirEpcwiGMskCPd1uv8DqEWwNDXCgM5VLV0iDLCliyAImPfrWSRnF8PZWo4vxnXnNXX03NixY/Hxxx9j4cKFCAoKwpkzZxAZGam52GhaWhoyMgzvd9FQHEu+iWc3VBXQB3VxwGoW0PWWWMfmRKS/+A6HWoze3vb4ZnJPmBhJ8c/5TLy69QwqDbCQ3tz2XcxC5HkFZFIJlo3xh1QqETskIiKiViciTgEAeKhjW9iYs5VLazKq+iKyO+MVIkeivW8Pp2BnnALGMglWTgiGg5Wp2CFRA8yePRtXr15FeXk5Tpw4gdDQUM339u/fjw0bNtT72A0bNmDbtm1NH2QLdPzKvwX0gV0csHpiCAvoREQGhEV0alH6dmqLNRODYSyTYMe5DMz79RzUasPtL9nUyipUWPhX1cVEn33IE74uvOgPERGRGCKqC6ij2Mql1Rni6wRjmQRJWUW4nGk4LV1OXLmJ5REXAQDvPuKHEI82IkdEpL9OXLmJqd+dRGmFCgM6O2DNxBDIjVlAJyIyJCyiU4sz2McJX44Lhkwqwe+nr+PtP+JYSK/Hyn1JSM8thbO1HHPCO4sdDhERUauUmlOMhIwCyKQSDPVjK5fWxlpujH6dqlu6GMhq9MyCMszafBoqtYAxQa6Y1NtD7JCI9FZ0Si6mbqgqoPfr1BZfT2IBnYjIELGITi3SCH9nfDY2CFIJsOVkOpZsPw9BYCH9dsnZRVhzIBkAsGi0HyxMjUSOiIiIqHWqKZz26WDPi3u3UiP9q1u6GEBf9AqVGrM2xSKnqBxdnKzw/v8FQCJhO0CiupxMzcWU76JRoqwqoK+b3IMFdCIiA8UiOrVYowNd8fF/AiGRAN8fu4r3d15gIb2aIAh4d1s8KlQCBnZxwIjqAzciIiJqfhHxVYVT5uPWa6ifE4ykElxUFOJKdpHY4dzT+zsv4NTVW7AyNcKaSSEwN+FCDKK6nErNxZT1VQX0vh1ZQCciMnQsolOL9n/Bbnj/8QAAwLpDKfhk1yWRI7o3KysrWFlZNfl+/jp7A0eTb8LUSIr3HvXn6iEiIqJGpE0+T88twblr+ZBKgGF+LKK3VrbmJgjrYA9Av1u6/HX2Br47kgoA+OSpQHi1tRA3ICI9FXM1F8+sj0axUoWHOtqzgG6AmuvYnIgMB5cNUIs3rld7VKjUWPjneXy1LwmmRlK8NKST2GHdRSaTISQkpMn3U1BWgWV/XwAAzBrUEe3tzZt8n0RERK2Ftvk8srpg2svLDg5Wpk0VFhmAUQEuOHQ5BxHxGZg1qKPY4dzlUmYh3vz1HADgxYEdMKwrP/QhqkvM1Vt4Zv1JFCtV6NPBHt9M7gkzExbQDUlzHZsTkWHhSnRqFSaHeWLBw74AgE+iLuHr6l7grdEn/yQiu7Ac3m0t8PwAb7HDIaJGsHLlSnh6ekIulyM0NBTR0dH3HP/LL7/Ax8cHcrkcAQEB2LlzZ63vC4KAhQsXwsXFBWZmZggPD8fly5drjfnvf/+LPn36wNzcHLa2tnXuJy0tDQ8//DDMzc3h6OiIN954A5WVlQ/0XIlamp3VrVxGBbiIHAmJbZifE6QSIP56AdJulogdTi2FZRV44ccYlFZUrap9bVgXsUMi0kuxabfwzPpoFJVXIszbHt8+wwI6EVFLwSI6tRrP9fPGG8Or3vAvj7iI746kiBxR84u7lo8fj18FACwd4w9TI76hIzJ0W7duxdy5c7Fo0SLExsYiMDAQw4cPR1ZWVp3jjx49inHjxmHatGk4ffo0xowZgzFjxiA+Pl4z5sMPP8QXX3yBNWvW4MSJE7CwsMDw4cNRVlamGaNUKvGf//wHM2fOrHM/KpUKDz/8MJRKJY4ePYrvv/8eGzZswMKFCxt3AogMWEZ+KU6n5UEiAYZzVW+rZ29pit7eVS1dIs/rzwVGBUHAG7+cw5WcYrjYyPHF090hk7IVINGdTqfdwjPfVhXQe3vb4dspPVhAJyJqQVhEp1Zl1qCOeHlw1emxS7YnYNOJqyJH9C+VSoXjx4/j+PHjUKlUjb99tYAF2+KgFoBHA13xUMe2jb4PImp+K1aswPTp0zF16lT4+flhzZo1MDc3x/r16+sc//nnn2PEiBF444034Ovri6VLlyI4OBhfffUVgKpiyWeffYYFCxbgscceQ7du3fDDDz/gxo0b2LZtm2Y7S5YswauvvoqAgIA697Nr1y4kJCRg48aNCAoKwsiRI7F06VKsXLkSSqWy0eeBSF9ok89rWrn08GgDJ2t5c4RHem5k9RkJO+P0py/62oNXEHleAWOZBKsmBMPekm2HiO50Jj0Pk7+NRmF5JUK97LB+Sk9edNeANfWxOREZJhbRqdV5dWhnTRuTd/6Ixy+n0kWO6F9lZWW1Vno2ps3RaTh7LR9Wpkaa1jZEZNiUSiViYmIQHh6uuU8qlSI8PBzHjh2r8zHHjh2rNR4Ahg8frhmfkpIChUJRa4yNjQ1CQ0Pr3WZ9+wkICICTk1Ot/RQUFOD8+fN1Pqa8vBwFBQW1bkSGqKH5PKK6UDrSn61cqMrwrk6QSKoKctfzSsUOB0eTc/BB5EUAwMLRXdG9fRuRIyLSP2fT8zDp2xMoLK9ELy87fDeVBfSWoCmPzYnIMLGITq2ORCLBWyN8MKWPJwBg3m/n8OeZ6+IG1cSyC8vxYfUB0GvDOsORq92IWoScnByoVKpahWoAcHJygkJR9ypGhUJxz/E1/2qzTW32c/s+7rR8+XLY2Nhobu7u7g3eH5GhySoow8mruQCAEf5s5UJVHK3k6OlpB+DfMxXEosgvw8s/nYZaAP4vuB0mhrYXNR4ifXTuWh4mfnsChWWV6OVph++4Ap2IqMViEZ1aJYlEgkWj/TA+tD0EAZj781lExOlP78nGtnznBRSWVaKrqzUmhXmKHQ4RUZ3mz5+P/Px8zS09XX/OFCJqbP+cV0AQgCB3W7jamokdDumRkdUfqoj53lRZqcaLm2KQU6SEj7MV/jsmABIJ+6AT3S7uWj4mflNVQO/p2QbfTe0JC1MW0ImIWioW0anVkkgkWPaYP54McYNKLeCln05jd0Km2GE1umPJN/H76euQSID/Ph7AC0ERtSBt27aFTCZDZmbt167MzEw4O9e9stXZ2fme42v+1Wab2uzn9n3cydTUFNbW1rVuRC1VTc/rUQFchU611ZyZcOrqLSjyxWkl8N+/ExCblgcruRG+nhTCiyMS3SH+ej4mfnsCBWWV6OHRBt9N7cUCOhFRC8ciOrVqUqkEHzzRDY8GuqJSLeDFTbE4cClb7LAajbJSjXf/jAcAjO/VHkHutuIGRESNysTEBCEhIdizZ4/mPrVajT179iAsLKzOx4SFhdUaDwBRUVGa8V5eXnB2dq41pqCgACdOnKh3m/XtJy4uDllZWbX2Y21tDT8/vwZvh6glyikqx4mUmwDYD53u5mJjhuD2tgCqzlhobttOX8f3x64CAD4bGwQPe4tmj4FIn8Vfz8eEb04gv7QCIR5tsOHZXrBkAZ2IqMVjEZ1aPZlUghVPBWKkvzOUKjVm/HAKR5NzxA6rUXxz+AqSsorQ1tIE84b7iB0OETWBuXPnYt26dfj+++9x4cIFzJw5E8XFxZg6dSoAYPLkyZg/f75m/CuvvILIyEh88sknuHjxIhYvXoxTp05h9uzZAKrO0pkzZw6WLVuGv/76C3FxcZg8eTJcXV0xZswYzXbS0tJw5swZpKWlQaVS4cyZMzhz5gyKiooAAMOGDYOfnx8mTZqEs2fP4p9//sGCBQswa9YsmJqaNt8EEemhXeczoRaAgHY2cLczFzsc0kOjAqo+XImIb96WLhcVBXjr93MAgJcGd8QQX6f7PIKodTl/o2oFen5pBYLb22LD1J4soBMRtRJ8tScCYCST4vOnu6NiUwx2X8jCtA2n8MO0XpoLOzUXc/PGO5BOzy3BF3suAwDeHuULG3PjRts2EemPsWPHIjs7GwsXLoRCoUBQUBAiIyM1F/FMS0uDVPrvZ+Z9+vTB5s2bsWDBArz99tvo1KkTtm3bBn9/f82YefPmobi4GDNmzEBeXh769u2LyMhIyOX/XpR44cKF+P777zVfd+/eHQCwb98+DBw4EDKZDDt27MDMmTMRFhYGCwsLPPPMM3jvvfeaekqIRHe/fF5TGB3JVi5UjxH+zlj29wVEp+Qiu7AcDlZN/+FjQVkFXvgxBmUVavTr1BZzwjs3+T6JDEnCjQJM+OYE8koq0L29Lb5/thes5DzGaqka89iciFoGiSAIgthBNIWCggLY2NggPz+fPVWpwcorVZj+QwwOXsqGpakRfpzWC93btxE7LJ089/1J7L6QhVAvO2yZ0ZsXgyLSM8xT98c5opboVrESPf67Gyq1gH2vD4RXW7bKoLo99tVhnL2Wj/8+7o8JoR5Nui+1WsDzG2MQlZCJdrZm2P5SX9hZmDTpPlsK5qr7awlzVFVAP45bJRUIcrfFD9N6wZoFdCKiFqGheYrtXIhuY2okw9cTQxDmbY+i8kpMXh+N+Ov5YoeltV3nFdh9IQtGUgmWjfFnAZ2IiEhPRF3IhEotwMfZigV0uqeRNS1d4pq+L/qag8mISsiEiUyKVROCWUAnus2FjH8L6IEsoBMRtVosohPdwcxEhm+e6YEeHm1QWFaJSd+ewEVFgdhhNViJshJLticAAKb390YnJyuRIyIiIqIaEXFVrVxqel4T1Wekf1W7n2NXbiK3WNlk+zmSlIOP/0kEACx+tCsCeSF6Io2LiqoWLrdKKhDoZoMfnmUBnYiotWIRnagOFqZG+G5qTwS62+JWSQUmfnMCSVlFTbpPlUqF6OhoREdHQ6VS6bydL/Yk4XpeKdrZmuGlwR0bMUIiIiK6n3vl8/zSChxOqrp4+Sj2Q6f78LC3gJ+LNVRqAVEJTbMa/UZeKV766TTUAvCfEDeM6+XeJPshMkSJikKMX3cCucVKdHOzwQ/TQmFjxgJ6a9BYx+ZE1LKwiE5UDyu5MX6Y2gtdXa2RU6TE+HXHkZpT3KT7LCkpQUlJic6Pv5RZiG8OXQFQtZLI3ITXDiYiImpu9eXzPRcyUaES0MnREh0deaYY3V/Nhy07m6ClS3mlCi9uikVusRJdXa2xlC0AiTQuZRZi/LrjyC1WIqCdDX58lgX01uZBj82JqOVhEZ3oHmzMjfHjtFB0cbJCVmE5xq87jvRc/UykgiBgwbZ4VKoFhPs6Yaifk9ghERER0W1qCqEj2cqFGqjmd+Vocg7ySyoaddtLdyTgTHoebMyMsXpCCOTGskbdPpGhulxdQL9ZrIR/O2tsnBYKG3MW0ImIWjsW0Ynuw87CBBufC0UHBwvcyC/D+G+OIyO/VOyw7vJ77HVEp+RCbizF4kf9xA6HiIiIblNUXomDl7MBsJULNVwHB0t0cbJChUrA7guZjbbd32OvYePxNEgkwGdjg9De3rzRtk1kyJKyCjFu3QnkFFWdocECOhER1WARnagBHKxMsXl6b3jYmyM9txTj151AVkGZ2GFp5JUo8f7OCwCAl4d0glsbHggRERHpk70Xs6CsVMO7rQW68KLfpIWR1R+6RMRnNMr2Em4U4O0/4gAALw/uhEE+jo2yXSJDl5RVhKfXnkBOUTn8XKyx6blQ2JqbiB0WERHpCRbRiRrIyVqOzdN7o52tGVJyijH+m6o3WPrgw38ScbNYiU6Olniur7fY4RAREdEdIuKqCqAj/J3Zd5q0Mqq6pcvBSzkoLHuwli75pRWYuSkGZRVqDOjsgFeGdGqMEIkMXlJWEcatO46conL4soBORER1YBGdSAvtbM3w0/TecLGRIymrCBO/OYG8EqWoMZ1Ou4WfotMAAEvH+MPEiH/WRERE+qREWYl9iVkA/i2IEjVUJ0dLeDtYQKlSY+/FLJ23o1YLeO3nM7h6swTtbM3w2dggSKX8QIcoObuqgJ5dWA4fZytsei4UbSxYQCciotpYbSPSUnt7c2x6LhQOVqa4qCjEpG+jUfCAq4JqyOVyyOXyBo+vVKmxYFs8BAH4v+7t0NvbvlHiICIiIt3dmc/3J2ajrEINdzszdHW1FjEyMkQSiQSj/Ks+fNkZp3tLl1X7k7D7QhZMjKRYMzGERUIiAFeyizBu7b8F9M3Te8OOfxsE7Y/NiajlYxGdSAfeDpbY/Fwo7CxMEHc9H8+sj0ZReeUDbVMmk6F3797o3bs3ZDJZgx7z4/GrOH+jANZyI7z9sO8D7Z+IiIgeXF35vKbwOcrfha1cSCc1fdH3J2ajWIf3nIcuZ+OTqEsAgKWPdUWAm02jxkdkiFJyijFu3XFkFZaji1PVCnQW0AnQ7diciFo+FtGJdNTJyarqau1mxjidlodnvzuJEuWDFdK1kVlQhk92VR0MzRvhg7aWps22byIiImqYsgqVpgXHSLZyIR35uVjDw94c5ZVq7E/M1uqx1/NK8fJPpyEIwNM93TG2Z/smipLIcKTmFGPc2uPILChHZydLbJoeCnseTxER0T2wiE70APxcrfHjtF6wMjVCdGoupv9wCmUVqmbZ97K/L6CovBKB7rYY14sHQ0RERProwKVslChVcLWRI5Crf0lHEokEI2tausQ3vKVLeaUKL26Mwa2SCgS0s8HiR7s2VYhEBiM1pxhPrz0ORUEZOjlaYvP03lyQRERE98UiOtED6uZmiw3P9oKFiQxHkm7ihY0xKK/UvpCuUqkQExODmJgYqFT3fvyhy9nYfvYGpBLgv2P8IeNFoYiIiPTCnfk8Ml4BoGoVOlu50IMYVd3SZd/FLJQqG/Zec8n2BJy9lg9bc2OsmhAMuTHbElDrdvVmVQsXFtDpXrQ5Niei1kOrIvry5cvRs2dPWFlZwdHREWPGjEFiYuJd444dO4bBgwfDwsIC1tbW6N+/P0pLSzXfz83NxYQJE2BtbQ1bW1tMmzYNRUVFtbZx7tw59OvXD3K5HO7u7vjwww91fIpETS/Eow3WT+kJubEU+xOzMXvzaVSo1Fpvp7CwEIWFhfccU1ahwsI/zwMAJod5wr8dV7URERHpk5p8Xl6pwu6ETAD/FkCJdBXQzgbtbM1QolThwKX7t3T55VQ6Np9Ig0QCfP50d7jbmTdDlET6K+1mCcatPY6M/DJ0rC6gO1ixgE51a8ixORG1LloV0Q8cOIBZs2bh+PHjiIqKQkVFBYYNG4bi4mLNmGPHjmHEiBEYNmwYoqOjcfLkScyePRtS6b+7mjBhAs6fP4+oqCjs2LEDBw8exIwZMzTfLygowLBhw+Dh4YGYmBh89NFHWLx4MdauXdsIT5moaYR62+ObyT1hYiRFVEIm5mw5g0odCun3s/bgFaTkFMPByhRzh3Vu9O0TERFR4zialIPC8ko4WZuiu3sbscMhA1fV0qXqw5iI+7R0ib+ejwXb4gEAc4Z0xoDODk0eH5E+S88twbh1x3EjvwwdHCyweXooC+hERKQVrYrokZGRmDJlCrp27YrAwEBs2LABaWlpiImJ0Yx59dVX8fLLL+Ott95C165d0aVLFzz11FMwNa1KUBcuXEBkZCS++eYbhIaGom/fvvjyyy+xZcsW3LhxAwCwadMmKJVKrF+/Hl27dsXTTz+Nl19+GStWrGjEp07U+Pp2aouvJ4XAWCbB33EZeOPXc1CphUbb/tWbxfhqXxIAYMHDvrCWGzfatomIiKhxRZ6vWoU+oqszpGy9Ro2g5uK0ey5k1XsdnvySCszcFIPySjUGdXHAS4M7NmeIZABWrlwJT09PyOVyhIaGIjo6ut6xv//+O3r06AFbW1tYWFggKCgIP/74YzNG++DSc0vw9NrjuJ5XCm8HC/w0vTccreRih0VERAbmgXqi5+fnAwDs7OwAAFlZWThx4gQcHR3Rp08fODk5YcCAATh8+LDmMceOHYOtrS169OihuS88PBxSqRQnTpzQjOnfvz9MTEw0Y4YPH47ExETcunXrQUImanKDujhi5fhgGEkl+OP0dbz9exzUjVBIFwQBC/88D2WlGg91tMejga6NEC0RERE1BZVaQFTCv/3QiRpDd3dbOFvLUVReicOXc+76vlotYM7W00jPLYW7nRk+G9udH+BQLVu3bsXcuXOxaNEixMbGIjAwEMOHD0dWVlad4+3s7PDOO+/g2LFjOHfuHKZOnYqpU6fin3/+aebIdVOrgN7WAlum94ajNQvoRESkPZ2L6Gq1GnPmzMFDDz0Ef39/AMCVK1cAAIsXL8b06dMRGRmJ4OBgDBkyBJcvXwYAKBQKODo61tqWkZER7OzsoFAoNGOcnJxqjan5umbMncrLy1FQUFDrRiSWYV2d8fnT3SGVAFtPpWPRX+chCA9WSI+IV+DApWyYyKRY+pg/L05GRESkx9JvlaCwrBJtLU3Q09NO7HCohZBKJRihaely93HRV/uSsC8xG6ZGUqyeEAIbc561SLWtWLEC06dPx9SpU+Hn54c1a9bA3Nwc69evr3P8wIED8fjjj8PX1xcdOnTAK6+8gm7dutVaKKevrt2qauFSU0D/aQYL6EREpDudi+izZs1CfHw8tmzZorlPra7q//z8889j6tSp6N69Oz799FN06dKl3qTcWJYvXw4bGxvNzd3dvUn3R3Q/D3dzwSdPBUIiAX48fhXL/r6gcyG9qLwS721PAAA8P8Ab3g6WjRkqERERNbKkzCIAwPCuzpBxJTA1olHVZzZEJSigrPz3+jv7E7Pw6e5LAIBlY/x58Xm6i1KpRExMDMLDwzX3SaVShIeH49ixY/d9vCAI2LNnDxITE9G/f/+mDPWBXc8rxbh1x3HtVim8qgvoTiygExHRAzDS5UGzZ8/WXBDUzc1Nc7+LS9UbOj8/v1rjfX19kZaWBgBwdna+61SxyspK5ObmwtnZWTMmMzOz1piar2vG3Gn+/PmYO3eu5uuCggIW0kl0j3d3g7JSjTd/i8O3h1NgaiTFG8O71LuK3Ni47tVCn0VdgqKgDO3tzDFrEPtaEhFR01OpBUSn5CKrsAyOVnL08rJjMbgBVGoB1/OVSMgsBmCOEV3rfu9KpKsQjzZwsDJFdmE5vj18Ba62ZpAAePfPeAgCMD60Pf7Tg8dBdLecnByoVKo6z/q+ePFivY/Lz89Hu3btUF5eDplMhlWrVmHo0KH1ji8vL0d5ebnm6+Y+S/xGXimeXnsM6bml8LQ3x0/TWUAn7dV3bE5ErZdWRXRBEPDSSy/hjz/+wP79++Hl5VXr+56ennB1dUViYmKt+y9duoSRI0cCAMLCwpCXl4eYmBiEhIQAAPbu3Qu1Wo3Q0FDNmHfeeQcVFRWaF66oqCh06dIFbdq0qTM2U1NTzcVLifTJ2J7toaxU490/z2PV/mSYGsnwSninu8bJZDI89NBDd91/IaMA3x1NBQAseawr5Maypg6ZiIhaucj4DCzZnoCM/DLNfS42ciwa7YcR/uzvXZ9/560CQFWR6o1fz2Hxo5w3ajwyqQR+LtY4UJiNDyJrH3d52Jtj0Wi/eh5JpBsrKyucOXMGRUVF2LNnD+bOnQtvb28MHDiwzvHLly/HkiVLmjfIalUF9ONIzy2Fh705fprRG842LKCTduo7Niei1k2rdi6zZs3Cxo0bsXnzZlhZWUGhUEChUKC0tBQAIJFI8MYbb+CLL77Ar7/+iqSkJLz77ru4ePEipk2bBqBqVfqIESMwffp0REdH48iRI5g9ezaefvppuLpWXShx/PjxMDExwbRp03D+/Hls3boVn3/+ea2V5kSGZFKYJxY87AsA+HT3Jazen9ygx6nVAhZsi4dKLWCkvzMGdXG8/4OIiIgeQGR8BmZujK1VQAcARX4ZZm6MRWR8hkiR6bf65i2zgPNGjSsyPgMHLmXX+b2rN0uw72LdF4gkatu2LWQyWZ1nfdd3xjdQ1fKlY8eOCAoKwmuvvYYnn3wSy5cvr3f8/PnzkZ+fr7mlp6c32nO4l4z8qhYuabklaG9XtQLdxcasWfZNREQtn1ZF9NWrVyM/Px8DBw6Ei4uL5rZ161bNmDlz5mD+/Pl49dVXERgYiD179iAqKgodOnTQjNm0aRN8fHwwZMgQjBo1Cn379sXatWs137exscGuXbuQkpKCkJAQvPbaa1i4cCFmzJjRCE+ZSBzP9fPGvBFdAAAfRF7E+sMp933MLzHpiLl6C+YmMrz7CFcVERFR01KpBSzZnoC6ruBRc9+S7QlQqR/sYtktDeeNmkvN71p9JODvGtXPxMQEISEh2LNnj+Y+tVqNPXv2ICwsrMHbUavVtdq13MnU1BTW1ta1bk1NkV+GcWuP4+rNqgL6lhm94WrLAjoRETUerdu5NMRbb72Ft956q97v29nZYfPmzffcRrdu3XDo0CFtwiPSey8O7IjyCjU+33MZ7+1IgImRFBN7ewAAVCoV4uLiAAABAQHIL1NheURVb8JXwzvzTSARETUKQRBQolThVokSeSUVuFWixK2SCtwqVuLctby7VlLXeiyAjPwy9P3fXpiZyGoVjWveJwqar29/nHD3fXW8rbxzG7ePE26799/76tre/cbdYx+3BVXH5m57brW3oVILKK++wKMEanSQ5QIAklV2ECDVzFt0Si7COtiDSFfRKbkN+hvl7xrVZ+7cuXjmmWfQo0cP9OrVC5999hmKi4sxdepUAMDkyZPRrl07zUrz5cuXo0ePHujQoQPKy8uxc+dO/Pjjj1i9erWYT6MWRX4Zxq07jtSbJXC3M8NPLKDTA7rz2FwmY0tVItLxwqJEpLs54Z1QXqnGmgPJWLAtHiYyKZ7qWXXxp7y8PM24/0VcQF5JBXycrTDlIU9xgiUiIr2mUgsoKK1AbokSeSVK3CquuKs4nleiRG7xv/fllVRAqVI/0H4zCuov4rV2EgCWknLN/28v1mcVct7owTT0d4i/a1SfsWPHIjs7GwsXLoRCoUBQUBAiIyM1FxtNS0uDVPrvCevFxcV48cUXce3aNZiZmcHHxwcbN27E2LFjxXoKtWQWVBXQU3KK4dbGDD9N7412LKBTI7j92JyICGARnajZSSQSvDmiC5SVaqw/koI3fz8HU2MpRnZ1QnpuCYqVlUg5moKfT10DACwb4w9jmVadl4iI6A4qtYDolFxkFZbB0UqOXl52kEklYodVS1mF6rbid+2i961iZXWhvHaRPL+0os4V3Q1hYiRFG3NjtDE3gW31v+WVauxtQD/lRaP94N/OBkBVobiGpNaUSu66T6IZJ7nrvtvHSm67V1LHj+l+4+71/caK9/ZxZ9Ly8MrWM3cHegdHK17cjh5MQ3+H+LtG9zJ79mzMnj27zu/t37+/1tfLli3DsmXLmiGq+7szl3vam2PCNyeQklOMdrZVBXS3NuZih0lERC0Ui+hEIpBIJHj3EV8oVSpsPJ6GV7eewWK5DB4VVYXzc2ckAKTo08EePTztxA2WiMjARcZnYMn2hFotEFxs5Fg02g8j/F0afX+CIKCwvBJ5xbUL4lWrwatbp9RREC9RqnTep5WpEWwtagriJprieBtzE7SxMK51X03B3NxEVqs4DFQVKPp+sBeK/LI6+3tLADjbyDE5zFPvPoQQk1sbc/wv8iIU9bTZqJm3Xl7M6fRgennZwcVGft+/Uf6uUUtTVy6XSSVQqQW0szXDlhm94W7HAjoRETUdFtGJRCKRSPDeo/5IzirGsSs3kV9acddf5LHkm4iMz2iSIg8RUWsQGZ+BmRtj7yo2KfLLMHNjLFZPDL7na2ylSo280pqWKDVF79sK4be1T6lpqZJXUoFKHS/qJ5NKYGtmDFtzY9hZmNxR/K76f82/Nd+3NTdutDOWZFIJFo32w8yNsXe1IqkpmS8a7ccC+h3unLfbcd6oMfFvlFqj+nJ5zQV0Zw7swAI6ERE1ORbRiUQkAEjJKb7nmCXbEzDUz5kHQ0REWlKpBSzZnlDnas2a+9787RwuKgqRX1rVMqWmh3hNkbywrFLn/cuNpbCrKX7ftRq8+v8W1avFq4vjVqZGkIr8ej/C3wWrJwbfteLPuQlX77cENfP23l/xQMm/93PeqLHxb5Rak3vl8hor9yVhXK/2PF4iIqImxSI6kYiiU3KhuMfF2QQAGflliE7JRVgH++YLjIioBYhOya1VYKpLfmklPtt9+b7bsjEzrrUKvKYQbqcpjt+2Sry6rYrcWNZYT6XZjfB3wVA/Z73vI69vRvi7YHAXB2zdXoFiZSXeCA5F744OnDdqdPwbpdaiIbmcx0tERNQcWEQnElFWYe03hOq7TgKvexwREd1fQ187wzrYIci9zW1F8n8L4nYWJrAxM26VhSmZVMKChA5kUgk82loCAHp3sG+VvzvUPPg3Sq1BQ3M5j5eosUmljdMqj4haDhbRiUTkaCXX/F8NKc5Vut53HBERNUxDXztfHtyZhShqNDKZDP379xc7DCKiFqGhuZzHS9SYmMuJqC78aI1IRL287OBiI69n/XnVBaJcbKpOzyUiIu3wNZaIiMiwMZcTEZG+YBGdSEQyqQSLRvsBwF1vDGu+XjTaj6eCExHpgK+xREREho25nIiI9AWL6EQiG+HvgtUTg+FibQpv2U14y25CAgHONnKsnhiMEf4uYodIRGSwal5jnW1qn+bN11hqKmq1GufOncO5c+egVqvFDoeIyOAxl1NzYy4norqwJzqRHhjh74LBXRywdfsuFCsrERDcC707OnBFBRFRIxjh74Khfs6ITslFVmEZHK2qTvvmayw1BUEQkJubq/k/ERE9OOZyak7M5URUFxbRifSETCqBu505AKB3B3u+ISQiakQyqYQXDyUiIjJgzOVERCQmtnMhIiIiIiIiIiIiIqoHi+hERERERERERERERPVgEZ2IiIiIiIiIiIiIqB4sohMRERERERERERER1aPFXli05grKBQUFIkdC1DAqlQrFxcUAqn5vZTKZyBERUVOqyU81+YruxlxOhoj5nKh1YT6/P+ZzMjTM5UStS0NzeYstohcWFgIA3N3dRY6EiIiofoWFhbCxsRE7DL3EXE5ERIaC+bx+zOdERGQI7pfLJUIL/chcrVbjxo0bsLKygkQieaBtFRQUwN3dHenp6bC2tm6kCFs+zpv2OGe64bxpj3Omm8acN0EQUFhYCFdXV0il7K5Wl8bM5QB/73XBOdMN5017nDPdcN6019hzxnx+fzw2Fx/nTXucM91w3rTHOdONGMfmLXYlulQqhZubW6Nu09ramr/QOuC8aY9zphvOm/Y4Z7pprHnjirV7a4pcDvD3XhecM91w3rTHOdMN5017jTlnzOf3xmNz/cF50x7nTDecN+1xznTTnMfm/KiciIiIiIiIiIiIiKgeLKITEREREREREREREdWDRfQGMDU1xaJFi2Bqaip2KAaF86Y9zpluOG/a45zphvNm2Pjz0x7nTDecN+1xznTDedMe58yw8eenG86b9jhnuuG8aY9zphsx5q3FXliUiIiIiIiIiIiIiOhBcSU6EREREREREREREVE9WEQnIiIiIiIiIiIiIqoHi+hERERERERERERERPVgEZ2IiIiIiIiIiIiIqB4somshPT0dAwcOhJ+fH7p164ZffvlF7JAMQl5eHnr06IGgoCD4+/tj3bp1YodkMEpKSuDh4YHXX39d7FAMhqenJ7p164agoCAMGjRI7HAMQkpKCgYNGgQ/Pz8EBASguLhY7JD0XmJiIoKCgjQ3MzMzbNu2TeywqIGYz7XHXK475nLtMZdrj7lce8zlho25XHvM5Q+G+Vw7zOW6YT7XTnPncokgCEKTbb2FycjIQGZmJoKCgqBQKBASEoJLly7BwsJC7ND0mkqlQnl5OczNzVFcXAx/f3+cOnUK9vb2Yoem99555x0kJSXB3d0dH3/8sdjhGARPT0/Ex8fD0tJS7FAMxoABA7Bs2TL069cPubm5sLa2hpGRkdhhGYyioiJ4enri6tWrzAcGgvlce8zlumMu1x5zufaYyx8Mc7nhYS7XHnP5g2E+1w5zuW6Yz3XXHLmcK9G14OLigqCgIACAs7Mz2rZti9zcXHGDMgAymQzm5uYAgPLycgiCAH52c3+XL1/GxYsXMXLkSLFDoRbs/PnzMDY2Rr9+/QAAdnZ2TNJa+uuvvzBkyBAetBkQ5nPtMZfrhrmcmgNz+YNjLjc8zOXaYy7XHfM5NQfm8wfTHLmcRfQ7LF++HD179oSVlRUcHR0xZswYJCYm3jUuJiYGKpUK7u7uIkSpf+43b3l5eQgMDISbmxveeOMNtG3bVsRo9cP95uz111/H8uXLRYxQP91v3iQSCQYMGICePXti06ZNIkaqP+41Z5cvX4alpSVGjx6N4OBgvP/++yJHqz8amg9+/vlnjB07VoQI6V6Yz7XHXK495nLdMJdrj7lcN8zlho25XHvM5bphPtcec7lumM+1p0+5nEX0Oxw4cACzZs3C8ePHERUVhYqKCgwbNqxWH6Lc3FxMnjwZa9euFTFS/XK/ebO1tcXZs2eRkpKCzZs3IzMzU+SIxXevOfvzzz/RuXNndO7cWeww9c79ftcOHz6MmJgY/PXXX3j//fdx7tw5kSMW373mrLKyEocOHcKqVatw7NgxREVFISoqSuyQ9UJD8kFBQQGOHj2KUaNGiRgp1YX5XHvM5dpjLtcNc7n2mMt1w1xu2JjLtcdcrhvmc+0xl+uG+Vx7epXLBbqnrKwsAYBw4MABQRAEoaysTOjXr5/www8/iByZfrtz3m43c+ZM4ZdffhEhKv12+5y99dZbgpubm+Dh4SHY29sL1tbWwpIlS8QOUS/d63ft9ddfF7777rvmD0rP3T5nR48eFYYNG6b53ocffih8+OGHIkanv+r6Xfvhhx+ECRMmiBgVNRTzufaYy7XHXK4b5nLtMZfrhrncsDGXa4+5XDfM59pjLtcN87n2xMzlXIl+H/n5+QCqehEJgoApU6Zg8ODBmDRpksiR6bfb5y0zMxOFhYWa+w8ePIguXbqIGZ5eun3Oli9fjvT0dKSmpuLjjz/G9OnTsXDhQpEj1E+3z1txcbHmd62oqAh79+5F165dxQxPL90+Zz179kRWVhZu3boFtVqNgwcPwtfXV+QI9dPt81aDp38bDuZz7TGXa4+5XDfM5dpjLtcNc7lhYy7XHnO5bpjPtcdcrhvmc+2JmcslgsArSdRHrVbj0UcfRV5eHg4fPozDhw+jf//+6Natm2bMjz/+iICAABGj1D93zlt0dDRmzJihuXDJrFmz8Pzzz4sdpl65c85ut2HDBsTHx/MK4HW4c96uXLmCxx9/HEDV1eenT5+OV155ReQo9Utdv2sRERGYN28eBEHAsGHDsGLFCpGj1D91zVt+fj46d+6M9PR0mJiYiBwh3QvzufaYy7XHXK4b5nLtMZfrhrncsDGXa4+5XDfM59pjLtcN87n2xM7lLKLfw8yZMxEREYHDhw/Dzc1N7HAMBudNe5wz3XDetMc50w3nzbDx56c9zpn2OGe64bxpj3OmG86bYePPT3ucM91w3rTHOdMN5017Ys+ZUbPv0UDMnj0bO3bswMGDB/nLrAXOm/Y4Z7rhvGmPc6Ybzpth489Pe5wz7XHOdMN50x7nTDecN8PGn5/2OGe64bxpj3OmG86b9vRizpq867qBUavVwqxZswRXV1fh0qVLYodjMDhv2uOc6Ybzpj3OmW44b4aNPz/tcc60xznTDedNe5wz3XDeDBt/ftrjnOmG86Y9zpluOG/a06c540r0O8yaNQubN2/Gn3/+CSsrKygUCgCAjY0NzMzMRI5Of3HetMc50w3nTXucM91w3gwbf37a45xpj3OmG86b9jhnuuG8GTb+/LTHOdMN5017nDPdcN60p09zxp7od5BIJHXe/91332HKlCnNG4wB4bxpj3OmG86b9jhnuuG8GTb+/LTHOdMe50w3nDftcc50w3kzbPz5aY9zphvOm/Y4Z7rhvGlPn+aMRXQiIiIiIiIiIiIionpIxQ6AiIiIiIiIiIiIiEhfsYhORERERERERERERFQPFtGJiIiIiIiIiIiIiOrBIjoRERERERERERERUT1YRCciIiIiIiIiIiIiqgeL6ERERERERERERERE9WARnYiIiIiIiIiIiIioHiyiExERERERERERERHVg0V0IiIiIiIiIiIiIqJ6sIhORM1mypQpGDNmjCj7lkgk2LZtW6Ntb+DAgZgzZ06jbY+IiMhQMJ8TEREZNuZyIu2xiE5ED2zx4sUICgpqln2lpqZCIpHgzJkzzbK/+vz+++9YunSpqDEQERE1JuZzIiIiw8ZcTtR0jMQOgIjuTRAEqFQqGBnxz1Wf2NnZiR0CEREZEOZz/cR8TkREDcVcrp+Yy6m5cCU6tVgDBw7ESy+9hDlz5qBNmzZwcnLCunXrUFxcjKlTp8LKygodO3ZERERErcfFx8dj5MiRsLS0hJOTEyZNmoScnBzN9yMjI9G3b1/Y2trC3t4ejzzyCJKTkzXfVyqVmD17NlxcXCCXy+Hh4YHly5cDqPuT2ry8PEgkEuzfvx8AsH//fkgkEkRERCAkJASmpqY4fPgw1Go1li9fDi8vL5iZmSEwMBC//vqrZjs1j/vnn3/QvXt3mJmZYfDgwcjKykJERAR8fX1hbW2N8ePHo6SkRPO4hm53z5496NGjB8zNzdGnTx8kJiYCADZs2IAlS5bg7NmzkEgkkEgk2LBhwz1/NkuWLIGDgwOsra3xwgsvQKlUNnh+vby8AADdu3eHRCLBwIEDNd9bv349unbtClNTU7i4uGD27Nm19puTk4PHH38c5ubm6NSpE/766697xrlq1Sp06tQJcrkcTk5OePLJJzXfu/2UsZo5uvM2ZcoUzfg///wTwcHBkMvl8Pb2xpIlS1BZWXnP/RMREfM58znzORGRoWMuZy5nLqcWQSBqoQYMGCBYWVkJS5cuFS5duiQsXbpUkMlkwsiRI4W1a9cKly5dEmbOnCnY29sLxcXFgiAIwq1btwQHBwdh/vz5woULF4TY2Fhh6NChwqBBgzTb/fXXX4XffvtNuHz5snD69Glh9OjRQkBAgKBSqQRBEISPPvpIcHd3Fw4ePCikpqYKhw4dEjZv3iwIgiCkpKQIAITTp09rtnfr1i0BgLBv3z5BEARh3759AgChW7duwq5du4SkpCTh5s2bwrJlywQfHx8hMjJSSE5OFr777jvB1NRU2L9/f63H9e7dWzh8+LAQGxsrdOzYURgwYIAwbNgwITY2Vjh48KBgb28v/O9//9Psv6HbDQ0NFfbv3y+cP39e6Nevn9CnTx9BEAShpKREeO2114SuXbsKGRkZQkZGhlBSUlLnz+SZZ54RLC0thbFjxwrx8fHCjh07BAcHB+Htt99u8PxGR0cLAITdu3cLGRkZws2bNwVBEIRVq1YJcrlc+Oyzz4TExEQhOjpa+PTTTzXbBSC4ubkJmzdvFi5fviy8/PLLgqWlpebxdzp58qQgk8mEzZs3C6mpqUJsbKzw+eef1/r9euWVVwRBEITy8nLNc8/IyBD27t0ryOVy4dtvvxUEQRAOHjwoWFtbCxs2bBCSk5OFXbt2CZ6ensLixYvr3DcREf2L+Zz5nPmciMiwMZczlzOXU0vAIjq1WAMGDBD69u2r+bqyslKwsLAQJk2apLkvIyNDACAcO3ZMEARBWLp0qTBs2LBa20lPTxcACImJiXXuJzs7WwAgxMXFCYIgCC+99JIwePBgQa1W3zVWm0S9bds2zZiysjLB3NxcOHr0aK3tTZs2TRg3blytx+3evVvz/eXLlwsAhOTkZM19zz//vDB8+PAH2u7ff/8tABBKS0sFQRCERYsWCYGBgXXOz+2eeeYZwc7OTvPGSBAEYfXq1YKlpaUmEd/pzvmtaw4FQRBcXV2Fd955p959AxAWLFig+bqoqEgAIERERNQ5/rfffhOsra2FgoKCOr9/e6K+XU5OjuDt7S28+OKLmvuGDBkivP/++7XG/fjjj4KLi0u98RIRURXmc+bz2zGfExEZHuZy5vLbMZeToWI7F2rRunXrpvm/TCaDvb09AgICNPc5OTkBALKysgAAZ8+exb59+2Bpaam5+fj4AIDmtKXLly9j3Lhx8Pb2hrW1NTw9PQEAaWlpAKqucn3mzBl06dIFL7/8Mnbt2qVT7D169ND8PykpCSUlJRg6dGit2H744Ydap1Pd+ZydnJxgbm4Ob2/vWvfVPF9dt+vi4lJr3rQRGBgIc3NzzddhYWEoKipCeno6gPvPb12ysrJw48YNDBky5J77vv05WFhYwNraut7nMHToUHh4eMDb2xuTJk3Cpk2bap1qV5eKigo88cQT8PDwwOeff665/+zZs3jvvfdqzfH06dORkZFx320SERHzOfN5bcznRESGh7mcuby+58BcToaCV0OgFs3Y2LjW1xKJpNZ9EokEQFXvMQAoKirC6NGj8cEHH9y1rZrkNHr0aHh4eGDdunVwdXWFWq2Gv7+/pndYcHAwUlJSEBERgd27d+Opp55CeHg4fv31V0ilVZ9bCYKg2W5FRUWdsVtYWGj+X1RUBAD4+++/0a5du1rjTE1N633Odz7fmvtuf766bhf4d94a0/3mty5mZmYN2va95uJOVlZWiI2Nxf79+7Fr1y4sXLgQixcvxsmTJ2Fra1vnY2bOnIn09HRER0fXuthMUVERlixZgv/7v/+76zFyubxBsRMRtWbM58znt2M+JyIyPMzlzOW3Yy4nQ8QiOtFtgoOD8dtvv8HT07POK27fvHkTiYmJWLduHfr16wcAOHz48F3jrK2tMXbsWIwdOxZPPvkkRowYgdzcXDg4OAAAMjIy0L17dwCodSGT+vj5+cHU1BRpaWkYMGDAAzzDptmuiYkJVCpVg8aePXsWpaWlmuR6/PhxWFpawt3dvUHza2JiAgC19mdlZQVPT0/s2bMHgwYN0vl53MnIyAjh4eEIDw/HokWLYGtri71799aZcFesWIGff/4ZR48ehb29fa3vBQcHIzExER07dmy02IiIqH7M57phPmc+JyLSF8zlumEuZy6npsMiOtFtZs2ahXXr1mHcuHGYN28e7OzskJSUhC1btuCbb75BmzZtYG9vj7Vr18LFxQVpaWl46623am1jxYoVcHFxQffu3SGVSvHLL7/A2dkZtra2kEql6N27N/73v//By8sLWVlZWLBgwX3jsrKywuuvv45XX30VarUaffv2RX5+Po4cOQJra2s888wzOj3fxtqup6cnUlJScObMGbi5ucHKyuquT8trKJVKTJs2DQsWLEBqaioWLVqE2bNnQyqVNmh+HR0dYWZmhsjISLi5uUEul8PGxgaLFy/GCy+8AEdHR4wcORKFhYU4cuQIXnrpJZ3mZseOHbhy5Qr69++PNm3aYOfOnVCr1ejSpctdY3fv3o158+Zh5cqVaNu2LRQKBYCqT+FtbGywcOFCPPLII2jfvj2efPJJSKVSnD17FvHx8Vi2bJlO8RERUf2Yz5nPazCfExEZJuZy5vIazOWkL9gTneg2rq6uOHLkCFQqFYYNG4aAgADMmTNHk2SlUim2bNmCmJgY+Pv749VXX8VHH31UaxtWVlb48MMP0aNHD/Ts2ROpqanYuXOn5nSx9evXo7KyEiEhIZgzZ06DX6iXLl2Kd999F8uXL4evry9GjBiBv//+G15eXg/0nBtju0888QRGjBiBQYMGwcHBAT/99FO9Y4cMGYJOnTqhf//+GDt2LB599FEsXrwYABo0v0ZGRvjiiy/w9ddfw9XVFY899hgA4JlnnsFnn32GVatWoWvXrnjkkUdw+fJl7Sekmq2tLX7//XcMHjwYvr6+WLNmDX766Sd07dr1rrGHDx+GSqXCC5/wyQQAAAEJSURBVC+8ABcXF83tlVdeAQAMHz4cO3bswK5du9CzZ0/07t0bn376KTw8PHSOj4iI6sd8znxeg/mciMgwMZczl9dgLid9IRFubwBFREREREREREREREQaXIlORERERERERERERFQPFtGJiIiIiIiIiIiIiOrBIjoRERERERERERERUT1YRCciIiIiIiIiIiIiqgeL6ERERERERERERERE9WARnYiIiIiIiIiIiIioHiyiExERERERERERERHVg0V0IiIiIiIiIiIiIqJ6sIhORERERERERERERFQPFtGJiIiIiIiIiIiIiOrBIjoRERERERERERERUT1YRCciIiIiIiIiIiIiqsf/A+MqdAoMYqW2AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n", + "\n", + "sweep_cols = [f\"cross_chi_net@bs{S}\" for S in MEASURE_BATCH_SIZES]\n", + "row = metrics.dropna(subset=sweep_cols).iloc[-1]\n", + "\n", + "fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n", + "for ax, metric in zip(axes, [\"chi_net\", \"chi_coup\", \"grad_dot_product\"]):\n", + " ys = [row[f\"cross_{metric}@bs{S}\"] for S in MEASURE_BATCH_SIZES]\n", + " ax.plot(MEASURE_BATCH_SIZES, ys, \"o-\")\n", + " ax.axvline(MEASURE_MAX_SINGLE_PASS, color=\"gray\", linestyle=\"--\", alpha=0.5,\n", + " label=\"max single-pass size\")\n", + " ax.set_xscale(\"log\", base=2)\n", + " ax.set_xlabel(\"measurement batch size\")\n", + " ax.set_title(f\"cross_{metric}\")\n", + "axes[0].legend()\n", + "\n", + "fig.suptitle(f\"Measurement batch-size sweep @ step {int(row['step'])}\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2086f0994ba148e58e57906c0b51eb68": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_592116f5074648528db363f98703056a", + "max": 1.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_5f69781c8cf44938a8df9761777d102b", + "tabbable": null, + "tooltip": null, + "value": 1.0 + } + }, + "592116f5074648528db363f98703056a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": "2", + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f69781c8cf44938a8df9761777d102b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "602335a52b5748d7acbfc07a7f611aaa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": "inline-flex", + "flex": null, + "flex_flow": "row wrap", + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "100%" + } + }, + "662062d25a374c95a430a67df0a8236c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "7b7e75172bc843ad9957a168fc37d782": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "83ba3e3aa3b84cc9811c806f9f4398ef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8e32e47451ec477bb6a2356389ac6da9", + "IPY_MODEL_2086f0994ba148e58e57906c0b51eb68", + "IPY_MODEL_8b8cac2e24cb4c9491944f2d01137041" + ], + "layout": "IPY_MODEL_602335a52b5748d7acbfc07a7f611aaa", + "tabbable": null, + "tooltip": null + } + }, + "8b8cac2e24cb4c9491944f2d01137041": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_aa81e02672cb4efcbd1bdca8d44a5e79", + "placeholder": "​", + "style": "IPY_MODEL_7b7e75172bc843ad9957a168fc37d782", + "tabbable": null, + "tooltip": null, + "value": " 200/200 [00:02<00:00, 83.48it/s, v_num=0, train_loss=0.320, train_acc=0.891]" + } + }, + "8e32e47451ec477bb6a2356389ac6da9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f4c92b02245343469d0a5111affbdb1e", + "placeholder": "​", + "style": "IPY_MODEL_662062d25a374c95a430a67df0a8236c", + "tabbable": null, + "tooltip": null, + "value": "Epoch 0: 100%" + } + }, + "aa81e02672cb4efcbd1bdca8d44a5e79": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f4c92b02245343469d0a5111affbdb1e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/measurement_batch_sweep.md b/examples/measurement_batch_sweep.md new file mode 100644 index 0000000..823c026 --- /dev/null +++ b/examples/measurement_batch_sweep.md @@ -0,0 +1,314 @@ +# Independent Measurement Batch Size — Implementation Notes + +## The problem this solves + +`analyzer(..., cross_response=True)` measures the model's linear response against a +held-out "measure" batch by expecting a dict batch `{"train": ..., "measure": ...}`, +produced with a `CombinedLoader({"train": train_loader, "measure": measure_loader}, +mode="max_size_cycle")`. See `cifar10_CrossReseponse.ipynb`. + +When gradient accumulation is active on the train side +(`micro_batch_size`/`effective_batch_size`, see `batch_accumulation_notes.md`), the +legacy cross-response code accumulates the measure gradient over the *same* K +micro-batches as the train side, and divides both by the same `K` +(`perspic/analyzer.py`, `_finalize_accumulated_analysis`). There was no way to: + +1. Give the measure batch a size independent of the train effective batch size, or +2. Accumulate the measure side on its own when that size doesn't fit in one forward + pass. + +This was an open `TODO` in the code: + +```python +# Accumulate linearizer gradients (measure side) +# TODO: How would a measure batchsize different to the effective batch size work here? +# !!! We would need to accumulate separately and then combine at the end. +``` + +This document describes the fix: `measure_dataloader` / `measure_batch_size` / +`measure_subset_seed`, a new, independent path that decouples the measurement batch +size from the training batch/accumulation entirely, and — as a second phase — lets one +analysis step sweep an **array** of measurement batch sizes. + +The legacy `cross_response=True` + `CombinedLoader` path is untouched and still works +exactly as before. The new path is a separate, opt-in mechanism. + +--- + +## Quick usage + +### A single, independently sized measurement batch + +```python +from torch.utils.data import DataLoader +from perspic import analyzer + +# Sized however you like — independent of the train DataLoader's batch_size. +measure_loader = DataLoader(measurement_set, batch_size=500, drop_last=True) + +model = analyzer( + ClassificationModule, + model=backbone, + lr=0.1, + measure_dataloader=measure_loader, + measure_batch_size=2000, # 4x measure-side accumulation (2000 // 500) +) +trainer.fit(model, train_dataloaders=train_loader) # plain (x, y) loader, no CombinedLoader +``` + +`measure_batch_size` defaults to `measure_dataloader.batch_size` (no measure-side +accumulation) if omitted. + +### Sweeping multiple measurement batch sizes + +```python +from perspic import analyzer, logarithmic_windows + +schedule = logarithmic_windows(max_steps=10_000, points_per_decade=5) + +model = analyzer( + ClassificationModule, + model=backbone, + lr=0.1, + measure_dataloader=measure_loader, + measure_batch_size=[500, 1000, 2000, 4000], + measure_subset_seed=0, # reproducible subset draws across runs + analysis_schedule=schedule, # see the warning below +) +``` + +Each analyzed step now logs one `cross_*` metric set **per swept size**, suffixed +`@bs{S}`: `cross_chi_net@bs500`, `cross_chi_net@bs1000`, ..., +`cross_grad_dot_product@bs4000`, etc. + +**Warning:** sweeping without a logarithmic `analysis_schedule` runs the *entire* +sweep at *every* analyzed step (every step, if `analyze_every`/`analysis_schedule` are +both unset), which is expensive — each swept size does its own forward+backward passes +per micro-batch chunk. `analyzer()` emits a `UserWarning` at construction time if you +set a `measure_batch_size` list without also setting `analysis_schedule`. + +Sizes below `measure_dataloader.batch_size` are valid too — they're measured as a +single direct pass instead of being rejected: + +```python +measure_loader = DataLoader(measurement_set, batch_size=128, drop_last=True) + +model = analyzer( + ClassificationModule, + model=backbone, + lr=0.1, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8, 16, 32, 64, 128, 256, 512, 1024], + measure_subset_seed=0, + analysis_schedule=schedule, +) +# 4..128 -> single direct pass each (K_measure=1) +# 256, 512, 1024 -> accumulated (K_measure=2, 4, 8) +``` + +--- + +## How it works + +### 1. An independent, persistent measure data source + +`measure_dataloader` is any `DataLoader`. Unlike the train batch, the analyzer does +**not** expect it bundled into the training batch via `CombinedLoader`. Instead the +`Analyzer` owns its own iterator over `measure_dataloader` +(`_next_measure_micro_batch`), lazily created on first use and transparently refilled +(`iter(...)` again) whenever it's exhausted — so a finite measure dataset just cycles. +A `MultiEpochsDataLoader` (see `perspic/utils.py`) works too and is recommended for +small measurement sets, since it avoids re-spawning DataLoader workers every cycle. + +Because the measure source is independent, the training batch is a **plain `(x, y)` +tuple** when `measure_dataloader` is set — no `CombinedLoader` dict, no `"measure"` +key. (The two mechanisms are mutually exclusive per run: pick `cross_response=True` + +`CombinedLoader`, *or* `measure_dataloader`.) + +The measure **micro-batch size** — the largest chunk pulled from the loader in one +forward/backward pass — is inferred from `measure_dataloader.batch_size`. Use +`drop_last=True` (or a dataset size divisible by `batch_size`) so every pulled +micro-batch is full; a short final batch raises a `ValueError` rather than silently +building an undersized pool. + +### 2. Sizing and validation + +`measure_dataloader.batch_size` is the **maximum single-pass batch size** — the +largest batch your GPU can process in one forward+backward pass. `measure_batch_size` +accepts an `int` (single size) or a `list[int]` (sweep), and each entry `S` is handled +by one of two regimes depending on how it compares to that max single-pass size +(`micro = measure_dataloader.batch_size`): + +- **`S <= micro`** — measured with a **single direct pass** on exactly `S` samples, + no accumulation (this deliberately runs the GPU below its max capacity; there's + nothing to validate here beyond `S` being a positive integer). +- **`S > micro`** — `S` must be an **exact multiple** of `micro`; measured by + accumulating `S // micro` passes of size `micro` each into one combined measurement + (the same rule as `effective_batch_size` vs `micro_batch_size` on the train side). + +``` +chunk_size = min(S, micro) +K_measure = S // chunk_size +``` + +This single formula covers both regimes: `chunk_size = S` (so `K_measure = 1`, a lone +direct pass) when `S <= micro`, and `chunk_size = micro` (so `K_measure = S // micro`, +accumulated passes) when `S > micro`. `K_measure` is computed independently per swept +size and is completely decoupled from the train side's `accumulation_steps`. A run can +combine train-side accumulation (`micro_batch_size`/`effective_batch_size`) with +measure-side accumulation (`measure_dataloader`/`measure_batch_size`) freely — the two +`K`s do not need to match, and in general won't. A sweep like +`measure_batch_size=[4, 8, 16, 32, 64, 128, 256, 512, 1024]` with a 128-sample max +single pass measures `4, ..., 128` as direct passes (`K_measure=1` each) and +`256, 512, 1024` as accumulated measurements (`K_measure=2, 4, 8` respectively) — all +in the same analyzed step, each producing exactly one measurement. + +### 3. Gather once, then subset — largest to smallest + +Naively, sweeping N sizes could mean N independent pulls from the measure loader per +analyzed step (S₁ samples for size 1, S₂ for size 2, ...) — wasteful, and it makes +smaller sizes' samples *unrelated* to larger sizes' samples, which is usually not what +you want when studying how a metric depends on batch size. + +Instead, on every analyzed step, `_measure_response` gathers **one pool** of +`S_max = max(measure_batch_size)` samples from the persistent iterator — pulling +`ceil(S_max / micro)` micro-batches (at least one, even when `S_max < micro`, i.e. the +whole sweep is below the max single-pass size) and slicing the concatenation down to +exactly `S_max` samples — and processes every requested size **largest → smallest**: + +- The largest size uses the whole pool. +- Every smaller size `S` uses a **seed-fixable random subset** of `S` samples drawn + from that *same* pool, via a single `torch.Generator` created once at `analyzer()` + construction time and seeded (optionally) by `measure_subset_seed`. The generator is + *not* reseeded between steps or between sizes, so a full run's sequence of subset + draws is reproducible end-to-end given the same seed — rerun the same training script + with the same `measure_subset_seed` and every swept metric matches. + +Processing largest-first (rather than, say, smallest-first or in list order) is what +makes the smaller sizes' samples an actual *subset* of the larger sizes' samples, +rather than an independently-drawn batch — useful when you want to see how a metric +changes as you add more samples to the same pool, not how it varies across unrelated +draws. + +``` +pool = pull(ceil(S_max / micro)) micro-batches, concatenated, sliced to S_max samples +for S in sorted(measure_batch_size, reverse=True): + if S == S_max: + subset = pool # use it all + else: + subset = pool[ randperm(S_max, generator=measure_gen)[:S] ] + chunk_size = min(S, micro) + K_measure = S // chunk_size # 1 if S <= micro + for chunk in chunks(subset, chunk_size): # K_measure chunks + accumulate gradient + per-sample chi over `chunk` + combine and log cross_* metrics (suffixed @bs{S} if sweeping) +``` + +Each chunk is wrapped in its own `BatchStatSnapshot(self.model, chunk)` +(`perspic/utils.py`), so BatchNorm statistics are frozen to *that chunk's own* +statistics before computing its per-sample gradients. This is a deliberate difference +from the legacy `cross_response=True` path, which freezes the measure computation to +the *train* batch's statistics — the independent path has no train batch to borrow +statistics from once the measure size diverges from the train size, so each measure +chunk uses its own. + +### 4. Combining the metrics + +The aggregation semantics mirror the existing **single-step** (no accumulation) +reference case — not the legacy accumulated-cross-response path's per-micro-batch +geometric-mean-then-average, which conflates the train and measure micro-batch +granularities. For each measure size `S`: + +``` +chi_net_measure(S) = mean over the S's K_measure chunks of batch_grad_norms_network +chi_loss_measure(S) = mean over the S's K_measure chunks of batch_grad_norms_loss + +cross_chi_net(S) = sqrt( chi_net_self(train) * chi_net_measure(S) ) # compute_cross_metrics +cross_chi_loss(S) = sqrt( chi_loss_self(train) * chi_loss_measure(S) ) + +grad_train_mean = (train accumulated gradient) / K_train # K_train = 1 without accumulation +grad_measure_mean(S) = (measure accumulated gradient for S) / K_measure + +cross_grad_dot_product(S) = +cross_loss(S) = mean loss over the S measure samples +cross_chi_coup(S) = cross_grad_dot_product(S) / (cross_chi_loss(S) * cross_chi_net(S)) +``` + +Same intensive/extensive reasoning as train-side accumulation applies here (see +"Issue 1" in `batch_accumulation_notes.md`): both `chi_net` and `chi_loss` are +computed with `normalize=True`, which makes the *correct* aggregate over accumulated +chunks the **mean**, not `K_measure * sum(...)`. + +In the train-side-accumulated case, `_measure_response` is called from +`_finalize_accumulated_analysis` with `grad_train_mean = accumulated_train_grad / +accumulation_steps` (the same accumulated train gradient already used for the train +`chi_coup`). In the non-accumulated (single-step) case, the train gradient normally +gets discarded inside `Linearizer.compute()`, so `_analyze_single_step` recomputes it +once explicitly (a single extra forward/backward, only on analyzed steps) before +calling `_measure_response`. + +`_measure_response` always saves and restores `self.model`'s gradients around its own +forward/backward passes (mirroring the existing analysis save/restore pattern in +`_analyze_accumulated_step`), so it never corrupts the live (possibly partially +accumulated) training gradient — this is covered by +`test_measure_backward_does_not_corrupt_training_grad` in `tests/unit/test_analyzer.py`. + +### 5. Logging keys + +| case | keys logged | +|---|---| +| single `measure_batch_size` (int, or a 1-element list) | `cross_chi_net`, `cross_chi_loss`, `cross_chi_coup`, `cross_loss`, `cross_grad_dot_product`, `cross_batch_size` — identical to the legacy `cross_response=True` keys | +| sweep (`measure_batch_size` list, length > 1) | the same set, suffixed `@bs{S}` per size, e.g. `cross_chi_net@bs500`, `cross_chi_net@bs2000`, ... | + +`cross_effective_batch_size` is intentionally **not** logged for the independent +measure path: `batch_size` in the logged metric is already the full measure size `S` — +multiplying by the *train* `accumulation_steps` (what the generic logging helper does +for the train side) would be meaningless here, since `K_measure` is independent of +`K_train`. + +--- + +## Validation rules (raised at `analyzer()` construction time) + +- `measure_dataloader.batch_size` must not be `None` (i.e. it must use automatic + batching). +- Every value in `measure_batch_size` must be a positive integer. Values `<= + measure_dataloader.batch_size` need no further constraint (single direct pass). + Values `>` it must be an exact multiple of it (measured via accumulation). +- `measure_batch_size` / `measure_subset_seed` may only be set together with + `measure_dataloader`. +- A `measure_batch_size` list of length > 1 without `analysis_schedule` triggers a + `UserWarning` (see the sweep-cost warning above). + +## Edge cases + +- **Partial final measure batch.** If `measure_dataloader` has `drop_last=False` and + its dataset size isn't divisible by `batch_size`, the last batch of an epoch is + short. Pulling it into the pool would silently build a short/misaligned pool, so + `_next_measure_micro_batch` raises a `ValueError` instead, recommending + `drop_last=True`. +- **Measure dataset smaller than `S_max`.** The persistent iterator just cycles, so the + pool may contain repeated samples. This is fine for most uses but worth knowing if + you're sizing `measure_batch_size` close to (or larger than) the measurement + dataset's size. +- **Every swept size below the max single-pass size (`S_max < micro`).** The pool + gather still pulls at least one micro-batch (`ceil(S_max / micro) = 1`) and slices + it down to `S_max` samples — it never pulls zero micro-batches. +- **`disable_analyzer=True`.** `_measure_response` is never called (the whole analysis + hook is skipped), so the measure loader is never touched — zero overhead. +- **`log_metrics=False`.** `_measure_response` is gated on `log_metrics` (matching the + "no consumer, skip the expensive sweep" intent), so no measure-side computation runs + at all in that case. + +## Where to look in the code + +- `perspic/analyzer.py`: `__init__` validation block ("Independent measure data + source"), `_next_measure_micro_batch`, `_measure_response`, and the two call + sites in `_analyze_single_step` and `_finalize_accumulated_analysis`. +- `tests/unit/test_analyzer.py`: `TestIndependentMeasureResponse` — validation, + single-measure logging, the `K_measure`-vs-`K_train` mean divisor, a numeric + gradient-dot-product reference test, subset-seed determinism, sweep suffixing, and + iterator-cycling/partial-batch edge cases. +- `tests/integration/test_analyzer_deployment.py`: `TestAnalyzerWithIndependentMeasure` + — end-to-end training with a real `Trainer`, combined with train-side accumulation, + a sweep under a logarithmic schedule, and cross-run reproducibility. diff --git a/perspic/analyzer.py b/perspic/analyzer.py index bbc76d7..c2b591e 100644 --- a/perspic/analyzer.py +++ b/perspic/analyzer.py @@ -1,5 +1,6 @@ +import math import warnings -from typing import Optional +from typing import Optional, Union import pytorch_lightning as pl import torch @@ -24,6 +25,9 @@ def analyzer( cross_response: bool = False, micro_batch_size: Optional[int] = None, effective_batch_size: Optional[int] = None, + measure_dataloader: Optional[torch.utils.data.DataLoader] = None, + measure_batch_size: Optional[Union[int, list[int]]] = None, + measure_subset_seed: Optional[int] = None, **model_kwargs, ): """Factory function that wraps a LightningModule with analysis capabilities. @@ -65,6 +69,31 @@ def analyzer( divisible by micro_batch_size. When set, the optimizer step is only performed every (effective_batch_size // micro_batch_size) micro-batches. + measure_dataloader: An independent DataLoader supplying the + measurement ("cross") batch, decoupled from the training + batch/accumulation. When set, the analyzer owns a + persistent iterator over this loader and the training + batch is expected to be a plain (x, y) tuple (not a + CombinedLoader dict). The measure micro-batch size is + inferred from measure_dataloader.batch_size; use + drop_last=True (or MultiEpochsDataLoader) so every pulled + micro-batch is full. Setting this activates the + independent measure-response path and implies + cross-response-style analysis regardless of + cross_response. + measure_batch_size: The desired measurement batch size(s), + achieved through measure-side gradient accumulation when + larger than measure_dataloader.batch_size. Accepts a + single int or a list[int] to sweep multiple sizes in one + analysis step (each swept size is logged with a + "@bs{S}" suffix). Each value must be >= and divisible by + measure_dataloader.batch_size. Defaults to + measure_dataloader.batch_size (no measure accumulation). + Only valid together with measure_dataloader. + measure_subset_seed: Seed for the random generator used to + draw reproducible subsets of the measure pool when + sweeping multiple measure_batch_size values. Only valid + together with measure_dataloader. **model_kwargs: Additional keyword arguments passed to the LightningModule constructor. @@ -129,6 +158,9 @@ def __init__( cross_response=cross_response, micro_batch_size=micro_batch_size, effective_batch_size=effective_batch_size, + measure_dataloader=measure_dataloader, + measure_batch_size=measure_batch_size, + measure_subset_seed=measure_subset_seed, **model_kwargs, ): super().__init__(**model_kwargs) @@ -194,19 +226,13 @@ def __init__( self.micro_batch_size = micro_batch_size self.effective_batch_size = effective_batch_size - if ( - effective_batch_size is not None - and micro_batch_size is None - ): + if effective_batch_size is not None and micro_batch_size is None: raise ValueError( "micro_batch_size must be specified when " "effective_batch_size is set." ) - if ( - micro_batch_size is not None - and effective_batch_size is not None - ): + if micro_batch_size is not None and effective_batch_size is not None: if effective_batch_size < micro_batch_size: raise ValueError( f"effective_batch_size " @@ -220,22 +246,91 @@ def __init__( f"divisible by micro_batch_size " f"({micro_batch_size})." ) - self.accumulation_steps = ( - effective_batch_size // micro_batch_size - ) + self.accumulation_steps = effective_batch_size // micro_batch_size else: self.accumulation_steps = 1 - if ( - self.accumulation_steps > 1 - and self.delegate_optimization - ): + if self.accumulation_steps > 1 and self.delegate_optimization: raise ValueError( "Gradient accumulation is not supported " "when the wrapped model uses manual " "optimization (delegate_optimization=True)." ) + # --- Independent measure data source (cross-response, decoupled sizing) --- + self._measure_dataloader = measure_dataloader + self._independent_measure = measure_dataloader is not None + + if not self._independent_measure and ( + measure_batch_size is not None or measure_subset_seed is not None + ): + raise ValueError( + "measure_batch_size and measure_subset_seed are only " + "valid together with measure_dataloader." + ) + + if self._independent_measure: + measure_micro = getattr(measure_dataloader, "batch_size", None) + if measure_micro is None: + raise ValueError( + "measure_dataloader must have an integer batch_size " + "(its batch_size attribute is None). Provide a " + "DataLoader whose batch_size divides every " + "measure_batch_size." + ) + self._measure_micro_batch_size = measure_micro + + if measure_batch_size is None: + sizes = [measure_micro] + elif isinstance(measure_batch_size, int): + sizes = [measure_batch_size] + else: + sizes = list(measure_batch_size) + if len(sizes) == 0: + raise ValueError("measure_batch_size list must be non-empty.") + + for s in sizes: + if not isinstance(s, int): + raise ValueError( + f"measure_batch_size entries must be integers, " + f"got {type(s)}." + ) + if s < 1: + raise ValueError( + f"measure_batch_size entries must be positive, " f"got {s}." + ) + if s > measure_micro and s % measure_micro != 0: + raise ValueError( + f"measure_batch_size ({s}) is larger than the " + f"measure_dataloader batch_size ({measure_micro}) " + f"— the maximum single-pass size — and must then " + f"be an exact multiple of it, to be measured via " + f"gradient accumulation. Sizes <= {measure_micro} " + f"need no such constraint (they run as a single " + f"direct pass)." + ) + self._measure_batch_sizes = sizes + + if len(sizes) > 1 and analysis_schedule is None: + warnings.warn( + "A measure batch-size sweep (measure_batch_size " + f"list of length {len(sizes)}) without a " + "logarithmic analysis_schedule runs the full sweep " + "at EVERY analyzed step and is very expensive. Pass " + "analysis_schedule=logarithmic_windows(...) to " + "restrict analysis to logarithmically spaced steps." + ) + + self._measure_gen = torch.Generator() + if measure_subset_seed is not None: + self._measure_gen.manual_seed(measure_subset_seed) + self._measure_iter = None + else: + self._measure_micro_batch_size = None + self._measure_batch_sizes = None + self._measure_gen = None + self._measure_iter = None + self._accumulation_count = 0 self._optimizer_step_count = 0 @@ -277,7 +372,7 @@ def training_step(self, batch, batch_idx): Output from the wrapped module's training_step. """ batch_measure = None - if self.cross_response: + if self.cross_response and not self._independent_measure: # Unpack batch if provided as tuple (batch, batch_idx, dataloader_idx) if type(batch) is tuple and len(batch) == 3: batch, _batch_idx, dataloader_idx = batch @@ -304,17 +399,13 @@ def training_step(self, batch, batch_idx): # BEFORE logic if not self.disable_analyzer: - self._before_training_step( - batch, batch_idx, batch_measure - ) + self._before_training_step(batch, batch_idx, batch_measure) # Original training step output = super().training_step(batch, batch_idx) if not self.delegate_optimization: # Scale loss for gradient accumulation - scaled_output = ( - output / self.accumulation_steps - ) + scaled_output = output / self.accumulation_steps self.manual_backward(scaled_output) self._accumulation_count += 1 @@ -334,10 +425,7 @@ def training_step(self, batch, batch_idx): ) # Step optimizer only at end of accumulation cycle - if ( - self._accumulation_count - >= self.accumulation_steps - ): + if self._accumulation_count >= self.accumulation_steps: opt.step() self._optimizer_step_count += 1 self._accumulation_count = 0 @@ -346,21 +434,14 @@ def training_step(self, batch, batch_idx): self._accum_step_losses.clear() # Step schedulers with interval='step' - if ( - self._trainer is not None - and self.trainer.lr_scheduler_configs - ): - for config in ( - self.trainer.lr_scheduler_configs - ): + if self._trainer is not None and self.trainer.lr_scheduler_configs: + for config in self.trainer.lr_scheduler_configs: if config.interval == "step": config.scheduler.step() # AFTER logic if not self.disable_analyzer: - self._after_training_step( - batch, batch_idx, output - ) + self._after_training_step(batch, batch_idx, output) return output @@ -413,9 +494,7 @@ def _before_training_step(self, batch, batch_idx, cross_response_batch=None): None """ if self.accumulation_steps == 1: - return self._analyze_single_step( - batch, batch_idx, cross_response_batch - ) + return self._analyze_single_step(batch, batch_idx, cross_response_batch) else: return self._analyze_accumulated_step( batch, batch_idx, cross_response_batch @@ -436,23 +515,31 @@ def _analyze_single_step(self, batch, batch_idx, cross_response_batch=None): with BatchStatSnapshot(self.model, x): # Compute sample-wise metrics and self response samples_results["self"] = self.sample_calc.compute( - self.model, self.criterion, x, y, + self.model, + self.criterion, + x, + y, ) # Compute sample-wise metrics and cross response if applicable if x2 is not None and y2 is not None: cross_preliminary = self.sample_calc.compute( - self.model, self.criterion, x2, y2, + self.model, + self.criterion, + x2, + y2, ) - samples_results["cross"] = ( - self.sample_calc.compute_cross_metrics( - sample_wise_metrics_self=samples_results["self"], - sample_wise_metrics_cross=cross_preliminary, - ) + samples_results["cross"] = self.sample_calc.compute_cross_metrics( + sample_wise_metrics_self=samples_results["self"], + sample_wise_metrics_cross=cross_preliminary, ) # Linearizer probe probe_results = self.linearizer.compute( - model=self.model, criterion=self.criterion, - x1=x, y1=y, x2=x2, y2=y2, + model=self.model, + criterion=self.criterion, + x1=x, + y1=y, + x2=x2, + y2=y2, ) # Get "self" result for coupling calculation @@ -470,6 +557,21 @@ def _analyze_single_step(self, batch, batch_idx, cross_response_batch=None): chi_loss=samples_results["cross"]["batch_grad_norms_loss"], chi_net=samples_results["cross"]["batch_grad_norms_network"], ) + + # Capture the train gradient for the independent measure-response + # path. Linearizer.compute() zeroes grads internally and discards + # its own gradient, so we recompute it here (K_train=1) rather than + # reaching into the linearizer's internals. + grad_train_mean = None + if self._independent_measure: + self.model.zero_grad() + loss_t = self.criterion(self.model(x), y) + loss_t.backward() + grad_train_mean = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + self.model.zero_grad() # Log results with fixed metric names if self.log_metrics: self._log_analysis_results( @@ -498,9 +600,17 @@ def _analyze_single_step(self, batch, batch_idx, cross_response_batch=None): self.log("window_center", window_info["window_center"]) self.log("window_width", window_info["window_width"]) + if self._independent_measure: + self._measure_response( + grad_train_mean=grad_train_mean, + self_chi_metrics=samples_results["self"], + ) + return None - def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None): + def _analyze_accumulated_step( + self, batch, batch_idx, cross_response_batch=None + ): """Run analysis with gradient accumulation across micro-batches. On each micro-batch: accumulate sample-wise metrics and linearizer @@ -508,9 +618,7 @@ def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None) """ # On first micro-batch of cycle, decide whether to analyze if self._accumulation_count == 0: - self._analysis_active = self._should_analyze( - self.effective_step - ) + self._analysis_active = self._should_analyze(self.effective_step) if self._analysis_active: self._clear_accumulation_buffers() @@ -534,18 +642,20 @@ def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None) with BatchStatSnapshot(self.model, x): # Accumulate sample-wise metrics self_metrics = self.sample_calc.compute( - self.model, self.criterion, x, y, - ) - self._accum_chi_net.append( - self_metrics["batch_grad_norms_network"] - ) - self._accum_chi_loss.append( - self_metrics["batch_grad_norms_loss"] + self.model, + self.criterion, + x, + y, ) + self._accum_chi_net.append(self_metrics["batch_grad_norms_network"]) + self._accum_chi_loss.append(self_metrics["batch_grad_norms_loss"]) if x2 is not None and y2 is not None: cross_preliminary = self.sample_calc.compute( - self.model, self.criterion, x2, y2, + self.model, + self.criterion, + x2, + y2, ) cross_metrics = self.sample_calc.compute_cross_metrics( sample_wise_metrics_self=self_metrics, @@ -559,25 +669,23 @@ def _analyze_accumulated_step(self, batch, batch_idx, cross_response_batch=None) ) # Accumulate linearizer gradients (train side) - self._accumulate_linearizer_grads( - x, y, is_train=True - ) - # Accumulate linearizer gradients (measure side) - # TODO: How would a measure batchsize different to the effective batch size work here? - # !!! We would need to accumulate separately and then combine at the end. + self._accumulate_linearizer_grads(x, y, is_train=True) + # Accumulate linearizer gradients (measure side). This legacy + # path ties the measure batch to the train accumulation cycle + # (CombinedLoader "measure" key, size = K_train * micro). For + # an independently sized (and independently accumulated) + # measure batch, use measure_dataloader/measure_batch_size + # instead (see _measure_response), which accumulates and + # combines the measure side separately with its own K. if x2 is not None and y2 is not None: - self._accumulate_linearizer_grads( - x2, y2, is_train=False - ) + self._accumulate_linearizer_grads(x2, y2, is_train=False) # Restore training grads clobbered by analysis backward passes for p, s in zip(self.model.parameters(), saved_grads): p.grad = s # On last micro-batch: finalize and log - is_last = ( - self._accumulation_count == self.accumulation_steps - 1 - ) + is_last = self._accumulation_count == self.accumulation_steps - 1 if is_last: self._finalize_accumulated_analysis(x, x2) @@ -603,9 +711,7 @@ def _accumulate_linearizer_grads(self, x, y, is_train=True): for p in self.model.parameters() ] else: - for acc, p in zip( - self._accum_grad_train, self.model.parameters() - ): + for acc, p in zip(self._accum_grad_train, self.model.parameters()): if acc is not None and p.grad is not None: acc.add_(p.grad) else: @@ -642,9 +748,8 @@ def _finalize_accumulated_analysis(self, x, x2): # Compute self linearizer result from accumulated grads grad_norm_sq = sum( - (g ** 2).sum().item() - for g in self._accum_grad_train if g is not None - ) / (K ** 2) + (g**2).sum().item() for g in self._accum_grad_train if g is not None + ) / (K**2) avg_train_loss = self._accum_train_loss / K delta_loss_self = -grad_norm_sq @@ -679,7 +784,7 @@ def _finalize_accumulated_analysis(self, x, x2): self._accum_grad_measure, ) if g1 is not None and g2 is not None - ) / (K ** 2) + ) / (K**2) avg_measure_loss = self._accum_measure_loss / K delta_loss_cross = -cross_dot @@ -720,6 +825,15 @@ def _finalize_accumulated_analysis(self, x, x2): self.log("window_center", window_info["window_center"]) self.log("window_width", window_info["window_width"]) + if self._independent_measure: + grad_train_mean = [ + g / K if g is not None else None for g in self._accum_grad_train + ] + self._measure_response( + grad_train_mean=grad_train_mean, + self_chi_metrics=samples_result_self, + ) + self._clear_accumulation_buffers() def _clear_accumulation_buffers(self): @@ -734,6 +848,173 @@ def _clear_accumulation_buffers(self): self._accum_measure_loss = 0.0 self._accum_step_losses.clear() + def _next_measure_micro_batch(self): + """Pull one measure micro-batch from the persistent iterator. + + The iterator is created lazily on first use and refilled on + exhaustion so a finite measure_dataloader cycles indefinitely + (a MultiEpochsDataLoader is already infinite and simply keeps + yielding). Returns tensors moved to self.device. + """ + if self._measure_iter is None: + self._measure_iter = iter(self._measure_dataloader) + try: + xb, yb = next(self._measure_iter) + except StopIteration: + self._measure_iter = iter(self._measure_dataloader) + xb, yb = next(self._measure_iter) + + micro = self._measure_micro_batch_size + if xb.shape[0] != micro: + raise ValueError( + f"measure_dataloader yielded a micro-batch of size " + f"{xb.shape[0]}, expected {micro}. Use drop_last=True " + f"(or a dataset size divisible by batch_size) so every " + f"measure micro-batch is full." + ) + return xb.to(self.device), yb.to(self.device) + + def _measure_response(self, grad_train_mean, self_chi_metrics): + """Compute and log cross-response metrics against an independent + measure batch, decoupled from the training accumulation. + + Gathers a single pool of size max(measure_batch_size) from the + persistent measure iterator, then processes every requested + measure batch size largest-to-smallest: the largest uses the + whole pool, each smaller size uses a seed-fixable random subset + of that same pool (so subset draws are reproducible given + measure_subset_seed). For each size, the measure gradient and + per-sample chi metrics are accumulated over its own micro-batch + chunks (measure-side gradient accumulation), then combined and + logged with a "@bs{S}" suffix when sweeping multiple sizes. + + Args: + grad_train_mean: List aligned with self.model.parameters(), + the mean training gradient (accumulated grad / K_train). + self_chi_metrics: Dict with "batch_grad_norms_network" and + "batch_grad_norms_loss", the aggregated train self chi + metrics used as the "self" side of the cross metric. + + Note: + Runs its own forward/backward passes; saves and restores + self.model gradients so the surrounding (possibly partially + accumulated) training gradient is never corrupted. + """ + saved_grads = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + + sizes = sorted(self._measure_batch_sizes, reverse=True) + S_max = sizes[0] + micro = self._measure_micro_batch_size + sweep = len(self._measure_batch_sizes) > 1 + + # Gather ONE pool of S_max samples from the persistent iterator. + # S_max need not be a multiple of micro (e.g. every swept size is + # below the max single-pass size), so pull enough micro-batches + # to cover it and slice down to exactly S_max samples. + pool_x, pool_y = [], [] + for _ in range(math.ceil(S_max / micro)): + xb, yb = self._next_measure_micro_batch() + pool_x.append(xb) + pool_y.append(yb) + x_pool = torch.cat(pool_x, dim=0)[:S_max] + y_pool = torch.cat(pool_y, dim=0)[:S_max] + + for S in sizes: + if S == S_max: + idx = torch.arange(S_max, device=x_pool.device) + else: + # Seed-fixable random subset of the same pool, drawn + # after larger sizes so the sweep is reproducible given + # measure_subset_seed regardless of how many sizes ran. + perm = torch.randperm(S_max, generator=self._measure_gen) + idx = perm[:S].to(x_pool.device) + x_sel, y_sel = x_pool[idx], y_pool[idx] + + # chunk_size = min(S, micro) unifies both regimes: when + # S <= micro this gives chunk_size=S, K_measure=1 (a single + # direct pass using less than the max single-pass capacity, + # no accumulation); when S > micro this gives + # chunk_size=micro, K_measure=S // micro (accumulated passes + # of the max single-pass size — exact, since S > micro must + # be a multiple of micro per __init__ validation). + chunk_size = min(S, micro) + K_measure = S // chunk_size + grad_measure_acc = None + measure_loss_sum = 0.0 + chi_net_chunks, chi_loss_chunks = [], [] + + for k in range(K_measure): + xc = x_sel[k * chunk_size : (k + 1) * chunk_size] + yc = y_sel[k * chunk_size : (k + 1) * chunk_size] + with BatchStatSnapshot(self.model, xc): + m = self.sample_calc.compute(self.model, self.criterion, xc, yc) + chi_net_chunks.append(m["batch_grad_norms_network"]) + chi_loss_chunks.append(m["batch_grad_norms_loss"]) + + self.model.zero_grad() + loss = self.criterion(self.model(xc), yc) + loss.backward() + measure_loss_sum += loss.detach().item() + if grad_measure_acc is None: + grad_measure_acc = [ + p.grad.clone() if p.grad is not None else None + for p in self.model.parameters() + ] + else: + for acc, p in zip( + grad_measure_acc, self.model.parameters() + ): + if acc is not None and p.grad is not None: + acc.add_(p.grad) + + measure_chi = { + "batch_grad_norms_network": sum(chi_net_chunks) / K_measure, + "batch_grad_norms_loss": sum(chi_loss_chunks) / K_measure, + } + cross_metrics = self.sample_calc.compute_cross_metrics( + sample_wise_metrics_self=self_chi_metrics, + sample_wise_metrics_cross=measure_chi, + ) + + grad_measure_mean = [ + g / K_measure if g is not None else None for g in grad_measure_acc + ] + cross_dot = sum( + (g1 * g2).sum().item() + for g1, g2 in zip(grad_train_mean, grad_measure_mean) + if g1 is not None and g2 is not None + ) + avg_measure_loss = measure_loss_sum / K_measure + delta_loss_cross = -cross_dot + probe_result_cross = ( + avg_measure_loss, + avg_measure_loss + delta_loss_cross, + delta_loss_cross, + ) + chi_coup_cross = self.coupling_calc.calculate( + delta_loss=delta_loss_cross, + chi_loss=cross_metrics["batch_grad_norms_loss"], + chi_net=cross_metrics["batch_grad_norms_network"], + ) + + if self.log_metrics: + suffix = f"@bs{S}" if sweep else "" + self._log_analysis_results( + prefix="cross_", + samples_result=cross_metrics, + probe_result=probe_result_cross, + chi_coup=chi_coup_cross, + batch_size=S, + suffix=suffix, + log_effective_batch_size=False, + ) + + for p, s in zip(self.model.parameters(), saved_grads): + p.grad = s + def _log_analysis_results( self, prefix: str, @@ -741,33 +1022,56 @@ def _log_analysis_results( probe_result: tuple, chi_coup: Optional[float], batch_size: int, + suffix: str = "", + log_effective_batch_size: Optional[bool] = None, ): - """Helper method to log analysis metrics with a given prefix.""" + """Helper method to log analysis metrics with a given prefix. + + Args: + suffix: Appended to every logged key. Used to disambiguate + swept measure batch sizes, e.g. "@bs2000". + log_effective_batch_size: Whether to additionally log + "{prefix}effective_batch_size{suffix}" as + batch_size * accumulation_steps. Defaults to + accumulation_steps > 1 (existing behavior). Independent + measure batches pass False explicitly since the train + accumulation_steps multiplier has no meaning for them + (batch_size is already the full measure size). + """ + if log_effective_batch_size is None: + log_effective_batch_size = self.accumulation_steps > 1 + # Log sample-wise metrics if "batch_grad_norms_network" in samples_result: - self.log(f"{prefix}chi_net", samples_result["batch_grad_norms_network"]) + self.log( + f"{prefix}chi_net{suffix}", + samples_result["batch_grad_norms_network"], + ) if "batch_grad_norms_loss" in samples_result: - self.log(f"{prefix}chi_loss", samples_result["batch_grad_norms_loss"]) + self.log( + f"{prefix}chi_loss{suffix}", + samples_result["batch_grad_norms_loss"], + ) # Log coupling if provided if chi_coup is not None: - self.log(f"{prefix}chi_coup", chi_coup) + self.log(f"{prefix}chi_coup{suffix}", chi_coup) - self.log(f"{prefix}batch_size", batch_size) - if self.accumulation_steps > 1: + self.log(f"{prefix}batch_size{suffix}", batch_size) + if log_effective_batch_size: self.log( - f"{prefix}effective_batch_size", + f"{prefix}effective_batch_size{suffix}", batch_size * self.accumulation_steps, ) - # Only log analysis_step once (usually with empty prefix) - if prefix == "": + # Only log analysis_step once (usually with empty prefix/suffix) + if prefix == "" and suffix == "": self.log("analysis_step", self.effective_step) # Log probe results (linearization) if probe_result is not None: loss, _, delta_loss = probe_result - self.log(f"{prefix}loss", loss) + self.log(f"{prefix}loss{suffix}", loss) # For cross response, we might want to name it differently or keep # consistent. @@ -777,7 +1081,7 @@ def _log_analysis_results( metric_name = ( "grad_norm_squared" if prefix == "" else "grad_dot_product" ) - self.log(f"{prefix}{metric_name}", -delta_loss) + self.log(f"{prefix}{metric_name}{suffix}", -delta_loss) def _after_training_step(self, batch, batch_idx, output): """Hook executed after the wrapped training step. @@ -802,5 +1106,8 @@ def _after_training_step(self, batch, batch_idx, output): cross_response=cross_response, micro_batch_size=micro_batch_size, effective_batch_size=effective_batch_size, + measure_dataloader=measure_dataloader, + measure_batch_size=measure_batch_size, + measure_subset_seed=measure_subset_seed, **model_kwargs, ) diff --git a/tests/integration/test_analyzer_deployment.py b/tests/integration/test_analyzer_deployment.py index ed92e08..11e0711 100644 --- a/tests/integration/test_analyzer_deployment.py +++ b/tests/integration/test_analyzer_deployment.py @@ -1247,6 +1247,278 @@ def mock_log(name, value, *args, **kwargs): assert not torch.isnan(torch.tensor(logged_metrics["cross_grad_dot_product"])) +class TestAnalyzerWithIndependentMeasure: + """Integration tests for the independent measure_dataloader / + measure_batch_size path (decoupled from train batch/accumulation).""" + + class _MetricsTracker(pl.Callback): + """Collects trainer.callback_metrics after every training batch.""" + + def __init__(self): + self.metrics = [] + + def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx): + self.metrics.append(dict(trainer.callback_metrics)) + + def test_independent_measure_end_to_end(self, simple_lightning_module): + """A plain train DataLoader + measure_dataloader (no CombinedLoader) + trains end-to-end and logs cross-response metrics.""" + torch.manual_seed(42) + + train_x = torch.randn(32, 10) + train_y = torch.randint(0, 2, (32,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=8, drop_last=True + ) + + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_epochs=1, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + last = tracker.metrics[-1] + for key in ( + "cross_loss", + "cross_grad_dot_product", + "cross_chi_net", + "cross_chi_loss", + "cross_chi_coup", + ): + assert key in last + + def test_independent_measure_with_accumulation(self, simple_lightning_module): + """Independent measure batch combined with train-side gradient + accumulation exercises the _finalize_accumulated_analysis call site.""" + torch.manual_seed(42) + + train_x = torch.randn(32, 10) + train_y = torch.randint(0, 2, (32,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=4) + + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + measure_dataloader=measure_loader, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_epochs=1, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + logged_cross = [m for m in tracker.metrics if "cross_grad_dot_product" in m] + assert len(logged_cross) > 0 + + def test_independent_measure_sweep_suffixed_keys(self, simple_lightning_module): + """A measure_batch_size sweep under a logarithmic schedule logs + @bs{S}-suffixed cross metrics instead of the unsuffixed cross_* keys.""" + from perspic.logger import logarithmic_windows + + torch.manual_seed(42) + + train_x = torch.randn(64, 10) + train_y = torch.randint(0, 2, (64,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + schedule = logarithmic_windows(max_steps=8) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + measure_subset_seed=0, + analysis_schedule=schedule, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_steps=4, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + all_keys = set() + for m in tracker.metrics: + all_keys.update(m.keys()) + + assert "cross_grad_dot_product@bs4" in all_keys + assert "cross_grad_dot_product@bs8" in all_keys + assert "cross_grad_dot_product" not in all_keys + + def test_independent_measure_sweep_mixed_regime(self, simple_lightning_module): + """A sweep spanning sizes below, at, and above the measure loader's + batch_size (the max single-pass size) trains end-to-end and logs + every size: below/at it as a single direct pass, above it as an + accumulated measurement.""" + from perspic.logger import logarithmic_windows + + torch.manual_seed(42) + + train_x = torch.randn(64, 10) + train_y = torch.randint(0, 2, (64,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + measure_x = torch.randn(32, 10) + measure_y = torch.randint(0, 2, (32,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=8, drop_last=True + ) + + schedule = logarithmic_windows(max_steps=8) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[2, 8, 32], + measure_subset_seed=0, + analysis_schedule=schedule, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_steps=4, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + all_keys = set() + for m in tracker.metrics: + all_keys.update(m.keys()) + + for S in (2, 8, 32): + assert f"cross_grad_dot_product@bs{S}" in all_keys + assert f"cross_batch_size@bs{S}" in all_keys + + def test_independent_measure_reproducible(self, simple_lightning_module): + """Two fits with the same measure_subset_seed produce an identical + swept cross metric (the measure loader is shuffle=False, so only the + subset draw needs a fixed seed for full reproducibility).""" + from perspic.logger import logarithmic_windows + + def run(): + torch.manual_seed(42) + train_x = torch.randn(32, 10) + train_y = torch.randint(0, 2, (32,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=8) + + torch.manual_seed(7) + measure_x = torch.randn(16, 10) + measure_y = torch.randint(0, 2, (16,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + schedule = logarithmic_windows(max_steps=8) + torch.manual_seed(42) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + measure_subset_seed=99, + analysis_schedule=schedule, + log_metrics=True, + ) + + tracker = self._MetricsTracker() + trainer = pl.Trainer( + max_steps=4, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + callbacks=[tracker], + ) + trainer.fit(model, train_loader) + + for m in reversed(tracker.metrics): + if "cross_grad_dot_product@bs4" in m: + return m["cross_grad_dot_product@bs4"] + return None + + val_a = run() + val_b = run() + + assert val_a is not None + assert torch.allclose(torch.as_tensor(val_a), torch.as_tensor(val_b)) + + def test_measure_loader_smaller_than_train(self, simple_lightning_module): + """A measure_dataloader much smaller than the training run cycles via + the persistent iterator without raising StopIteration.""" + torch.manual_seed(42) + + train_x = torch.randn(64, 10) + train_y = torch.randint(0, 2, (64,)) + train_loader = DataLoader(TensorDataset(train_x, train_y), batch_size=4) + + measure_x = torch.randn(4, 10) + measure_y = torch.randint(0, 2, (4,)) + measure_loader = DataLoader( + TensorDataset(measure_x, measure_y), batch_size=4, drop_last=True + ) + + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + + trainer = pl.Trainer( + max_steps=16, + accelerator="cpu", + enable_progress_bar=False, + enable_model_summary=False, + logger=False, + ) + + # Should complete without StopIteration + trainer.fit(model, train_loader) + assert trainer.global_step == 16 + + class TestSchedulerIntegration: """Integration tests for learning rate schedulers.""" diff --git a/tests/unit/test_analyzer.py b/tests/unit/test_analyzer.py index 3266b43..c68b86e 100644 --- a/tests/unit/test_analyzer.py +++ b/tests/unit/test_analyzer.py @@ -1,5 +1,6 @@ """Unit tests for the analyzer module.""" +import warnings from unittest.mock import Mock, patch import pytest @@ -7,6 +8,7 @@ import torch import torch.nn as nn import torch.nn.functional as F +from torch.utils.data import DataLoader, TensorDataset from perspic.analyzer import analyzer from perspic.calculator.linearizer import Linearizer @@ -104,6 +106,16 @@ def sample_batch(): return x, y +def _make_measure_dataloader(n_samples, micro, seed=123, drop_last=True): + """Build a deterministic measure DataLoader (shuffle=False) for testing + the independent measure-batch-size path.""" + g = torch.Generator().manual_seed(seed) + x = torch.randn(n_samples, 10, generator=g) + y = torch.randint(0, 2, (n_samples,), generator=g) + ds = TensorDataset(x, y) + return DataLoader(ds, batch_size=micro, shuffle=False, drop_last=drop_last) + + # Test Classes class TestAnalyzerFactoryFunction: """Test the analyzer factory function.""" @@ -786,31 +798,24 @@ def test_no_window_info_without_schedule( class TestGradientAccumulation: """Test gradient accumulation functionality.""" + # The tests are categorized into sections A-J for clarity. # --- A. Parameter validation --- - def test_accumulation_steps_default( - self, simple_lightning_module - ): + def test_accumulation_steps_default(self, simple_lightning_module): """No params → accumulation_steps=1.""" model = analyzer(simple_lightning_module) assert model.accumulation_steps == 1 - def test_batch_size_only_no_accumulation( - self, simple_lightning_module - ): + def test_batch_size_only_no_accumulation(self, simple_lightning_module): """micro_batch_size alone → no accumulation, value stored.""" - model = analyzer( - simple_lightning_module, micro_batch_size=8 - ) + model = analyzer(simple_lightning_module, micro_batch_size=8) assert model.accumulation_steps == 1 assert model.micro_batch_size == 8 assert model.effective_batch_size is None - def test_accumulation_steps_computed( - self, simple_lightning_module - ): + def test_accumulation_steps_computed(self, simple_lightning_module): """micro=8, effective=32 → accumulation_steps=4.""" model = analyzer( simple_lightning_module, @@ -819,55 +824,39 @@ def test_accumulation_steps_computed( ) assert model.accumulation_steps == 4 - def test_effective_without_micro_batch_raises( - self, simple_lightning_module - ): + def test_effective_without_micro_batch_raises(self, simple_lightning_module): """effective_batch_size alone → ValueError.""" - with pytest.raises( - ValueError, match="micro_batch_size must be specified" - ): + with pytest.raises(ValueError, match="micro_batch_size must be specified"): analyzer( simple_lightning_module, effective_batch_size=32, ) - def test_effective_less_than_micro_batch_raises( - self, simple_lightning_module - ): + def test_effective_less_than_micro_batch_raises(self, simple_lightning_module): """effective=8, micro=32 → ValueError.""" - with pytest.raises( - ValueError, match="must be >= micro_batch_size" - ): + with pytest.raises(ValueError, match="must be >= micro_batch_size"): analyzer( simple_lightning_module, micro_batch_size=32, effective_batch_size=8, ) - def test_not_divisible_raises( - self, simple_lightning_module - ): + def test_not_divisible_raises(self, simple_lightning_module): """effective=30, micro=8 → ValueError (not divisible).""" - with pytest.raises( - ValueError, match="must be divisible" - ): + with pytest.raises(ValueError, match="must be divisible"): analyzer( simple_lightning_module, micro_batch_size=8, effective_batch_size=30, ) - def test_accumulation_with_delegate_raises( - self, manual_optimization_module - ): + def test_accumulation_with_delegate_raises(self, manual_optimization_module): """Accumulation + manual optimization module → ValueError.""" with pytest.raises( ValueError, match="Gradient accumulation is not supported", ): - with pytest.warns( - UserWarning, match="manual optimization" - ): + with pytest.warns(UserWarning, match="manual optimization"): analyzer( manual_optimization_module, micro_batch_size=8, @@ -876,9 +865,7 @@ def test_accumulation_with_delegate_raises( # --- B. Optimizer behavior --- - def test_zero_grad_once_per_cycle( - self, simple_lightning_module, sample_batch - ): + def test_zero_grad_once_per_cycle(self, simple_lightning_module, sample_batch): """4 micro-steps, accum=4: zero_grad called exactly 1x.""" model = analyzer( simple_lightning_module, @@ -896,9 +883,7 @@ def test_zero_grad_once_per_cycle( assert mock_opt.zero_grad.call_count == 1 - def test_step_once_per_cycle( - self, simple_lightning_module, sample_batch - ): + def test_step_once_per_cycle(self, simple_lightning_module, sample_batch): """4 micro-steps, accum=4: opt.step() called exactly 1x.""" model = analyzer( simple_lightning_module, @@ -916,9 +901,7 @@ def test_step_once_per_cycle( assert mock_opt.step.call_count == 1 - def test_step_not_called_mid_cycle( - self, simple_lightning_module, sample_batch - ): + def test_step_not_called_mid_cycle(self, simple_lightning_module, sample_batch): """3 of 4 micro-steps done: opt.step() never called.""" model = analyzer( simple_lightning_module, @@ -936,9 +919,7 @@ def test_step_not_called_mid_cycle( mock_opt.step.assert_not_called() - def test_loss_scaled_for_backward( - self, simple_lightning_module, sample_batch - ): + def test_loss_scaled_for_backward(self, simple_lightning_module, sample_batch): """manual_backward receives loss / accumulation_steps.""" model = analyzer( simple_lightning_module, @@ -957,9 +938,7 @@ def test_loss_scaled_for_backward( expected = output / 4 assert torch.allclose(backward_arg, expected) - def test_unscaled_loss_returned( - self, simple_lightning_module, sample_batch - ): + def test_unscaled_loss_returned(self, simple_lightning_module, sample_batch): """training_step returns the original unscaled loss.""" model = analyzer( simple_lightning_module, @@ -975,9 +954,7 @@ def test_unscaled_loss_returned( output_accum = model.training_step((x, y), 0) assert isinstance(output_accum, torch.Tensor) - def test_two_full_cycles( - self, simple_lightning_module, sample_batch - ): + def test_two_full_cycles(self, simple_lightning_module, sample_batch): """4 steps, accum=2: zero_grad 2x, opt.step() 2x.""" model = analyzer( simple_lightning_module, @@ -1002,9 +979,7 @@ def test_no_accumulation_backwards_compatible( self, simple_lightning_module, sample_batch ): """No accum params: every call does zero_grad + step.""" - model = analyzer( - simple_lightning_module, disable_analyzer=True - ) + model = analyzer(simple_lightning_module, disable_analyzer=True) mock_opt = Mock(zero_grad=Mock(), step=Mock()) model.optimizers = Mock(return_value=mock_opt) model.manual_backward = Mock() @@ -1037,10 +1012,7 @@ def test_single_step_path_unchanged( model._before_training_step((x, y), 0) - logged = { - call[0][0]: call[0][1] - for call in model.log.call_args_list - } + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} assert torch.allclose(logged["chi_net"], torch.tensor(1.5)) assert torch.allclose(logged["chi_loss"], torch.tensor(2.5)) assert logged["loss"] == 1.0 @@ -1049,9 +1021,7 @@ def test_single_step_path_unchanged( # --- D. Effective step --- - def test_effective_step_with_accumulation( - self, simple_lightning_module - ): + def test_effective_step_with_accumulation(self, simple_lightning_module): """_optimizer_step_count=2 → effective_step=2.""" model = analyzer( simple_lightning_module, @@ -1064,9 +1034,7 @@ def test_effective_step_with_accumulation( model._optimizer_step_count = 0 assert model.effective_step == 0 - def test_effective_step_without_accumulation( - self, simple_lightning_module - ): + def test_effective_step_without_accumulation(self, simple_lightning_module): """_optimizer_step_count=42 → effective_step=42.""" model = analyzer(simple_lightning_module) model._optimizer_step_count = 42 @@ -1154,10 +1122,7 @@ def test_analysis_step_logs_effective_step( model._accumulation_count = 1 model._before_training_step((x, y), 1) - logged = { - call[0][0]: call[0][1] - for call in model.log.call_args_list - } + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} assert logged["analysis_step"] == 3 # --- F. Sample-wise metric accumulation --- @@ -1192,10 +1157,7 @@ def test_accumulated_chi_net_is_mean( model._accumulation_count = 1 model._before_training_step((x, y), 1) - logged = { - call[0][0]: call[0][1] - for call in model.log.call_args_list - } + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} # chi_net_eff = mean([2.0, 4.0]) = 3.0 assert torch.allclose(logged["chi_net"], torch.tensor(3.0)) @@ -1235,10 +1197,7 @@ def test_accumulated_chi_loss_is_mean( model._accumulation_count = 1 model._before_training_step((x, y), 1) - logged = { - call[0][0]: call[0][1] - for call in model.log.call_args_list - } + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} # chi_loss_eff = mean([3.0, 5.0]) = 4.0 assert torch.allclose(logged["chi_loss"], torch.tensor(4.0)) @@ -1260,7 +1219,8 @@ def test_linearizer_accumulated_grad_norm( # Run a full accumulation cycle (K=2) with patch.object( - SamplewiseCalculatorOpacus, "compute", + SamplewiseCalculatorOpacus, + "compute", return_value={ "batch_grad_norms_network": torch.tensor(1.0), "batch_grad_norms_loss": torch.tensor(1.0), @@ -1271,10 +1231,7 @@ def test_linearizer_accumulated_grad_norm( model._accumulation_count = 1 model._before_training_step((x, y), 1) - logged = { - call[0][0]: call[0][1] - for call in model.log.call_args_list - } + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} # grad_norm_squared should be a positive float assert "grad_norm_squared" in logged @@ -1286,23 +1243,20 @@ def test_linearizer_accumulated_grad_norm( loss0 = model.criterion(model.model(x), y) loss0.backward() grads_0 = [ - p.grad.clone() for p in model.model.parameters() - if p.grad is not None + p.grad.clone() for p in model.model.parameters() if p.grad is not None ] model.model.zero_grad() loss1 = model.criterion(model.model(x), y) loss1.backward() grads_1 = [ - p.grad.clone() for p in model.model.parameters() - if p.grad is not None + p.grad.clone() for p in model.model.parameters() if p.grad is not None ] model.model.zero_grad() - expected_norm_sq = sum( - ((g0 + g1) ** 2).sum().item() - for g0, g1 in zip(grads_0, grads_1) - ) / 4 # K² = 2² = 4 + expected_norm_sq = ( + sum(((g0 + g1) ** 2).sum().item() for g0, g1 in zip(grads_0, grads_1)) / 4 + ) # K² = 2² = 4 assert abs(logged["grad_norm_squared"] - expected_norm_sq) < 1e-4 @@ -1322,7 +1276,8 @@ def test_coupling_from_accumulated_values( x, y = sample_batch with patch.object( - SamplewiseCalculatorOpacus, "compute", + SamplewiseCalculatorOpacus, + "compute", return_value={ "batch_grad_norms_network": torch.tensor(2.0), "batch_grad_norms_loss": torch.tensor(3.0), @@ -1333,17 +1288,14 @@ def test_coupling_from_accumulated_values( model._accumulation_count = 1 model._before_training_step((x, y), 1) - logged = { - call[0][0]: call[0][1] - for call in model.log.call_args_list - } + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} # chi_net_eff = mean([2.0, 2.0]) = 2.0 # chi_loss_eff = mean([3.0, 3.0]) = 3.0 # coupling = grad_norm_sq / (3.0 * 2.0) assert "chi_coup" in logged - expected_coupling = ( - logged["grad_norm_squared"] / (logged["chi_loss"] * logged["chi_net"]) + expected_coupling = logged["grad_norm_squared"] / ( + logged["chi_loss"] * logged["chi_net"] ) assert abs(logged["chi_coup"] - expected_coupling) < 1e-5 @@ -1402,10 +1354,7 @@ def test_effective_batch_size_logged( model._accumulation_count = 1 model._before_training_step((x, y), 1) - logged = { - call[0][0]: call[0][1] - for call in model.log.call_args_list - } + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} assert "effective_batch_size" in logged # micro_batch_size=4, accumulation_steps=2 → 4*2=8 @@ -1486,9 +1435,7 @@ def side_effect(*args, **kwargs): # Get chi_net from cycle 2 (the last logged value) chi_net_calls = [ - call[0][1] - for call in model.log.call_args_list - if call[0][0] == "chi_net" + call[0][1] for call in model.log.call_args_list if call[0][0] == "chi_net" ] # Cycle 1: mean([10, 10]) = 10, Cycle 2: mean([20, 20]) = 20 assert len(chi_net_calls) == 2 @@ -1542,3 +1489,595 @@ def run_two_steps(with_analysis): f"Analysis pass corrupted training gradients: " f"max diff {(g_with - g_without).abs().max().item()}" ) + + +class TestIndependentMeasureResponse: + """Test the independent measure_dataloader / measure_batch_size path.""" + + # --- A. Parameter validation --- + + def test_infers_measure_micro_from_dataloader(self, simple_lightning_module): + """measure micro-batch size is inferred from the loader's batch_size.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer(simple_lightning_module, measure_dataloader=measure_loader) + assert model._measure_micro_batch_size == 4 + assert model._measure_batch_sizes == [4] + + def test_measure_batch_size_int_normalized_to_list(self, simple_lightning_module): + """A scalar measure_batch_size is normalized to a length-1 list.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=8, + ) + assert model._measure_batch_sizes == [8] + + def test_measure_batch_size_not_divisible_raises(self, simple_lightning_module): + """measure_batch_size above micro, not an exact multiple, raises.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + with pytest.raises(ValueError, match="exact multiple"): + analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=6, + ) + + def test_measure_batch_size_below_micro_allowed(self, simple_lightning_module): + """measure_batch_size below the measure micro size is valid: it runs + as a single direct pass (no accumulation), not an error.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=2, + ) + assert model._measure_batch_sizes == [2] + + def test_measure_dataloader_batch_size_none_raises(self, simple_lightning_module): + """A measure_dataloader with batch_size=None (manual batching) raises.""" + ds = TensorDataset(torch.randn(4, 10), torch.randint(0, 2, (4,))) + loader = DataLoader(ds, batch_size=None) + with pytest.raises(ValueError, match="batch_size"): + analyzer(simple_lightning_module, measure_dataloader=loader) + + def test_measure_params_without_dataloader_raises(self, simple_lightning_module): + """measure_batch_size/measure_subset_seed require measure_dataloader.""" + with pytest.raises( + ValueError, match="only valid together with measure_dataloader" + ): + analyzer(simple_lightning_module, measure_batch_size=8) + + def test_sweep_without_schedule_warns(self, simple_lightning_module): + """A multi-size sweep without an analysis_schedule warns.""" + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + with pytest.warns(UserWarning, match="batch-size sweep"): + analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + ) + + def test_sweep_with_schedule_does_not_warn(self, simple_lightning_module): + """A multi-size sweep with a logarithmic analysis_schedule doesn't warn.""" + from perspic.logger import logarithmic_windows + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + schedule = logarithmic_windows(max_steps=100) + + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + analysis_schedule=schedule, + ) + assert not any("batch-size sweep" in str(w.message) for w in record) + + # --- B. Single-measure logging behavior --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_logs_cross_keys_single_measure( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """A single measure_batch_size logs the existing unsuffixed cross_* keys.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + model._before_training_step((x, y), 0) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + for key in ( + "cross_chi_net", + "cross_chi_loss", + "cross_chi_coup", + "cross_loss", + "cross_grad_dot_product", + "cross_batch_size", + ): + assert key in logged + assert logged["cross_batch_size"] == 8 + assert "cross_chi_net@bs8" not in logged + + @patch.object(SamplewiseCalculatorOpacus, "compute_cross_metrics") + @patch.object(SamplewiseCalculatorOpacus, "compute") + def test_measure_chi_is_mean_over_kmeasure( + self, + mock_compute, + mock_compute_cross, + simple_lightning_module, + sample_batch, + ): + """Measure-side chi is averaged over K_measure = S // measure_micro, + not the (different) train accumulation_steps.""" + # 3 train micro-batches (accumulation_steps=3), then 2 measure chunks + # (measure_batch_size=8, measure_micro=4 -> K_measure=2). + values = [ + { + "batch_grad_norms_network": torch.tensor(2.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(4.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(6.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(10.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + { + "batch_grad_norms_network": torch.tensor(20.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ] + mock_compute.side_effect = values + mock_compute_cross.return_value = { + "batch_grad_norms_network": torch.tensor(0.0), + "batch_grad_norms_loss": torch.tensor(0.0), + } + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=5) + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=12, + measure_dataloader=measure_loader, + measure_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._accumulation_count = 0 + model._before_training_step((x, y), 0) + model._accumulation_count = 1 + model._before_training_step((x, y), 1) + model._accumulation_count = 2 + model._before_training_step((x, y), 2) + + assert mock_compute_cross.call_count == 1 + _, kwargs = mock_compute_cross.call_args + measure_agg = kwargs["sample_wise_metrics_cross"] + # mean([10.0, 20.0]) = 15.0, NOT sum/3 (train K) == 10.0 + assert torch.allclose( + measure_agg["batch_grad_norms_network"], torch.tensor(15.0) + ) + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_measure_grad_dot_reference( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """cross_grad_dot_product matches a hand-computed .""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=4, micro=4, seed=99) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._before_training_step((x, y), 0) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + assert "cross_grad_dot_product" in logged + + # Hand-compute: grad_train = ∇L(x, y); grad_measure = ∇L(x_m, y_m), + # where x_m, y_m is the same (only) batch measure_loader yields + # (shuffle=False), reused via a fresh iterator over the same loader. + x_m, y_m = next(iter(measure_loader)) + + model.model.zero_grad() + loss_t = model.criterion(model.model(x), y) + loss_t.backward() + grad_train = [p.grad.clone() for p in model.model.parameters()] + + model.model.zero_grad() + loss_m = model.criterion(model.model(x_m), y_m) + loss_m.backward() + grad_measure = [p.grad.clone() for p in model.model.parameters()] + model.model.zero_grad() + + expected_dot = sum( + (g1 * g2).sum().item() for g1, g2 in zip(grad_train, grad_measure) + ) + assert abs(logged["cross_grad_dot_product"] - expected_dot) < 1e-4 + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_direct_pass_below_micro_reference( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """A measure_batch_size below the measure micro size runs as a + single direct pass (K_measure=1, no accumulation/padding) on exactly + S samples taken from the front of the gathered pool.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=8, micro=8, seed=99) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=4, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + model._before_training_step((x, y), 0) + + # 1 call for the train self chi + exactly 1 measure-side call + # (K_measure=1: a single direct pass, not padded up to micro=8). + assert mock_compute.call_count == 2 + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + assert logged["cross_batch_size"] == 4 + + # Hand-compute the reference: the pool is built from ceil(4/8)=1 + # micro-batch pull (the loader's only 8-sample batch), sliced to the + # first S_max=4 samples. + x_full, y_full = next(iter(measure_loader)) + x_m, y_m = x_full[:4], y_full[:4] + + model.model.zero_grad() + loss_t = model.criterion(model.model(x), y) + loss_t.backward() + grad_train = [p.grad.clone() for p in model.model.parameters()] + + model.model.zero_grad() + loss_m = model.criterion(model.model(x_m), y_m) + loss_m.backward() + grad_measure = [p.grad.clone() for p in model.model.parameters()] + model.model.zero_grad() + + expected_dot = sum( + (g1 * g2).sum().item() for g1, g2 in zip(grad_train, grad_measure) + ) + assert abs(logged["cross_grad_dot_product"] - expected_dot) < 1e-4 + + def test_measure_backward_does_not_corrupt_training_grad( + self, simple_lightning_module, sample_batch + ): + """Measure-side backward passes inside _measure_response must not + corrupt the (possibly partial) accumulated training gradient.""" + x, y = sample_batch + + def run_two_steps(with_measure): + torch.manual_seed(0) + kwargs = {} + if with_measure: + measure_loader = _make_measure_dataloader(n_samples=4, micro=4, seed=3) + kwargs = {"measure_dataloader": measure_loader} + + model = analyzer( + simple_lightning_module, + micro_batch_size=4, + effective_batch_size=8, + log_metrics=False, + **kwargs, + ) + model.log = Mock() + opt = torch.optim.SGD(model.parameters(), lr=0.0) + model.optimizers = Mock(return_value=opt) + model.manual_backward = lambda loss: loss.backward() + model._trainer = None + + opt.zero_grad() + model._accumulation_count = 0 + model.training_step((x, y), 0) + model.training_step((x, y), 1) + + return [ + p.grad.clone() if p.grad is not None else None + for p in model.model.parameters() + ] + + grads_with = run_two_steps(with_measure=True) + grads_without = run_two_steps(with_measure=False) + + for g_with, g_without in zip(grads_with, grads_without): + assert g_with is not None and g_without is not None + assert torch.allclose(g_with, g_without, atol=1e-6), ( + f"Measure-response backward corrupted training gradients: " + f"max diff {(g_with - g_without).abs().max().item()}" + ) + + # --- C. Subset sampling --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_measure_subset_deterministic_with_seed( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """The same measure_subset_seed yields the same subset (and hence the + same swept cross metric); a different seed yields a different one.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + x, y = sample_batch + + def run(seed): + # Fix the model init seed too: cross_grad_dot_product is a real + # gradient dot product, so it depends on model weights as well + # as on which subset is drawn. + torch.manual_seed(0) + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=11) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[8, 4], + measure_subset_seed=seed, + log_metrics=True, + ) + model.log = Mock() + model._before_training_step((x, y), 0) + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + return logged["cross_grad_dot_product@bs4"] + + val_a = run(seed=123) + val_b = run(seed=123) + val_c = run(seed=456) + + assert val_a == val_b + assert val_a != val_c + + @patch.object(Linearizer, "compute") + def test_measure_subset_size_matches_S( + self, mock_probe, simple_lightning_module, sample_batch + ): + """The number of measure micro-batch chunks processed for each swept + size equals S // measure_micro_batch_size.""" + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=16, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[16, 8], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ) as mock_compute: + model._before_training_step((x, y), 0) + # 1 call for train self chi + (16 // 4 = 4) + (8 // 4 = 2) measure + # chunks. + assert mock_compute.call_count == 1 + 4 + 2 + + @patch.object(Linearizer, "compute") + def test_pool_gather_when_S_max_below_micro( + self, mock_probe, simple_lightning_module, sample_batch + ): + """When every swept size is below the measure micro size, the pool + gather still pulls at least one micro-batch (ceil(S_max/micro)=1) + and slices it down to S_max samples, rather than pulling zero.""" + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=32, micro=16, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[2, 4, 8], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + model, + "_next_measure_micro_batch", + wraps=model._next_measure_micro_batch, + ) as mock_pull, patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ): + model._before_training_step((x, y), 0) + + # S_max=8 < micro=16 -> ceil(8/16)=1 pull, independent of how many + # sizes are swept below it. + assert mock_pull.call_count == 1 + + @patch.object(Linearizer, "compute") + def test_mixed_regime_sweep_chunk_counts( + self, mock_probe, simple_lightning_module, sample_batch + ): + """A sweep spanning sizes below, at, and above the measure micro + size produces the correct chunk count for each: K_measure=1 for + sizes <= micro (a single direct pass), K_measure=S // micro for + sizes above it (accumulated).""" + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=32, micro=8, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[2, 8, 32], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ) as mock_compute: + model._before_training_step((x, y), 0) + # 1 train self chi call + (2 // 2 = 1) + (8 // 8 = 1) + # + (32 // 8 = 4) measure chunks. + assert mock_compute.call_count == 1 + 1 + 1 + 4 + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + for S in (2, 8, 32): + assert logged[f"cross_batch_size@bs{S}"] == S + + # --- D. Sweep logging --- + + @patch.object(SamplewiseCalculatorOpacus, "compute") + @patch.object(Linearizer, "compute") + def test_sweep_logs_suffixed_keys( + self, mock_probe, mock_compute, simple_lightning_module, sample_batch + ): + """Sweeping multiple measure_batch_size values logs @bs{S}-suffixed + keys instead of the unsuffixed cross_* keys.""" + mock_compute.return_value = { + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + } + mock_probe.return_value = {"self": (1.0, 0.0, -1.0), "cross": None} + + measure_loader = _make_measure_dataloader(n_samples=8, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=[4, 8], + measure_subset_seed=0, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + model._before_training_step((x, y), 0) + + logged = {call[0][0]: call[0][1] for call in model.log.call_args_list} + for S in (4, 8): + for key in ( + "cross_chi_net", + "cross_chi_loss", + "cross_chi_coup", + "cross_grad_dot_product", + "cross_loss", + "cross_batch_size", + ): + assert f"{key}@bs{S}" in logged + assert "cross_chi_net" not in logged + + # --- E. Persistent iterator --- + + def test_measure_iterator_refills_on_exhaustion( + self, simple_lightning_module, sample_batch + ): + """A measure_dataloader smaller than what's pulled across several + analysis steps cycles instead of raising StopIteration.""" + measure_loader = _make_measure_dataloader(n_samples=4, micro=4, seed=1) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ), patch.object( + Linearizer, + "compute", + return_value={"self": (1.0, 0.0, -1.0), "cross": None}, + ): + for i in range(5): + model._before_training_step((x, y), i) + + def test_measure_partial_last_batch_raises( + self, simple_lightning_module, sample_batch + ): + """drop_last=False measure loader with a non-divisible dataset raises + a clear ValueError instead of silently building a short pool.""" + measure_loader = _make_measure_dataloader( + n_samples=6, micro=4, seed=2, drop_last=False + ) + model = analyzer( + simple_lightning_module, + measure_dataloader=measure_loader, + measure_batch_size=8, + log_metrics=True, + ) + model.log = Mock() + x, y = sample_batch + + with patch.object( + SamplewiseCalculatorOpacus, + "compute", + return_value={ + "batch_grad_norms_network": torch.tensor(1.0), + "batch_grad_norms_loss": torch.tensor(1.0), + }, + ), patch.object( + Linearizer, + "compute", + return_value={"self": (1.0, 0.0, -1.0), "cross": None}, + ): + with pytest.raises(ValueError, match="drop_last"): + model._before_training_step((x, y), 0)