From b23b8d67edd2072a6f0be2293517e2e4226e20ac Mon Sep 17 00:00:00 2001
From: DLUT-LYZ <88069893+DLUT-LYZ@users.noreply.github.com>
Date: Thu, 29 Aug 2024 03:51:49 +0000
Subject: [PATCH] add coda_lm_eval
---
coda_lm_eval/README.md | 163 ++++++++++
coda_lm_eval/convert2vqa.py | 115 +++++++
coda_lm_eval/evaluation/README.md | 50 ++++
coda_lm_eval/evaluation/convert2eval.py | 39 +++
.../evaluation/scene_few_shot/high.json | 5 +
.../evaluation/scene_few_shot/low.json | 5 +
coda_lm_eval/evaluation/stage1_eval_batch.py | 226 ++++++++++++++
coda_lm_eval/evaluation/stage2_eval_batch.py | 223 ++++++++++++++
coda_lm_eval/evaluation/stage3_eval_batch.py | 282 ++++++++++++++++++
.../evaluation/suggestion_few_shot/high.json | 5 +
.../evaluation/suggestion_few_shot/low.json | 5 +
...3\346\215\267\346\226\271\345\274\217.lnk" | Bin 0 -> 874 bytes
coda_lm_eval/images/overview.png | Bin 0 -> 241602 bytes
coda_lm_eval/images/visual.png | Bin 0 -> 404170 bytes
coda_lm_eval/labeling/README.md | 12 +
coda_lm_eval/labeling/codalm_labeling_tool.py | 192 ++++++++++++
16 files changed, 1322 insertions(+)
create mode 100644 coda_lm_eval/README.md
create mode 100644 coda_lm_eval/convert2vqa.py
create mode 100644 coda_lm_eval/evaluation/README.md
create mode 100644 coda_lm_eval/evaluation/convert2eval.py
create mode 100644 coda_lm_eval/evaluation/scene_few_shot/high.json
create mode 100644 coda_lm_eval/evaluation/scene_few_shot/low.json
create mode 100644 coda_lm_eval/evaluation/stage1_eval_batch.py
create mode 100644 coda_lm_eval/evaluation/stage2_eval_batch.py
create mode 100644 coda_lm_eval/evaluation/stage3_eval_batch.py
create mode 100644 coda_lm_eval/evaluation/suggestion_few_shot/high.json
create mode 100644 coda_lm_eval/evaluation/suggestion_few_shot/low.json
create mode 100644 "coda_lm_eval/images/CODA-LM - \345\277\253\346\215\267\346\226\271\345\274\217.lnk"
create mode 100644 coda_lm_eval/images/overview.png
create mode 100644 coda_lm_eval/images/visual.png
create mode 100644 coda_lm_eval/labeling/README.md
create mode 100644 coda_lm_eval/labeling/codalm_labeling_tool.py
diff --git a/coda_lm_eval/README.md b/coda_lm_eval/README.md
new file mode 100644
index 00000000000..9288d73413c
--- /dev/null
+++ b/coda_lm_eval/README.md
@@ -0,0 +1,163 @@
+# CODA-LM
+[](https://arxiv.org/abs/2404.10595) [](https://coda-dataset.github.io/coda-lm/)
+
+This repository contains the implementation of the paper:
+
+> Automated Evaluation of Large Vision-Language Models on Self-driving Corner Cases
+> [Kai Chen](https://kaichen1998.github.io)\*, [Yanze Li]()\*, [Wenhua Zhang]()\*, [Yanxin Liu](), [Pengxiang Li](https://scholar.google.com/citations?user=rUp_4RgAAAAJ&hl=en), [Ruiyuan Gao](https://gaoruiyuan.com/), [Lanqing Hong](https://scholar.google.com.sg/citations?user=2p7x6OUAAAAJ&hl=en)†, [Meng Tian](), [Xinhai Zhao](), [Zhenguo Li](https://scholar.google.com/citations?user=XboZC1AAAAAJ&hl=en&oi=ao), [Dit-Yan Yeung](https://sites.google.com/view/dyyeung), [Huchuan Lu](https://scholar.google.com/citations?user=D3nE0agAAAAJ&hl=en), [Xu Jia](https://stephenjia.github.io/)†
+> *Equal Contribution †Corresponding Author
+
+
+
+
+
+## Data Preparation
+
+The instructions for downloading CODA-LM are listed as follows:
+
+1. Download the image files following the CODA official instructions [here](https://coda-dataset.github.io/download.html#instructions)
+2. Download the CODA-LM annotation files and then decompress them in the same root directory.
+
+| Split | Size | Image Source | Download |
+| :---: | :--: | :-----------: | :--------: |
+| Train | 4884 | CODA2022 val | [HF Hub](https://huggingface.co/datasets/KaiChen1998/coda-lm) |
+| Val | 4384 | CODA2022 test | [HF Hub](https://huggingface.co/datasets/KaiChen1998/coda-lm) |
+| Test | 500 | CODA2022 test | [HF Hub](https://huggingface.co/datasets/KaiChen1998/coda-lm) |
+| Mini | 50 | CODA2022 test | [HF Hub](https://huggingface.co/datasets/KaiChen1998/coda-lm) |
+
+Note that:
+
+1. Images of **CODA-LM train** set come from **CODA2022 val** set, while images of **CODA-LM val and test** sets come from **CODA2022 test** set.
+2. **CODA-LM mini** set is a 50-image subset of **CODA-LM val** set for demonstration.
+
+After decompression, the data organization is listed as follows:
+
+```
+├── val -- CODA2022 val (we only use images)
+│ │── images
+│ │ │── *.jpg
+├── test -- CODA2022 test (we only use images)
+│ │── images
+│ │ │── *.jpg
+├── CODA-LM
+│ │── Train -- CODA-LM train (we use 4884 images from CODA2022 val)
+│ │ │── val_*.json
+│ │── Val -- CODA-LM val (we use 4384 images from CODA2022 test)
+│ │ │── test_*.json
+│ │── Test -- CODA-LM test (we use 500 images from CODA2022 test)
+│ │ │── test_*.json
+│ │── Mini -- CODA-LM mini (a 50-image subset of CODA-LM val)
+│ │ │── test_*.json
+```
+
+
+
+## Data Format
+
+The annotation files contains question-answering pairs for all three tasks as following,
+
+```
+{
+ "general_perception":{
+ "vehicles": [ -- list containing information on all vehicles
+ {
+ "description": , -- description about a single vehicle
+ "explanation": " -- explanation why it affects the ego car
+ },
+ "vulnerable_road_users": [...], -- list containing information on all VRUs
+ "traffic signs": [...], -- list containing information on all traffic signs
+ "traffic lights": [...], -- list containing information on all traffic lights
+ "traffic cones": [...], -- list containing information on all traffic cones
+ "barriers": [...], -- list containing information on all barriers
+ "other objects": [...], -- list containing information on all other objects
+ "description and explanation": -- summarization of information on all categories
+ },
+ "region_perception":{
+ "1": { -- region index
+ "description and explanation": , -- description of road users in the specific region with explanation on why it affects the ego car
+ "box": , -- xywh coordinates
+ "category_name": -- object category
+ },
+ "2": {...},
+ "3": {...}
+ },
+ "driving_suggestion": ,
+}
+```
+
+## Data Usage
+To help users better understand the structure of CODA-LM, we provide a python script to convert our annotations to basic VQA formats, as follows:
+
+1. Download the data and make sure the directory organization follows [Data Prepration](https://github.com/DLUT-LYZ/CODA-LM?tab=readme-ov-file#data-preparation).
+
+2. Run `convert2vqa.py` as follows:
+
+ ```bash
+ # English
+ python convert2vqa.py --coda_root $CODA_ROOT --codalm_ann_name CODA-LM
+
+ # Chinese
+ python convert2vqa.py --coda_root $CODA_ROOT --codalm_ann_name CODA-LM-chinese
+ ```
+
+3. After that, the resulting data organization will be like this:
+
+ ```
+ ├── val
+ │ │── images
+ │ │── images_w_bboxes -- Images with bboxes drawn for region perception
+ │ │ │── *.jpg
+ ├── test
+ │ │── images
+ │ │── images_w_bboxes -- Images with bboxes drawn for region perception
+ │ │ │── *.jpg
+ ├── CODA-LM
+ │ │── Train
+ │ │ │── vqa_anno
+ │ │ │ │── general_perception.jsonl -- VQA annotations for general perception
+ │ │ │ │── region_perception.jsonl -- VQA annotations for region perception
+ │ │ │ │── driving_suggestion.jsonl -- VQA annotations for driving suggestion
+ │ │── Val
+ │ │ │── vqa_anno
+ │ │── Test
+ │ │ │── vqa_anno
+ │ │── Mini
+ │ │ │── vqa_anno
+ ```
+
+4. The basic VQA format saves data sample simply with a dictionary containing `question_id`, `image`, `question`, and `answer`, as follows:
+
+ ```
+ {"question_id": 0, "image": val/images/0001.jpg, "question": , "answer": }
+ {"question_id": 1, "image": val/images/0002.jpg, "question": , "answer": }
+ {"question_id": 2, "image": val/images/0003.jpg, "question": , "answer": }
+ ...
+ ```
+
+5. Note that for regional perception, there are various possible manners to utilize the bbox annotations. Here we provide a simple implementation by drawing the bboxes with red rectangles on images, which are saved in the `images_w_bboxes` directory.
+
+ 
+
+
+## Data Annotation
+
+Check [CODA-LM Annotation Tool](./labeling) for more details.
+
+
+# Evaluation
+
+Check [CODA-LM Evaluation](./evaluation) for more details.
+
+
+
+## Citation
+
+```bibtex
+@article{li2024automated,
+ title={Automated Evaluation of Large Vision-Language Models on Self-driving Corner Cases},
+ author={Li, Yanze and Zhang, Wenhua and Chen, Kai and Liu, Yanxin and Li, Pengxiang and Gao, Ruiyuan and Hong, Lanqing and Tian, Meng and Zhao, Xinhai and Li, Zhenguo and others},
+ journal={arXiv preprint arXiv:2404.10595},
+ year={2024}
+}
+```
+
diff --git a/coda_lm_eval/convert2vqa.py b/coda_lm_eval/convert2vqa.py
new file mode 100644
index 00000000000..5e74fd8b1e6
--- /dev/null
+++ b/coda_lm_eval/convert2vqa.py
@@ -0,0 +1,115 @@
+from PIL import Image, ImageDraw
+import os
+import numpy as np
+import json
+import argparse
+from tqdm import tqdm
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--coda_root", type=str, default="./")
+ parser.add_argument("--codalm_ann_name", type=str, default="CODA-LM", choices=["CODA-LM", "CODA-LM-chinese"])
+ args = parser.parse_args()
+
+ # user notice for directory structure
+ print("=======================\nBefore started, please make sure your directory has been organized as in https://github.com/DLUT-LYZ/CODA-LM?tab=readme-ov-file#data-preparation\n=======================\n")
+
+ ########################
+ # Start pre-processing
+ ########################
+ ann_root = os.path.join(args.coda_root, args.codalm_ann_name)
+ for split in os.listdir(ann_root):
+ assert split in ['Train', 'Val', 'Test', 'Mini']
+ split_root = os.path.join(ann_root, split)
+ json_list = sorted([each for each in os.listdir(split_root) if each.endswith('.json')])
+
+ stage1_index = -1
+ stage2_index = -1
+ stage3_index = -1
+ stage1_all_data = []
+ stage2_all_data = []
+ stage3_all_data = []
+ for json_name in tqdm(json_list, desc=split):
+ with open(os.path.join(split_root, json_name), 'r', encoding='utf-8') as f:
+ json_data = json.load(f)
+ img_dir = json_name.split("_")[0]
+ img_name = json_name.split("_")[1][:-5] + '.jpg'
+
+ ########################
+ # Step 1: Prepare general perception data
+ ########################
+ stage1_index += 1
+ stage1_data = dict(
+ question_id=stage1_index,
+ image=os.path.join(img_dir, 'images', img_name),
+ question="There is an image of traffic captured from the perspective of the ego car. Focus on objects influencing the ego car's driving behavior: vehicles (cars, trucks, buses, etc.), vulnerable road users (pedestrians, cyclists, motorcyclists), traffic signs (no parking, warning, directional, etc.), traffic lights (red, green, yellow), traffic cones, barriers, miscellaneous(debris, dustbin, animals, etc.). You must not discuss any objects beyond the seven categories above. Please describe each object's appearance, position, direction, and explain why it affects the ego car's behavior."
+ )
+ if split != 'Test':
+ stage1_data['answer'] = json_data['general_perception']['description and explanation']
+ stage1_all_data.append(stage1_data)
+
+ ########################
+ # Step 2: Prepare driving suggestion data
+ ########################
+ stage2_index += 1
+ stage2_data = dict(
+ question_id=stage2_index,
+ image=os.path.join(img_dir, 'images', img_name),
+ question="There is an image of traffic captured from the perspective of the ego car. Focus on objects influencing the ego car's driving behavior: vehicles (cars, trucks, buses, etc.), vulnerable road users (pedestrians, cyclists, motorcyclists), traffic signs (no parking, warning, directional, etc.), traffic lights (red, green, yellow), traffic cones, barriers, miscellaneous(debris, dustbin, animals, etc.). You must not discuss any objects beyond the seven categories above. Please provide driving suggestions for the ego car based on the current scene."
+ )
+ if split != 'Test':
+ stage2_data['answer'] = json_data['driving_suggestion']
+ stage2_all_data.append(stage2_data)
+
+ ########################
+ # Step 3: Prepare region perception data
+ ########################
+ regional_data = json_data["region_perception"]
+ for key, value in regional_data.items():
+ # preprare image paths
+ output_root = os.path.join(args.coda_root, img_dir, 'images_w_boxes')
+ output_path = os.path.join(output_root, "{}_object_{}.jpg".format(json_name.split("_")[1][:-5], key))
+ os.makedirs(output_root, exist_ok=True)
+
+ # prepare images
+ if not os.path.exists(output_path):
+ img = Image.open(os.path.join(args.coda_root, img_dir, 'images', img_name))
+ draw = ImageDraw.Draw(img)
+ rect = [value['box'][0], value['box'][1], value['box'][0] + value['box'][2], value['box'][1] + value['box'][3]]
+ draw.rectangle(rect, outline="red", width=2)
+ img.save(output_path)
+
+ # prepare annotation
+ stage3_index += 1
+ stage3_data = dict(
+ question_id=stage3_index,
+ image=os.path.join(img_dir, 'images_w_boxes', "{}_object_{}.jpg".format(json_name.split("_")[1][:-5], key)),
+ question="Please describe the object inside the red rectangle in the image and explain why it affect ego car driving."
+ )
+ if split != 'Test':
+ stage3_data['answer'] = value['description and explanation']
+ stage3_all_data.append(stage3_data)
+
+ ########################
+ # Step 4: Save annotation
+ ########################
+ save_root = os.path.join(split_root, 'vqa_anno')
+ os.makedirs(save_root, exist_ok=True)
+
+ # save stage1
+ with open(os.path.join(save_root, 'general_perception.jsonl'), 'w') as file:
+ for entry in stage1_all_data:
+ json_str = json.dumps(entry)
+ file.write(json_str + '\n')
+
+ # save stage2
+ with open(os.path.join(save_root, 'driving_suggestion.jsonl'), 'w') as file:
+ for entry in stage2_all_data:
+ json_str = json.dumps(entry)
+ file.write(json_str + '\n')
+
+ # save stage3
+ with open(os.path.join(save_root, 'region_perception.jsonl'), 'w') as file:
+ for entry in stage3_all_data:
+ json_str = json.dumps(entry)
+ file.write(json_str + '\n')
diff --git a/coda_lm_eval/evaluation/README.md b/coda_lm_eval/evaluation/README.md
new file mode 100644
index 00000000000..6413ad3e266
--- /dev/null
+++ b/coda_lm_eval/evaluation/README.md
@@ -0,0 +1,50 @@
+# CODA-LM Evaluation
+
+## Data Preparation
+
+1. First of all, follow the instructions on [CODA-LM submission](https://coda-dataset.github.io/w-coda2024/track1/index.html#submission) to run inference with your LVLM and save your results as requested. Your results should be saved in three `jsonl` files for three tasks separately.
+2. If you are using a subset of CODA-LM validation set (e.g., CODA-LM Mini set) for evaluation, you need to copy the corresponding JSON files in a separate `$ROOT_TO_GT` directory like `./CODA_LM/Mini`. Directly setting `$ROOT_TO_GT=./CODA-LM/Val`, but not running inference on the whole validation set will result in a running error.
+3. Now the data organization will be like:
+
+```
+├── CODA_LM
+│ ├── Train
+│ ├── Val
+│ ├── Test
+│ ├── ROOT_TO_GT
+│ │ │── test_*.json
+├── ROOT_TO_RESULTS
+│ │── general_perception_answer.jsonl
+│ │── region_perception_answer.jsonl
+│ │── driving_suggestion_answer.jsonl
+```
+
+
+
+## Instructions
+
+0. Install additional dependencies for evaluation via pip.
+
+ ```bash
+ pip install OpenAI
+ ```
+
+1. Insert the ground truth `label name` of each corner case object collected in regional perception.
+
+ ```bash
+ # Results will be saved in $ROOT_TO_RESULTS/region_perception_answer_w_label.jsonl
+ python convert2eval.py --reference_path ./CODA-LM/$ROOT_TO_GT --prediction_path $ROOT_TO_RESULTS/region_perception_answer.jsonl
+ ```
+
+2. Run evaluation for each task separately. By default, we prompt `gpt-4o-2024-05-13` for evaluation.
+
+ ```bash
+ # General perception
+ python stage1_eval_batch.py --reference_path ./CODA-LM/$ROOT_TO_GT --prediction_path $ROOT_TO_RESULTS/general_perception_answer.jsonl --save_path eval/general_perception_answer --model_name gpt-4o-2024-05-13 --api_key $OPENAI_KEY
+
+ # Driving suggestion
+ python stage2_eval_batch.py --reference_path ./CODA-LM/$ROOT_TO_GT --prediction_path $ROOT_TO_RESULTS/driving_suggestion_answer.jsonl --save_path eval/driving_suggestion_answer --model_name gpt-4o-2024-05-13 --api_key $OPENAI_KEY
+
+ # Regional perception
+ python stage3_eval_batch.py --reference_path ./CODA-LM/$ROOT_TO_GT --prediction_path $ROOT_TO_RESULTS/region_perception_answer_w_label.jsonl --save_path eval/region_perception_answer_w_label --model_name gpt-4o-2024-05-13 --api_key $OPENAI_KEY
+ ```
diff --git a/coda_lm_eval/evaluation/convert2eval.py b/coda_lm_eval/evaluation/convert2eval.py
new file mode 100644
index 00000000000..c9f1c6b74f8
--- /dev/null
+++ b/coda_lm_eval/evaluation/convert2eval.py
@@ -0,0 +1,39 @@
+import os
+import json
+import argparse
+from tqdm import tqdm
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--reference_path", type=str, default="./CODA-LM/Test")
+ parser.add_argument("--prediction_path", type=str, default="./region_perception_answer.jsonl")
+ args = parser.parse_args()
+
+ ######################
+ # load ground truth
+ ######################
+ reference_data = {}
+ for each in tqdm(os.listdir(args.reference_path), desc='Loading GT'):
+ if not each.endswith('.json'):
+ continue
+ with open(os.path.join(args.reference_path, each), "r", encoding='utf-8') as f:
+ each_data = json.load(f)
+ reference_data[each[:-5]] = each_data
+
+ ######################
+ # load prediction
+ ######################
+ prediction_data = [json.loads(q) for q in open(os.path.expanduser(args.prediction_path), "r", encoding='utf-8')]
+ for each in tqdm(prediction_data, desc='Processing prediction'):
+ image_name = each['image'].split('/')[-1]
+ json_name = 'test_' + image_name.split('_')[0]
+ object_id = image_name.split('_')[-1][:-4]
+ assert object_id.isdigit()
+ each['label_name'] = reference_data[json_name]['region_perception'][object_id]['category_name']
+
+ ######################
+ # save converted prediction
+ ######################
+ with open(args.prediction_path[:-6] + '_w_label.jsonl', "w", encoding='utf-8') as file:
+ for each in prediction_data:
+ file.write(json.dumps(each) + "\n")
\ No newline at end of file
diff --git a/coda_lm_eval/evaluation/scene_few_shot/high.json b/coda_lm_eval/evaluation/scene_few_shot/high.json
new file mode 100644
index 00000000000..0d11a779375
--- /dev/null
+++ b/coda_lm_eval/evaluation/scene_few_shot/high.json
@@ -0,0 +1,5 @@
+{
+ "reference": "There is a black SUV in the opposite direction to the ego car in the left lane and a white van in front of the ego car in the ego car lane. A motorcyclist wearing yelow hat is visible on the right side of the ego car, moving through the intersection. Motorcyclists are vulnerable, potentially leading to safety-critical situations. A directional sign indicating lanes for straight travel, assisting the ego car in navigating its route. A green traffic light is visible straight ahead, allowing traffic to proceed through the intersection. On the side of the road, the orange barrier marks a risky area, alerting the ego car to potential hazards and the need for caution.",
+ "prediction": "The green traffic light ahead informs the ego car that it has the right of way to proceed. The presence of a pedestrian crossing indicates that drivers should be alert for pedestrians. A black sedan and a sliver car in front of th ego car directly influences the ego car's driving behavior, requiring the driver to adjust speed and maintain a safe following distance. To the right of the image, there is a person riding a scooter. This is a vulnerable road user who may behave unpredictably. There are barriers set up along the edge of the road on the right side, indicating some form of road work or a closed-off area. The ego car should stay clear of these barriers to avoid entering a potentially unsafe area.",
+ "response": "The prediction text correctly identifies the green traffic light, barriers, two vehicles and the person riding a scooter. Although the colors and types of vehicles are slightly different, silver and white colors are similar, both the reference text and the prediction text describe small vehicles (SUV, sedan, van, and car are all equivalent), so SUV is predicted to be sedan, and van is predicted to be car. While there is no explicit mention of crosswalk markings in the reference text, they usually appear at intersections, so this is not an illusion. However, it did not mention the directional sign indicating straight line travel and the relative position of the vehicle was incorrect. So, Rating: [[8]]."
+}
\ No newline at end of file
diff --git a/coda_lm_eval/evaluation/scene_few_shot/low.json b/coda_lm_eval/evaluation/scene_few_shot/low.json
new file mode 100644
index 00000000000..d34a75b47ab
--- /dev/null
+++ b/coda_lm_eval/evaluation/scene_few_shot/low.json
@@ -0,0 +1,5 @@
+{
+ "reference": "Numerous cars are parked on both sides of the road, resulting in a narrower space available for driving, which requires drivers to pay close attention. Ahead, there is a black truck traveling in the same lane in front of ego car, necessitating the maintenance of a safe driving distance. On the right, there are motorcyclists and cyclists moving in the same direction as the ego car, with a significant number of motorcycles and bicycles parked on the road. Traffic cones placed around indicate potentially hazardous areas. The driver's vehicle should recognize these to avoid obstacles, reducing the possibility of damage or accidents. A traffic light is visible ahead, but its color is unclear. Traffic lights control the flow of traffic, and the driver's vehicle must respond correctly to their signals to ensure safe operation. There is a no-entry sign ahead.",
+ "prediction": "In this scenario, several objects could influence the driver's behavior. Motorcycles parked along the road require the driver to be cautious and maintain a safe distance from them to prevent accidents. Ahead, there is a black truck, and the driver should keep a proper distance. Pedestrians are present on the sidewalk ahead, and the driver should pay attention to them and be prepared to stop or slow down as necessary to ensure driving safety. There are many trees and buildings on both sides, which should not distract the driver's attention. To avoid being late, the driver wants to pass through the traffic light before it turns red, intending to overtake the vehicles ahead.",
+ "response": "The prediction text successfully mentions the motorcycles parked by the road and the truck ahead. However, it completely overlooks the vehicles parked on both sides mentioned in the reference text, the people riding motorcycles and bicycles on the right side, and ignores the traffic cones and the no-entry sign. Additionally, the prediction text hallucinates the presence of pedestrians. As for the trees and buildings, they actually do not have an impact and do not need to be considered. The prediction text imagines the driver's psychological thoughts, which, although not present in the reference text, are acceptable if reasonable. So, Rating: [[2]]."
+}
\ No newline at end of file
diff --git a/coda_lm_eval/evaluation/stage1_eval_batch.py b/coda_lm_eval/evaluation/stage1_eval_batch.py
new file mode 100644
index 00000000000..2a9041b133c
--- /dev/null
+++ b/coda_lm_eval/evaluation/stage1_eval_batch.py
@@ -0,0 +1,226 @@
+from openai import OpenAI
+from concurrent.futures import ThreadPoolExecutor, wait
+from functools import partial
+from tqdm import tqdm
+import time
+import os
+import json
+import argparse
+
+class GPTBatcher:
+ """
+ Borrow from https://github.com/fengsxy/gpt_batch
+
+ Parameters:
+ api_key (str): API key for authenticating requests to the OpenAI API.
+ model_name (str, optional): Specifies the GPT model version. Default is 'gpt-3.5-turbo-0125'.
+ system_prompt (str, optional): Initial text or question to seed the model with. Default is empty.
+ temperature (float, optional): Sets the creativity of the responses. Default is 1.
+ num_workers (int, optional): Number of parallel workers for request handling. Default is 64.
+ timeout_duration (int, optional): Timeout for API responses in seconds. Default is 60.
+ retry_attempts (int, optional): How many times to retry a failed request. Default is 2.
+ """
+
+ def __init__(self,
+ api_key,
+ model_name="gpt-3.5-turbo",
+ system_prompt="",
+ temperature=0,
+ num_workers=32,
+ timeout_duration=60,
+ retry_attempts=2,
+ api_base_url=None):
+
+ self.client = OpenAI(api_key=api_key, base_url = api_base_url)
+ self.model_name = model_name
+ self.system_prompt = "You are an impartial judge tasked with evaluating the quality of predicted text provided by autonomous driving AI assistant. You will compare this prediction text to a reference text, focusing on the description of objects that influence the driving behavior of ego car, and the explanation of why these objects impact. Your evaluation criteria should include accuracy(checking if the predicted text correctly identifies objects mentioned the reference text), suppression hallucination(ensuring that objects not mentioned in the reference text are not erroneously included in the predicted text), correlation(sessing if the reasons for the objects' impact on the ego car's driving behavior are consistent between the reference and predicted text). Be as objective as possible. Do not allow the length of the predicted text to influence your evaluation. Maximize your text comprehension capabilities to freely match objects with high similarity, appropriately ignoring the relative positions and color attributes of the objects. After providing your short explanation, you must rate the response on a scale from 1 to 10 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[10]]\"."
+ self.temperature = temperature
+ self.num_workers = num_workers
+ self.timeout_duration = timeout_duration
+ self.retry_attempts = retry_attempts
+ self.miss_index =[]
+ if api_base_url:
+ self.client.base_url = api_base_url
+
+
+ def create_messages(self, message):
+ ret = []
+ # system prompt
+ ret.append({
+ "role": "system",
+ "content": self.system_prompt
+ })
+
+ # few shot example
+ few_shot = os.path.join(os.path.dirname(os.path.abspath(__file__)), "scene_few_shot")
+ with open(os.path.join(few_shot, "high.json")) as f:
+ high_data = json.load(f)
+ with open(os.path.join(few_shot, "low.json")) as f:
+ low_data = json.load(f)
+
+ template = "[The Start of Reference Text]\n{}\n[The End of Reference Text]\n\n[The Start of Prediction Text]\n{}\n[The End of Prediction Text]"
+
+ # high example
+ ret.append({
+ "role": "user",
+ "content": template.format(high_data["reference"], high_data["prediction"])
+ })
+ ret.append({
+ "role": "assistant",
+ "content": high_data["response"]
+ })
+
+ # low example
+ ret.append({
+ "role": "user",
+ "content": template.format(low_data["reference"], low_data["prediction"])
+ })
+ ret.append({
+ "role": "assistant",
+ "content": low_data["response"]
+ })
+
+ ret.append({
+ "role": "user",
+ "content": template.format(message["reference"], message["prediction"])
+ })
+ return ret
+
+ def get_attitude(self, ask_text):
+ index, ask_text = ask_text
+ try:
+ completion = self.client.chat.completions.create(
+ model=self.model_name,
+ messages=ask_text,
+ temperature=self.temperature,
+ )
+ return (index, completion.choices[0].message.content)
+ except Exception as e:
+ print(f"Error occurred: {e}")
+ self.miss_index.append(index)
+ return (index, None)
+
+ def process_attitude(self, message_list):
+ new_list = []
+ num_workers = self.num_workers
+ timeout_duration = self.timeout_duration
+ retry_attempts = self.retry_attempts
+
+ executor = ThreadPoolExecutor(max_workers=num_workers)
+ message_chunks = list(self.chunk_list(message_list, num_workers))
+ try:
+ for chunk in tqdm(message_chunks, desc="Processing messages"):
+ future_to_message = {executor.submit(self.get_attitude, message): message for message in chunk}
+ for _ in range(retry_attempts):
+ done, not_done = wait(future_to_message.keys(), timeout=timeout_duration)
+ for future in not_done:
+ future.cancel()
+ new_list.extend(future.result() for future in done if future.done())
+ if len(not_done) == 0:
+ break
+ future_to_message = {executor.submit(self.get_attitude, future_to_message[future]): future for future in not_done}
+ except Exception as e:
+ print(f"Error occurred: {e}")
+ finally:
+ executor.shutdown(wait=False)
+ return new_list
+
+ def complete_attitude_list(self,attitude_list, max_length):
+ completed_list = []
+ current_index = 0
+ for item in attitude_list:
+ index, value = item
+ # Fill in missing indices
+ while current_index < index:
+ completed_list.append((current_index, None))
+ current_index += 1
+ # Add the current element from the list
+ completed_list.append(item)
+ current_index = index + 1
+ while current_index < max_length:
+ print("Filling in missing index", current_index)
+ self.miss_index.append(current_index)
+ completed_list.append((current_index, None))
+ current_index += 1
+ return completed_list
+
+ def chunk_list(self, lst, n):
+ """Yield successive n-sized chunks from lst."""
+ for i in range(0, len(lst), n):
+ yield lst[i:i + n]
+
+ def handle_message_list(self,message_list):
+ indexed_list = [(index, data) for index, data in enumerate(message_list)]
+ max_length = len(indexed_list)
+ attitude_list = self.process_attitude(indexed_list)
+ attitude_list.sort(key=lambda x: x[0])
+ attitude_list = self.complete_attitude_list(attitude_list, max_length)
+ attitude_list = [x[1] for x in attitude_list]
+ return attitude_list
+
+ def get_miss_index(self):
+ return self.miss_index
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--reference_path", type=str, default="ann")
+ parser.add_argument("--prediction_path", type=str, default="prediction/xx.jsonl")
+ parser.add_argument("--save_path", type=str, default="eval_res/xx")
+ parser.add_argument("--num_workers", type=int, default=32)
+ parser.add_argument("--model_name", type=str, default="gpt-3.5-turbo")
+ parser.add_argument("--api_key", type=str, default=None)
+ parser.add_argument("--api_base_url", type=str, default=None)
+ parser.add_argument("--retry_attempts", type=int, default=16)
+ args = parser.parse_args()
+
+ os.makedirs(args.save_path, exist_ok=True)
+ json_list = sorted([each for each in os.listdir(args.reference_path) if each.endswith(".json")])
+ answers = [json.loads(q) for q in open(os.path.expanduser(args.prediction_path), "r", encoding='utf-8')]
+ answers.sort(key=lambda element: element['question_id'])
+ assert len(json_list) == len(answers)
+ batcher = GPTBatcher(
+ api_key=args.api_key,
+ model_name=args.model_name,
+ num_workers=args.num_workers,
+ retry_attempts=args.retry_attempts,
+ api_base_url=args.api_base_url)
+
+ all_score = []
+ rets = []
+ for idx, json_name in tqdm(enumerate(json_list)):
+ assert idx == int(answers[idx]['question_id'])
+ message= dict()
+ message["prediction"] = answers[idx]['answer']
+
+ with open(os.path.join(args.reference_path, json_name), "r", encoding='utf-8') as f:
+ data = json.load(f)
+ general_data = data["general_perception"]
+ message["reference"] = general_data['description and explanation']
+ ret = batcher.create_messages(message)
+ rets.append(ret)
+
+ results = batcher.handle_message_list(rets)
+ for idx, json_name in tqdm(enumerate(json_list)):
+ output = results[idx]
+ txt_name = json_name.replace(".json", ".txt")
+ if output == None:
+ continue
+ print(f"Missing {json_name} output")
+
+ try:
+ all_score.append(int(output.split("Rating: [[")[1].split("]]")[0]))
+ except:
+ try:
+ all_score.append(int(output.split("rating is: [[")[1].split("]]")[0]))
+ except:
+ try:
+ all_score.append(int(output.split("[[")[1].split("]]")[0]))
+ except:
+ print(f"Missing extract score from {txt_name}")
+ with open(os.path.join(args.save_path, txt_name), "w", encoding='utf-8') as f:
+ f.write(output)
+
+ # cal score
+ print(f"Stage1_score: {sum(all_score)/len(all_score)}")
+ with open(os.path.join(args.save_path, "all_score.txt"), "w", encoding='utf-8') as f:
+ f.write(f"Stage1_score: {sum(all_score)/len(all_score)}")
\ No newline at end of file
diff --git a/coda_lm_eval/evaluation/stage2_eval_batch.py b/coda_lm_eval/evaluation/stage2_eval_batch.py
new file mode 100644
index 00000000000..4bf443cf101
--- /dev/null
+++ b/coda_lm_eval/evaluation/stage2_eval_batch.py
@@ -0,0 +1,223 @@
+from openai import OpenAI
+from concurrent.futures import ThreadPoolExecutor, wait
+from functools import partial
+from tqdm import tqdm
+import time
+import os
+import json
+import argparse
+
+class GPTBatcher:
+ """
+ Borrow from https://github.com/fengsxy/gpt_batch
+
+ Parameters:
+ api_key (str): API key for authenticating requests to the OpenAI API.
+ model_name (str, optional): Specifies the GPT model version. Default is 'gpt-3.5-turbo-0125'.
+ system_prompt (str, optional): Initial text or question to seed the model with. Default is empty.
+ temperature (float, optional): Sets the creativity of the responses. Default is 1.
+ num_workers (int, optional): Number of parallel workers for request handling. Default is 64.
+ timeout_duration (int, optional): Timeout for API responses in seconds. Default is 60.
+ retry_attempts (int, optional): How many times to retry a failed request. Default is 2.
+ """
+
+ def __init__(self,
+ api_key,
+ model_name="gpt-3.5-turbo-0125",
+ system_prompt="",
+ temperature=0,
+ num_workers=32,
+ timeout_duration=60,
+ retry_attempts=2,
+ api_base_url=None):
+
+ self.client = OpenAI(api_key=api_key, base_url = api_base_url)
+ self.model_name = model_name
+ self.system_prompt = "You are an impartial judge tasked with evaluating the quality of predicted text provided by autonomous driving AI assistant. You will compare this prediction to a reference text, focusing on the ego car driving suggestion. Your evaluation should consider rationality, relevance, level of detail of the response. Predicted text should be specific and actionable, rather than vague or overly broad. Identify and correct any mistakes. Do not allow the length of the predicted text to influence your evaluation. Do not allow the length of the predicted text to influence your evaluation. Maximize your text comprehension capabilities to freely match objects with high similarity, appropriately ignoring the relative positions and color attributes of the objects. After providing your short explanation, you must rate the response on a scale from 1 to 10 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[10]]\"."
+ self.temperature = temperature
+ self.num_workers = num_workers
+ self.timeout_duration = timeout_duration
+ self.retry_attempts = retry_attempts
+ self.miss_index =[]
+ if api_base_url:
+ self.client.base_url = api_base_url
+
+
+ def create_messages(self, message):
+ ret = []
+ # system prompt
+ ret.append({
+ "role": "system",
+ "content": self.system_prompt
+ })
+
+ # few shot example
+ few_shot = os.path.join(os.path.dirname(os.path.abspath(__file__)), "suggestion_few_shot")
+ with open(os.path.join(few_shot, "high.json")) as f:
+ high_data = json.load(f)
+ with open(os.path.join(few_shot, "low.json")) as f:
+ low_data = json.load(f)
+
+ template = "[The Start of Reference Text]\n{}\n[The End of Reference Text]\n\n[The Start of Prediction Text]\n{}\n[The End of Prediction Text]"
+
+ # high example
+ ret.append({
+ "role": "user",
+ "content": template.format(high_data["reference"], high_data["prediction"])
+ })
+ ret.append({
+ "role": "assistant",
+ "content": high_data["response"]
+ })
+
+ # low example
+ ret.append({
+ "role": "user",
+ "content": template.format(low_data["reference"], low_data["prediction"])
+ })
+ ret.append({
+ "role": "assistant",
+ "content": low_data["response"]
+ })
+
+ ret.append({
+ "role": "user",
+ "content": template.format(message["reference"], message["prediction"])
+ })
+ return ret
+
+ def get_attitude(self, ask_text):
+ index, ask_text = ask_text
+ try:
+ completion = self.client.chat.completions.create(
+ model=self.model_name,
+ messages=ask_text,
+ temperature=self.temperature,
+ )
+ return (index, completion.choices[0].message.content)
+ except Exception as e:
+ print(f"Error occurred: {e}")
+ self.miss_index.append(index)
+ return (index, None)
+
+ def process_attitude(self, message_list):
+ new_list = []
+ num_workers = self.num_workers
+ timeout_duration = self.timeout_duration
+ retry_attempts = self.retry_attempts
+
+ executor = ThreadPoolExecutor(max_workers=num_workers)
+ message_chunks = list(self.chunk_list(message_list, num_workers))
+ try:
+ for chunk in tqdm(message_chunks, desc="Processing messages"):
+ future_to_message = {executor.submit(self.get_attitude, message): message for message in chunk}
+ for _ in range(retry_attempts):
+ done, not_done = wait(future_to_message.keys(), timeout=timeout_duration)
+ for future in not_done:
+ future.cancel()
+ new_list.extend(future.result() for future in done if future.done())
+ if len(not_done) == 0:
+ break
+ future_to_message = {executor.submit(self.get_attitude, future_to_message[future]): future for future in not_done}
+ except Exception as e:
+ print(f"Error occurred: {e}")
+ finally:
+ executor.shutdown(wait=False)
+ return new_list
+
+ def complete_attitude_list(self,attitude_list, max_length):
+ completed_list = []
+ current_index = 0
+ for item in attitude_list:
+ index, value = item
+ # Fill in missing indices
+ while current_index < index:
+ completed_list.append((current_index, None))
+ current_index += 1
+ # Add the current element from the list
+ completed_list.append(item)
+ current_index = index + 1
+ while current_index < max_length:
+ print("Filling in missing index", current_index)
+ self.miss_index.append(current_index)
+ completed_list.append((current_index, None))
+ current_index += 1
+ return completed_list
+
+ def chunk_list(self, lst, n):
+ """Yield successive n-sized chunks from lst."""
+ for i in range(0, len(lst), n):
+ yield lst[i:i + n]
+
+ def handle_message_list(self,message_list):
+ indexed_list = [(index, data) for index, data in enumerate(message_list)]
+ max_length = len(indexed_list)
+ attitude_list = self.process_attitude(indexed_list)
+ attitude_list.sort(key=lambda x: x[0])
+ attitude_list = self.complete_attitude_list(attitude_list, max_length)
+ attitude_list = [x[1] for x in attitude_list]
+ return attitude_list
+
+ def get_miss_index(self):
+ return self.miss_index
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--reference_path", type=str, default="ann")
+ parser.add_argument("--prediction_path", type=str, default="prediction/xx.jsonl")
+ parser.add_argument("--save_path", type=str, default="eval_res/xx")
+ parser.add_argument("--num_workers", type=int, default=32)
+ parser.add_argument("--model_name", type=str, default="gpt-3.5-turbo")
+ parser.add_argument("--api_key", type=str, default=None)
+ parser.add_argument("--api_base_url", type=str, default=None)
+ parser.add_argument("--retry_attempts", type=int, default=10)
+ args = parser.parse_args()
+
+ os.makedirs(args.save_path, exist_ok=True)
+ json_list = sorted([each for each in os.listdir(args.reference_path) if each.endswith(".json")])
+ answers = [json.loads(q) for q in open(os.path.expanduser(args.prediction_path), "r", encoding='utf-8')]
+ answers.sort(key=lambda element: element['question_id'])
+ assert len(json_list) == len(answers)
+ batcher = GPTBatcher(
+ api_key=args.api_key,
+ model_name=args.model_name,
+ num_workers=args.num_workers,
+ api_base_url=args.api_base_url)
+
+ all_score = []
+ rets = []
+ for idx, json_name in tqdm(enumerate(json_list)):
+ assert idx == int(answers[idx]['question_id'])
+ message= dict()
+ message["prediction"] = answers[idx]['answer']
+
+ with open(os.path.join(args.reference_path, json_name), "r", encoding='utf-8') as f:
+ data = json.load(f)
+ message["reference"] = data["driving_suggestion"]
+ ret = batcher.create_messages(message)
+ rets.append(ret)
+
+ results = batcher.handle_message_list(rets)
+ for idx, json_name in tqdm(enumerate(json_list)):
+ output = results[idx]
+ txt_name = json_name.replace(".json", ".txt")
+ if output == None:
+ continue
+ print(f"Missing {json_name} output")
+
+ try:
+ all_score.append(int(output.split("Rating: [[")[1].split("]]")[0]))
+ except:
+ try:
+ all_score.append(int(output.split("rating is: [[")[1].split("]]")[0]))
+ except:
+ try:
+ all_score.append(int(output.split("[[")[1].split("]]")[0]))
+ except:
+ print(f"Missing extract score from {txt_name}")
+ with open(os.path.join(args.save_path, txt_name), "w") as f:
+ f.write(output)
+
+ print(f"Stage2_score: {sum(all_score)/len(all_score)}")
+ with open(os.path.join(args.save_path, "all_score.txt"), "w", encoding='utf-8') as f:
+ f.write(f"Stage2_score: {sum(all_score)/len(all_score)}")
\ No newline at end of file
diff --git a/coda_lm_eval/evaluation/stage3_eval_batch.py b/coda_lm_eval/evaluation/stage3_eval_batch.py
new file mode 100644
index 00000000000..7cd1af87f97
--- /dev/null
+++ b/coda_lm_eval/evaluation/stage3_eval_batch.py
@@ -0,0 +1,282 @@
+from openai import OpenAI
+from concurrent.futures import ThreadPoolExecutor, wait
+from functools import partial
+from tqdm import tqdm
+import time
+import os
+import json
+import argparse
+from collections import defaultdict
+
+class RegionEval(object):
+
+ def __init__(self, reference_path, prediction_path):
+ self.system_prompt = "You are an impartial judge tasked with evaluating text similarity and relevance of the reference text and autonomous driving AI assistant's predicted text. Be as objective as possible. Do not allow the length of the predicted text to influence your evaluation. After providing your short explanation, you must rate on a scale from 1 to 10 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[10]]\"."
+ self.gt_data = defaultdict(list)
+ self.predict_data = defaultdict(list)
+ self.load_gt(reference_path)
+ self.load_predict(prediction_path)
+ # import pdb; pdb.set_trace()
+ for each in self.gt_data.keys():
+ assert len(self.gt_data[each]) == len(self.predict_data[each])
+
+ def load_predict(self, prediction_path):
+ answers = [json.loads(q.strip()) for q in open(os.path.expanduser(prediction_path), "r", encoding='utf-8')]
+ answers.sort(key=lambda element: element['question_id'])
+ for line_data in answers:
+ new_label_name = self.convert_label(line_data["label_name"])
+ self.predict_data[new_label_name].append({
+ "question": line_data["question"],
+ "answer": line_data["answer"]
+ })
+
+ def load_gt(self, reference_path):
+ gt_list = sorted([each for each in os.listdir(reference_path) if each.endswith(".json")])
+ for json_name in gt_list:
+ with open(os.path.join(reference_path, json_name), 'r', encoding='utf-8') as f:
+ json_data = json.load(f)
+ json_data = json_data["region_perception"]
+ for object_id, object_data in json_data.items():
+ new_label_name = self.convert_label(object_data['category_name'])
+
+ self.gt_data[new_label_name].append({
+ 'image_name': f"{json_name.split('.')[0]}_object_{object_id}.jpg",
+ 'box_type': 'xywh', # top xy
+ 'bbox': object_data['box'],
+ 'answer': object_data['description and explanation']
+ })
+
+ def convert_label(self, category_name):
+ label_dict = {
+ "vehicle": ["car", "truck", "tram", "tricycle","bus", "trailer", "construction_vehicle", "recreational_vehicle"],
+ "vru": ["pedestrian", "cyclist", "bicycle", "moped", "motorcycle", "stroller", "wheelchair", "cart"],
+ "traffic_sign": ["warning_sign", "traffic_sign"],
+ "traffic_light": ["traffic_light"],
+ "traffic_cone": ["traffic_cone"],
+ "barrier": ["barrier", "bollard"],
+ "miscellaneous": ["dog", "cat", "sentry_box", "traffic_box", "traffic_island", "debris", "suitcace", "dustbin", "concrete_block", "machinery", "chair", "phone_booth", "basket", "cardboard", "carton", "garbage", "garbage_bag", "plastic_bag", "stone", "tire", "misc"],
+ }
+ self.label_info = {
+ "vehicle": 0,
+ "vru": 1,
+ "traffic_sign": 2,
+ "traffic_light": 3,
+ "traffic_cone": 4,
+ "barrier": 5,
+ "miscellaneous": 6
+ }
+ for new_name, label_info in label_dict.items():
+ if category_name in label_info:
+ return new_name
+
+ def get_class(self):
+ return self.label_info
+
+ def create_messages(self, message):
+ ret = []
+ # system prompt
+ ret.append({
+ "role": "system",
+ "content": self.system_prompt
+ })
+
+ template = "[The Start of Reference Text]\n{}\n[The End of Reference Text]\n\n[The Start of Prediction Text]\n{}\n[The End of Prediction Text]"
+
+ ret.append({
+ "role": "user",
+ "content": template.format(message["reference"], message["prediction"])
+ })
+
+ return ret
+
+ def get_class_messages(self, label_name):
+ results = []
+ txt_names = []
+ gt, pred = self.gt_data[label_name], self.predict_data[label_name] # List[Dict]
+ for index in tqdm(range(len(gt))):
+ hypo = pred[index]
+ ref = gt[index]
+ message = dict()
+ message["prediction"] = hypo["answer"]
+ message["reference"] = ref["answer"]
+ results.append(self.create_messages(message))
+ txt_names.append(f"{ref['image_name'].split('.')[0]}.txt")
+
+ return results, txt_names
+
+class GPTBatcher:
+ """
+ Borrow from https://github.com/fengsxy/gpt_batch
+
+ Parameters:
+ api_key (str): API key for authenticating requests to the OpenAI API.
+ model_name (str, optional): Specifies the GPT model version. Default is 'gpt-3.5-turbo-0125'.
+ system_prompt (str, optional): Initial text or question to seed the model with. Default is empty.
+ temperature (float, optional): Sets the creativity of the responses. Default is 1.
+ num_workers (int, optional): Number of parallel workers for request handling. Default is 64.
+ timeout_duration (int, optional): Timeout for API responses in seconds. Default is 60.
+ retry_attempts (int, optional): How many times to retry a failed request. Default is 2.
+ """
+
+ def __init__(self,
+ api_key,
+ model_name="gpt-3.5-turbo-0125",
+ system_prompt="",
+ temperature=0,
+ num_workers=64,
+ timeout_duration=60,
+ retry_attempts=2,
+ api_base_url=None):
+
+ self.client = OpenAI(api_key=api_key, base_url = api_base_url)
+ self.model_name = model_name
+ self.system_prompt = system_prompt
+ self.temperature = temperature
+ self.num_workers = num_workers
+ self.timeout_duration = timeout_duration
+ self.retry_attempts = retry_attempts
+ self.miss_index =[]
+ if api_base_url:
+ self.client.base_url = api_base_url
+
+ def get_attitude(self, ask_text):
+ index, ask_text = ask_text
+ try:
+ completion = self.client.chat.completions.create(
+ model=self.model_name,
+ messages=ask_text,
+ temperature=self.temperature,
+ )
+ return (index, completion.choices[0].message.content)
+ except Exception as e:
+ print(f"Error occurred: {e}")
+ self.miss_index.append(index)
+ return (index, None)
+
+ def process_attitude(self, message_list):
+ new_list = []
+ num_workers = self.num_workers
+ timeout_duration = self.timeout_duration
+ retry_attempts = self.retry_attempts
+
+ executor = ThreadPoolExecutor(max_workers=num_workers)
+ message_chunks = list(self.chunk_list(message_list, num_workers))
+ try:
+ for chunk in tqdm(message_chunks, desc="Processing messages"):
+ future_to_message = {executor.submit(self.get_attitude, message): message for message in chunk}
+ for _ in range(retry_attempts):
+ done, not_done = wait(future_to_message.keys(), timeout=timeout_duration)
+ for future in not_done:
+ future.cancel()
+ new_list.extend(future.result() for future in done if future.done())
+ if len(not_done) == 0:
+ break
+ future_to_message = {executor.submit(self.get_attitude, future_to_message[future]): future for future in not_done}
+ except Exception as e:
+ print(f"Error occurred: {e}")
+ finally:
+ executor.shutdown(wait=False)
+ return new_list
+
+ def complete_attitude_list(self,attitude_list, max_length):
+ completed_list = []
+ current_index = 0
+ for item in attitude_list:
+ index, value = item
+ # Fill in missing indices
+ while current_index < index:
+ completed_list.append((current_index, None))
+ current_index += 1
+ # Add the current element from the list
+ completed_list.append(item)
+ current_index = index + 1
+ while current_index < max_length:
+ print("Filling in missing index", current_index)
+ self.miss_index.append(current_index)
+ completed_list.append((current_index, None))
+ current_index += 1
+ return completed_list
+
+ def chunk_list(self, lst, n):
+ """Yield successive n-sized chunks from lst."""
+ for i in range(0, len(lst), n):
+ yield lst[i:i + n]
+
+ def handle_message_list(self, message_list):
+ indexed_list = [(index, data) for index, data in enumerate(message_list)]
+ max_length = len(indexed_list)
+ attitude_list = self.process_attitude(indexed_list)
+ attitude_list.sort(key=lambda x: x[0])
+ attitude_list = self.complete_attitude_list(attitude_list, max_length)
+ attitude_list = [x[1] for x in attitude_list]
+ return attitude_list
+
+ def get_miss_index(self):
+ return self.miss_index
+
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--reference_path", type=str, default="ann")
+ parser.add_argument("--prediction_path", type=str, default="prediction/xx.jsonl")
+ parser.add_argument("--save_path", type=str, default="eval/xx")
+ parser.add_argument("--num_workers", type=int, default=1)
+ parser.add_argument("--model_name", type=str, default="gpt-3.5-turbo")
+ parser.add_argument("--api_key", type=str, default=None)
+ parser.add_argument("--api_base_url", type=str, default=None)
+ parser.add_argument("--retry_attempts", type=int, default=10)
+
+ args = parser.parse_args()
+ os.makedirs(args.save_path, exist_ok=True)
+ os.makedirs(os.path.join(args.save_path, "gpt_result"), exist_ok=True)
+ region_eval = RegionEval(args.reference_path, args.prediction_path)
+ batcher = GPTBatcher(
+ api_key=args.api_key,
+ model_name=args.model_name,
+ num_workers=args.num_workers,
+ retry_attempts=args.retry_attempts,
+ api_base_url=args.api_base_url)
+
+ all_score = []
+ label_info = region_eval.get_class()
+ for label_name in tqdm(label_info.keys()):
+ cls_score = []
+
+ rets, txt_names = region_eval.get_class_messages(label_name)
+ results = batcher.handle_message_list(rets)
+
+ for idx, txt_name in tqdm(enumerate(txt_names)):
+ output = results[idx]
+ if output == None:
+ continue
+ print(f"Missing {txt_name} output")
+
+ try:
+ cls_score.append(int(output.split("Rating: [[")[1].split("]]")[0]))
+ except:
+ try:
+ cls_score.append(int(output.split("rating is: [[")[1].split("]]")[0]))
+ except:
+ try:
+ cls_score.append(int(output.split("[[")[1].split("]]")[0]))
+ except:
+ print(f"Missing extract score from {txt_name}")
+
+ with open(os.path.join(args.save_path, "gpt_result", txt_name), "w") as f:
+ f.write(output)
+
+ with open(os.path.join(args.save_path, f"{label_name}.txt"), "w") as fp:
+ if len(cls_score) == 0:
+ fp.write(f"computing gpt score: 0.0\n")
+ else:
+ fp.write(f"computing gpt score: {sum(cls_score) / len(cls_score)}\n")
+ if len(cls_score) == 0:
+ print(f"Label: {label_name}, GPT-Score: 0.0")
+ else:
+ print(f"Label: {label_name}, GPT-Score: {sum(cls_score) / len(cls_score)}")
+
+ all_score += cls_score
+ print(f"Stage3_score: {sum(all_score)/len(all_score)}")
+ with open(os.path.join(args.save_path, "all_score.txt"), "w", encoding='utf-8') as f:
+ f.write(f"Stage3_score: {sum(all_score)/len(all_score)}")
\ No newline at end of file
diff --git a/coda_lm_eval/evaluation/suggestion_few_shot/high.json b/coda_lm_eval/evaluation/suggestion_few_shot/high.json
new file mode 100644
index 00000000000..1c0bac39ed0
--- /dev/null
+++ b/coda_lm_eval/evaluation/suggestion_few_shot/high.json
@@ -0,0 +1,5 @@
+{
+ "reference": "Maintain the current lane and speed, carefully monitor the motorcyclist to the right, and prepare to react if the motorcyclist changes lanes or stops suddenly. Stay alert for the vehicles in the left lane that may enter your lane or affect the flow of traffic.",
+ "prediction": "The ego car should maintain its current speed and continue straight, while remaining vigilant of the bicycle rider, ensuring a safe distance to prevent accidents. Additionally, the car must be ready to adapt to sudden changes in traffic, such as other vehicles switching lanes.",
+ "response": "The predictive text only mistakenly substitutes \"motorcyclist\" for \"bicycle rider\" . It correctly suggests maintaining speed and being watchful for lane changes and reflects an understanding of safe driving. So, Rating: [[8]]"
+}
\ No newline at end of file
diff --git a/coda_lm_eval/evaluation/suggestion_few_shot/low.json b/coda_lm_eval/evaluation/suggestion_few_shot/low.json
new file mode 100644
index 00000000000..6b1e379215c
--- /dev/null
+++ b/coda_lm_eval/evaluation/suggestion_few_shot/low.json
@@ -0,0 +1,5 @@
+{
+ "reference": "The ego car should maintain a safe distance from the white SUV ahead, and prepare to navigate around the roadwork or damaged road surface. It should also observe for any instructions from the traffic lights once they become visible.",
+ "prediction": "The ego car should maintain a safe distance from the other cars and the truck to avoid collisions. It should be cautious of pedestrians and cyclists, as they might be crossing the street or sharing the road and follow the speed limits and any posted road signs to ensure a safe driving experience for all road users.",
+ "response": "The prediction text deviates from the reference text by introducing irrelevant elements like pedestrians, cyclists, road signs, and a truck, while omitting key details like roadwork and traffilc light. It always describes general safe driving practices, lacking the specific analysis required for the given scenario. So, Rating: [[2]]"
+}
\ No newline at end of file
diff --git "a/coda_lm_eval/images/CODA-LM - \345\277\253\346\215\267\346\226\271\345\274\217.lnk" "b/coda_lm_eval/images/CODA-LM - \345\277\253\346\215\267\346\226\271\345\274\217.lnk"
new file mode 100644
index 0000000000000000000000000000000000000000..c8e11652fd6584c145e8ebf2c2c1968c7c8b1d7f
GIT binary patch
literal 874
zcmeZaU|?VrVFHp23XL!^zhSA`J2Y5Bx7Eyt2v^SagGfLEFG!t|WuLi&YF<5rUD%U~agFnz@M+RL69|m8bs2C7~90XDe3Iq@h@e2ckH;^p~#LPh4{T33CC*JP)
z#{dij_so)v(j*0!)Z*-t{DK&mV;Gni^nfq{qH93}kOqMih%(*u()?^w<2(kbz5<*0
zMfQFxw;X=b%0xOGQB_1ql2XhQPUK&NstFf0fKnr#>y5d79)?=5GB
zPWR{!2
zAcm($AUhF=LG~}pO4hKQdP#8WHHHKJHzh8(0>$Nk7;KUhkOnzA2Cvs*Fx&{z4+{EE6|JHqgN%TW009AkEGH|e1_1#Dgn;;j4hQq`Bv+On>f-|GswN{2
zQ9DU^_;K^uQcO_{0s@$b_+kS6aS!h#tK$j*fztoa4|3SC!u;bQnVXchn}o|Z6E|x|
z2hy+B_TM1bS=l-HSvmOGI7!*K_&M14*;qRO9_z-FU4v+}x=>(U)_tGFK8mpL#}^z-Ls8gbJ9N>rz7J+gFU}0V>W(
zz5hyr5&ie6*98~Rg0Uw$)n2>a7%DkWd0*B_Xugt8vbGk(W{EkYo7DZkZ~cf$`&R{h
z&mRt1@xRz^XXwXc{JR+ke)8@k@)d_`US_*bAEnY<^)B2iP-1BP|76T_h726n55`Ya
z95VAv4SgLTf4P!DvYXVZ8$|i`mM0ArZzQ&EskOzDBoGsPRAMK^RMd&ivJm8F6PH
zB5-yXZ^Ii|ikJ>^tEK0LN?>LWxVTo&{{#E)dokMosx{&s2ES(U9+ui3QLJ8pt=hU7
z+U4%vb~Ehv;;{_!Q}(EhQj3ozZ!td4kfp38m8$-Cd;UJ{^pRx#>R>-3n8uYC0Zl8l
z?VnLk)6@;V7}GV!|ssqMq)(OzSj&lA4N)56f!x
z*`WzVwtw*v@v_d#5BU!1l0?HAaxB!qzgv>r8J^CR>fhIwV9QQ3g`RQgwKYMZmn=Jj
z&Me!HmifJia|smH7HG5guOZNZG`w56Sj~c&@!9`JBO5p!PU&cCKGJ=8Gi_Mz>Pe+j
zXS*F^x!lz>gi@xa)r^
z{JHq3Pp$j)*d6t(`ctj}KS`{zxud#W=my^~7msRo$1P2DH1m0;+=AETb7@(HTB~
z?m+&y?oh3B#bGg(9bb@9J=leta*g%R=A?4MAD5e~2Dco{(5*jy;U0A#J{6)DcQwK5
z%6Vf+&Q)g62BUe(!Rh(3t0iN(`(UMWbc0{DG|7H^!+N-e|F5WhEE3xtA4s9Z$Y9$U
zFxoPi{p-kPx#BQT29;gdP+{0ic*7ky|JxmqkH-VUqm|?k^Uum~E!uY399Wn%esR#2
z_{PhziJ-1_7|^0m@FWkM-OTcDFS0U!UScE14y3T=z6&kGp3LA*G;cq;#Wx{T68{TC
zlu~PzUmF{3ol<oNIUZCQH?FXM+UuP3J&TCnGm=Pe)-zfADb0
z%Drsov3$A?5kCp>UL_yHBpah3n!PZi!7FjdoC_&oMU@JVEn*Aow`uKj7z>(kP&d?a
zJjLVQeMB-K9hHQU80+4h{bxg}{<|UJnp>p9eSecDGnSY(9p@Y`$ZiaxT6HnF9Or1h
zSZn8So(K7<{TqqYX#X1{GUWQQb>@JV95Vmc|Jq`#S>86hFb5syH
z)JlcOjhT!<_pk)RIruRj1Ke-fw#>j5jSM+l8H)aS=Nf@Vn(To_=}gbQl+Xj%v6-%#
zxH+5gzU|ocIw^-}33;|*G-XPIbM~$}n_~Z{FsP(-r_vm~a$5YPFV<3R?r-?`x^mJf
zNlu5~08EhtSrAGqtY50Xg%%VP^hZ^ieqAZVW#9C4qj!7JU)1y5c|Z!1ewk7Rceo2L
z+PoB^6F4qBgSyA3ssoU2ePwfhyd{+D-Svmmx&qI~y5O9J{lj@*p2zqZ}+l#@bMf0ztn*KBL*bVoNvcB#r;q(?Q@)5ib6ef#*-;l+?m#kv7N*F)DqBvwt
zo#c6h12p`sUqDc(ntS5oiNC_%(E@E8&-Sb;nXMKsKD3P@***VlA(X20o+|e`m{SoH
zI1J!K##UZtthavMRYUlIshZ?hg6dULjZEE!6{F!RZmpB%_CE>(*x}WX-!^B@o?_!Y
zv{wzI|JwY6iYu}z;NO}B8!e;^dQI;Fm>VDRstS#`YNd64rjuI}V|!kKeMqa}skRoKk%SrDS_OE1>6gCVoC
zwo`~TX(r5x#MxYqc5v)W5avPx;gK2dizN1<`Ul(7aXBxA9cDgB`brwEhBL`1wz=8C
zCJa6oed$a%nCWCIHn#^#lK)Z~q1Vka;j#pF1L`;9TBDb9Rl;iwv06;klZrLf@INxH
zi$PJ0X8sMf3G3WIx*SKu4aIUY=GfH=X#MF!~dSK~8Dz%>@K0b3Q
z*>i)`Nab~S`LjrhG|)_qO)#Bk&0U$5@mtY+B%RUsT~El*>b7P*G_Ds56inzg0?S*
z?tU5r7Ln=!NN*^rW7`ce^i$CHU@up{2(H6*zNH?o6v#t7r$a?X2oBKOh)j1
zKl~fn8j8OHF-C5Kn0!c3R*#BRzXk4dlp-#a{*5;c40kHEVzSS-uto>2sFuCLtr3vg
z)$!lTSXfhS#(Y$P1(D^f%BCodxyoiP2wETY%Q?7FdZ7dBd}9_K%(Kvw68qk3Dbybh
zOAxz&ncp9G!_PVuK;SyL5nTC*%ZABK&xkWjv*5wbY#^<%j
z*yiLobv&%|psyk4W@u`y{*J(Eg(Xn^q{FVjaEV
zgZ$OP7E;=gwFfF<2-?vp>+W+Wwn+ehc?1r;fh|sfR2T
z_8xN0F~F{LIQ2`2HzjHl++>S$Snu>?E5)B_QwM@wQ)J~avZB)-05$^|eQBXpSx`U0
zR3q*%IH8sv+|%1uC{i8t;)Eio6vxN|Zb-wqp>u4svyXy6DezRHNN+DQ_>vd8_7sEr
zofGzXbSuKwp)youe~l7Dj!snn{5qd7@}2kvLz!pnH(`cj(W(&)x^tyxmXN2VF}sd*p?&uoYuW=z_>_8UJ}2<;h8n7Aa|
z0q3gR!#nC2*R039U#3S}Z%%b!dD@NEP^=yu*Plega7|aJ-%-&yhuR3l>re<751b&~
zC`M72FV?iN>uxf;$
zcZSRtsR^R{h{Ik~N7iz{H1QpJZX*o6aioT^TFrGEE`5RpG9|i}HdW(Gfh&Pa+e_ao
zwsWIj?(13(dBvOzqh=-y4LPx2E*Z|M`ZA|f*Xdec7)MXj_G_bbof+cTMR0idnUDO9
zf?YjR;eg9+sBqn8a>jgJ5L3wBa4GlcG6g{8?aeYD{Wy5>cJ$C`{=P?QGGk>HqC~QO
zk-oAhUsmTKfZ%$UK5>W^dC~)5&)}hAspesl7vi4te)y1y0SG#WYWsd>$6GN|a`)X4
z_|k4taSGjA_2rqt;p@)P>2Zq>KAmY@_}l0GDHwdWO&uaCI`@wn;gWuSE=ih03Rl_D
zush9k+NeTZs{UsyNKRPU>SpC2q*w(#Qiprkk9_!99v?ry`cn=8$@OzZV)T;%BlocQ
zjplw0Pa-L!I|8>22aY!jdT6#6ZXlCxu>Fka$|}=Lvy9CA_;?!}WV^ZG837-z
ztJ!L=lMlh0{x6at6jb9ETeE3759V}PKh!no>h~wT+=C0=2rlH_Qsno3aAHUJBW!iW
zd^N@I`!?j(&2>m72if5S9Y;|3NKa!94tT_es@zoLdCWWUgo&*EX+Hq~^p&M@Gk}mz@_~Yas6P)jF%J%!;8QfVADLZOioLLGe>N4Lsl|2bU0*W
zrw--8CKRya+h?Bf>_iz`?>keiQx}!KuX%ZC(L?7_I2Lpo(+;c&4%WhLv@cC=<
zH8U7qI{HynFeOiMsMcGZHSiUN@!^aox0Vkfxe7@Iqg
z2e+YEoBUFWck5LHiHHk*Z4_m6{EsdQOAqWE3TeY8@^t=IaCZx|$w8!Jkz+iFg-UZ!
z;-_igu8Ui^l!YlGDwBQTOj2HHcc~yvV`H-T2Mh-&D$4PYsh5(>xEp_LXIfu)y3MLf
zF%1693RzJ2tZ6NQ3{aVN>1KEqtg67$lRI_EN98ONgCwTsYLPPA&5>YxX;ywEhbq;)fmTBzxtL6<+P6>yjoi*L`mW~t1LH@%
z7K3$r0}`Ero}ts0|0E@N$i<3qIr^BpNnTj1m6@gS30_HcOj=D4#Tu0PqjXJyQb2TW
zgs+h?xQvKNlg(BjQiS?7Dm7qgyB6Ojdm*B+yF0?9U!+IGUVxC!^J)@+b`cW-%O?b9{|zbU*9Q+}Thy3dFE&c_?|eo!Uv4
zR~wP1+zdKh*L?8>3ihG+g~5{K#k?+@O^N-(9!xxjfKUOAlB3>}5aZ8Pr5Pz66#&>$06&{v^Uzw
zsOS4L%|5AlO8w2QC;A3{rbs!4@S8Iue!Dki+RZ%<8DuWmrByt)&+AmjVaoNNnI2jZDt~@Y11Jy}w-v$^d-2w<=aRlLxm!`~oroovHl8E}*E;zbf
z7-4eH7G^(A9w?Jik@l?Ybe^nMBO5NQF4kwebz)f$7~!~<_~16-he5w$JtYsL;?Ex|gY!Ka6+9R{=aPIX~-@@~{9MckzV>D*WI-oxDl_vv-bYK#6h6l<^Vh9gZb8*C0E)K!4`B}kik
zN@`(bc)CM*i&WbO{>94$`iNRyllrh@@D<1M;NxCP^Vv&P$vtA?&K{fz1m?Zj{TV0e
z6?Ky%!)L~sbeUFX5);}*lUp{1K7Kc;$O>M&EDq>7EfOfL!c1=VKY(SFJ0}H>pe{(3?F=&)6Hn!TQcA`SokVXY
zElg^YX?}^UV&6DH`;gi|!4JSvaVM@$XlHCS~apg>7$6&KU&P>wg;r?Zk@&
zJ+#bu1Bj&;38D^h6Zl}u%~TdAOIxg79xrKlm%j-TI}(JIY$1t9vN5Py%x7pnEufim
z^QV6_EJD1m+i~x%;~&mS*3eG;5|)`?LPru424WFecJ`E)c9ww1JDFEDVlK(hkYvG5
z!@F|NV-TymUsvqg>RBH~%KUYaIs6iyM{98V_!6pG31^9N+NMapzGZlqd_NS873i1(
zOVsIhDqrsMS{#`@DT}A_g%;kF4#W>g9iT=)KQeF^r0QF66UUNt!+QogBbn4ZfzA?`
zX28AW)7`3zGh%A`A5|e8tDuHgOTArX+fo)7Vvr9FKtuSKPM&avXPTrlE
zU&uPppXdp7y$L`5`fgOZchq0bS&mHZy^oBxFHgP)@~?ulh9VeB%5tLxUkUV%
zmtj2wBWH!@p68>QE4&A7E0)cDS3u*DC2)m3b?dOLnPKJ5E>Jfax>kgQ`h+FXHOb!5
zq^%wTwV61~8fSI;wp$a_+~Pq=4Nb}^DV-`53;-tksVXO#lToIqhm)!hW1-p=N?eW}
z;7c(`MU`jlC#9m`Lcjw;(qk)JyEvi926e7#F;V2S>nYgvznay?0_UkRZ=8TGAzaUL=1oK9n1D7Orriv}B9p#dZ3C
z*7;1>Bk|ll48c=f2$p+3R&WYihz6z~#~wAI0DmHAo>54B<_{-5uQJv6<=(I4wv?-#
z9rIDew~l(E(`y~S2*0dj=SWQRbGL(Vet~^cv9+V6*v2qRw3ocP5eQ88))NULrzB@<
z3QXsUffeU_3E8By!DU8bKlL^sMi8sxa@MAh6!cX!neb`#+Nplu9YkgKV&}e`QmykGsh?D{
z0t%*HKb7Rv+l@r1BM$EXSFl(E_FSzp+`Z+aB|*ZV3}d@?-wWikq2W0iFfgZ~nUbrf
zzX~c)D)JZR49E1NKG|j1>n${I3_I?KV%@d61C}$5N3_NX34$DgfX24Bsh0OoAf{rJ
zL}8Wq2HE<;xiZ^W{Qk%-v)!*c-p*mOyf}MHR)1~VxHmeYP8!GFsl<5~C9<}V2|E}h
znAR^;-}PfZpg?HijW~srJ_cA?&2_t4;TKYY)5a&3&fX;?k#jDBQQ%EX>nF|$#9Ay*Fyq4MN^e_lv5$>|}G$23PG(EBsSOC0D5^!}34KnKGbQ6%v9ps&K8|m{8h2
zl9KTu`jl8Y5uGV_jo0XnS8XKDuK`{>fjyrlnyie
zeE$IUw`i773(vSA4rNraFkj&0t>#&*CwUd)v
z^_C%Cbfkz*C~iCPuHJeyq-Tp5R`+}THzPP{zbkAX4G*Dv5d8CTyi@Mq25*#|pBj6y
z4QmZ6^pcqu=(7si^?PAF%)k-^y;XuUg-!PO3F*xB@ujSaU5vOL@o{YBXdrZCcN>BI
z5VJ&2AxS+`Ip5W%u^qXCB@cv+y-e
zLk$ICfLQkZTOa0>P1g9wEGnwI!zZ~St@G2;pHZ}22{TH#Ve$!5*{_N36Gzxn}|Jlyv@BE!1bZ8;rTIYm+
zk;ve6l#o;#%LFJLH|!N2{z6Nbr?JJ}okV$lVS!!Qs_
zK`Fgoh()o7^_#9T_J>#BTc2-NmUj(Y>ei@oyrSKeElLg#sXEPD%B=^F56A<+BPrW1
z4N`ay>&Sx)?X51gq{x^h<7#d49q<;A0C4RbsZ?$O=i3=Eo;5#RI^VsX_sLJT=Dhy%
zbH$-w!H%Y1H$vPKc}3pI)?a-+#~)oFewtra+0PtbzTQm0b$(-gYN^eXxHtP4*it#{
z%z06qN`1jsGZg$C#^opL|~PoT1z*Lpd({rHPEQDGJjt577F0i%qLWr=LY+7V#aF
z`YCon8&K7?hwf!Y-9oA|t~yPh@K>Z*c&E?(SGkZI%pJU(!?
z5~wt%veSPtIYpD{EM^y7sk!^Ckl5o&yW)_d0hC|^b%6b!Y`%%#(6WT_
zaex7BrN}Y2h#3>|SzQ>sE(>pl;dt%XEPZXe1&4Z_d#nFkyGVQ`%j=q41
z*Lvb(%kJnMwH|B+c1+(qP+L?{>dkCL8)kZC#j!=hdPHahpcca@M4X$m$4Cc6y7HIS
z*OR9xSn9OcYUnPya|RslPV0G|Cbbfn^xO3-cp-s;GBX1hlIuNW!E5uT;~y73%y68tWyTX&9XWR)9a8d9g9(@
zn*?H%8)Q6vlfoiS)xXu%)X^zCk0^Nw0g)qfn*E#30Al}>fc1rjbuNP#
zs=<#SyF@1VmoRntaXN1`*-Wh7X>{dvfido57$8N5mn`dtmEP@4j6P*=fxJ$1Ve;rg
znE^I=N7jn3^v0oHU&LlPJB>W_wJ`C*rpo^P?27dJl#Y>mBg)cZayYWHidXk#*s<@&
zbL94lROP|l6XQ5;AR@giBtw#Z`0mA5qyt{;s+^ASwC=93MXj3q>?jSEyB?afGH?d)
zR^LO@I*>1cHAr8#gC#}-rsgBlAOSATutCkK(FN71(@=|r#x?((7`7hc)3HS*N0
zO+7YMebuDjUzQN90*7UmEm;!fHTqQRhM~MpW>gR6n&R~3hx`AyfR5qtmD5>WCVsV1
z-XkoNmSFGp#mmFQLiu)}{ZZ?Y_am}E(+(L(W=PjbL+0S+7yb=DDaK#$y^J#pbbF}w
zMO6z$qbefw4#aXeTopbu~ar1{_VQ*~Jgt(Dz1Xykj2wUc`xqUrQuWXG$m*
z@13g`JZAw0x7+X&o+`X=r_4yOIz(f&b_gNP4ZBNuf7oeBprtYMSyV=_sqE9LmfS%4
zEv&Hv@28Z27s3(({ej{PneC#f#hng=(4GqyxR#+E(_$=igUx~BEkm(LmHqP(*wrmi
zMF2f~ru8l-9D?@UZ{tSkl$E3FtuYOGUutd|tF%-XGv|}T=gV9r6WG6M+xKh~3TsMN
zP2DZm+9H;0YPwsoYexnG5X;3{J0&v2Lu=i(&d
z4Q8>uW#9CQY~2xcHIf@G4L-VV3VfXT>)Hq-(JyE0oOYi@BXh11+J>leI6q)dH*qm*
z@;6;kdq5)%FUbQ3iJgsM?6{&3DD|H0fB6<8v@HBg=Qi^m?tCv8LJ}7A4dyN;$RIIM
z1_k@mVuapSUOe6hIw?^wc((65Tepw97flAUC$vSL;W^{;qlaOHL#&_2iqB!YjR$YE
z*xC%-A7~0>a$D&6<8y;nIEJufdklfyXLSKroB-x($sK8&f=>xLj4_$ViamyP7p$%c
zj8osZkIbo5+b$rpqO}?tp<_FJs7^MDpZ<8S8rk!?5&8rse8P@?2c@Xq?=0-%uXF!|
zy1|*4j>I*tSciuapaO$=y8D?MuRG~i5i0Up&|b5fB&Tq&w;$4Kz0r#k=pz%*4C#V}hCuv?Ea?QG46IdE9sfqLhhj;q4-j8r$zHZJRe-2TQ80
zvYz+so8=Fk5CP#1Iq>~$FKpnYLxVBdJfB?KF8c@tg=OJZI(?dOudprZL?RF`;oAs(
zKL-ZB6Z(AB=#%j7v&~4uM=9FR;ZE*7`YxK6>~7wTXrZrTIQrO8YVGi2!2Li^?cvg|
zu+x+Zy`8{S3gOP@LJ9#>wkrg0t*k4~Q){T`ER@7_bYJ~KGt%Dcdc}G<3oSNJ0F}aF
zjQd31t!pFtbe_E9u~#s+F}%cipSvRceVTkE1OO}r7zUjn{6U;G*kyK`Kdf)`?jMwF
zH2q`#S<|JnkCqcka(uFZJU9*B?P(oS;pX6F7M?kijjM2`a|)aW`@5%pABS-h4|VO4
zD<;VQ5LINI%>*UDtA(N%Av1UZX4vF5g9E6b1Wrm72faTLtC#slL=rzBl#OzIv`%*t
ze|B#6CT5A>nfU$UBCe!C0i`m14u*DP^`2li8$_U*Y`x}Mo{u!cKkZbiQvGE66W4R4
zH6zuB_dRPLE<>rfZP!I;e3|3VvF*i+htPLMR{Nol0&Y+P5c!N?1rw_YX1?oK$j?iQ
zQy71qgyr1}BPuRz`Y~(SRBPvS26ZPkl#9*h$gRV-yfHBRN6$w`Z&Zba-!Mr28gBnu
zDmdU5=tpcko&i1wUU-@;p^6i`_Q2@p(t+h)c1o(Mi_MrykOKbL1K(lhPh&5)#s}YF
zmLz94e|b4flXhH{!*mK~#*^eTv;}Fv)dc@BJKyP&DK}{y|06Gt<-Clq4hyD6k~vj?
zy`1~O!*;KdK56w~2c0H<5!l4_S;+pt1Zk(j>s>*WQx>R>>ctB#h7nON0xn=MDwGv~
z>ZGI>DNQn&M9^~HLS=l_6^{=_eEHYvk_yxaU>6$LEh`YOK0H?h|1*B|7HN{Exx(+5
zA@gR{n6nN)1Y3OELrox$#%}0uq|DA~GugrW2$COv8}ocYGJtSpxf0MTF(MYGO6e&T
z$cIYJI89;^tB5`XT2i1));E^JYRAPam1xbWmEeawI-h7gOgSYC2(*Wb@b`jX+SHBW
zV{vb@i647m1OHK025mlf_Y$i^;G^Wk@ukQVJJ(--iyn|+8yxP%AdFVoRbvM~^x?2R
z1ds5loQnAg-K!)0%xDJ2z#khOhkClPfFeJ+T`{9PAJb*BnVAuIG$PM*VgI+CIGV<|
zKabf1uB?cYy#>Re1Pbl*&nrB;{(d5`t1n>*KeP)m$KaR~Ve3eUp*_vn4rtjZ#@wF?
zBY6_RRq$e`wHq@7K6u}JP-T{4psRzwvhG}TShD`zsRNFwA-;;d;*oG|1x1VKiv=^NpBzE)%O(?EU?BghtcEptBK9S?4C-|oH3`>CF
zx%2&z_vO_sYCb~1D9t<)l4w=3Y9fJjM>zt9f6<^dYPjoDoxLI0<~)}0v@X#5v*82c
z_4{fRc+ck`z||~uN5xc5Jb5MPdkmIJdZp%=^LU^IEAg8H%cqZgM#f{~bftoHlBD4z
zk;%}aD{fBR0_c(W1AmvBvyYN_yVJmDZMyv{_vra}#T!$6O&>jXQt_jMSBY*$>BuMj
z6N)O41vS`(Fx70wD;Mh$^QC}1yWx#-XKD`U!ujOF`CPmVGTYU(>)3UHUkR-Haq(dL
znmQRDreNDfVLH_>U!H=uB^b{5G5G%j4`s8Mz_=|)fQd5UuM4;ASkKYP4dciD!DyL9
zD5~F{&Y-x|Kk3iB^s$uW-C|05_|&p2a(UL+Sad9e#x#C>xw}?iyt1q5b@!~})vl%S
z5PSVRP&Z|EaWDn!UWLl!e^y^7@ELWS{6;w0
z$U^gBk>Ol==M--}ZQg;WZ|H+4N#k}k>Wl(|jlZ2$Jfd{kF@&09p5yZA?>9($YPIHX
z_t){PK>8H53z>D_nOLa_b6){4B3DcvddJ4wA1s~Sex|S>cVht_WGKDKA;qzbokwh!
zy^n@(eeS?-E_kciPQKcSNGecW#<1}Qkwf+ZDBYJROmAw
zi5!A>SteEh#*|b#-t(uMoaMF{VS~xLrHBWx0Pwz=XqSLfYJ+yc(6H|bUV`mnw;P@N
zhJo?Hg%`h~5l8L4_ju#?4*0w0<@weAG^MZjq90)(i*Q`Unb3?Z^=q9tx5t&Ll_(wW
zvR5yWX-d+H%Z{|vmrAA~sz-@S%lX(|?I?is!Ln(Wsd-S_xs`?J&L8F$IqKNi5$>mC
zc(zMx`F;o(l4XbiwxG3C7gc_V@nycSn(IUW#v5;AEuheDV9?A4LrI~56}zOtvMdYM
z`Z^CT=bbfWz^~+GwPl2fFAA!O?0tWRH~NNNKaigp_xEog&~8`E+-OgUoK7y8+b~J=pGAFSF;OH0+XM+gC-?jg^~fKb~gt+Rz#r_~^=Nv!Q4ahiRQT
zCh8Ht(;7imX#(pnvsvJ@5h5Cje8QlQ!=0MRHT@juj16d3@
zJU1fC0wG;$-aCN}9UE&V#ldemx~6_ix{~vD;YxXWmgx0z%WSvkaelj~5BHB~9KuuN
z#&SOFYF|8-Yd-8?!pnA^6RRa77{{_W${f3rM7{{S6W$LQgoA->ty(CcXwrAJ69ypr
z2^e0!%>HXbyUI&LUS<4kUkE`cMu)rGlxEn93}z)S5sEX%M3u9wRDeBuv%}V#Ue!tc
zK+VfJ05HXdppRgXK7J{n(i0JT9Ujm0@JiIDSrug{H_6J{>z~CuiH@M->1O)_eOFDZ
z!w`|hTpbv=hl$I(CvXB>5G&Vn6l+bmE#g*jLrX;F*fY*!ZOFi6M)Xu9zr?U((|#N#
z3)AWhdQ?XKt@+0$WMH`oiX`zB&h`r^d*zAYZs8zuk}1;cYMl;+Ou&?M=?^D@RydpknBBfWkDtF>NjmlLn`q??sd%V98=YJ&0n?M`L23R*cJk1ytHZkTIxbdG=&-+pDocjx
zZ|pM~L(g?(Btk6r&y(F9bUevG&nHaWmt~|A87QP2PS_M5!r|vS2dQT*Bk}X>;J$v|
z=q(X{7_%+?u>M7PTsAYrGUQs){S(;^C^Fp1+-n)=Zbh&Aw>LP
z)F0PJ6Hd6sPHDYTPl$5jq_WS3ycl*FKqh}4Q+u4;DSbp=HFIRuNZUS@;cl3p$3N-g
z3n-x@(+mNPH^AkT!L@RBpk%>a?q9n=%=*(Y8MYGxj{G-YX5#9!Qcdk#bz2?Nmm^(C
zq<8Gtt}f+V9_(CwAyk4Hm{!lKlzsrboaV=1mdvYy)^I}=W*|7^`Y6-cbxE*uHb|n0
zKa2s#uiWa}kMAMKbBWc1^98%!1HR5V?VquY++s$R4VN
zhd;=6g{SoXP@64<;4Rlyg%|qwWd(YYp*(Z-U9ebyuhHXsc^yG_br%N`@BKT4a6gFF
z!u{V%dunG&_6!P4cE)9<8Ex@WB|+9MSx~mIuLPLVNP|^@c#ZHnH0k}BZd3iTe(unQ
z)-_G=gGqGC+iOwnDm;6fHtQVtJX#Y%f)dYTha}ZsFDxSpFs+KS*Y8!LQA!9hx
z7yESP_A}a#iz-OIj#$iAMX6E(-Ho~twUGFap<3^pRI5!&3b9t!Sl{8O9
zCWhAZ(F0JFudn3%TYMzVH2Mr*?e~|A`!1ecup2&@Zx?ry4{s5PY{!jBR$kt8L&&d?
z?q%8?eh!PN!
zK0!zV8I#I_wyO8|Oq%WStn&7Y6|{4|FYqvIM;!h*97}>GmQCv~7Gy3S>#8p9lW1?I$iUg{-?>c2B0HU*6{^rl(Bh00=s)=c
zM0D?oL`eU#(q1(hVq>Ao?&K~R8O!ZuurSuq7(AjM!Qw@5`38YNUJK}X&Sm&A^K(6h
z$Eo@Uu2A|F$)e$Af|qUbRU?c@;5zO)ZhTzx!1;_h9j-@Qqg`_8(62{NFUE&7OiQ_W`ahbLm$vf
zB(-u0UFbKDbSj_=LpkM~P#g9ag3cIU^`p-xVIx0?#iQ08V8#$=QGe2Z)o3*$Hu}&|
zAUT*$2Q`qplYclIz$(L_2sO8^E|W9Fw@N|`JmXX31)-o-A#q0w2~+=1s@
zx`~PV+H+xXK~h};ip<>AP%SPV8E>8TkM_i4lMEA$)6f&iiOvrvDg!=Fgz9iz$h4v*
zn?Xyg0<8+00BWiE7}P&ob?Y%^encdOyP4CGr7Xv%*RvF1r=^Z5Ij`|sg3yU^*}>WB
z?*qPhc@x;Rv0l&>wr#>d6KwA*ZGlnfUO?KK^%c0i
z)IN$0g&mU}(jl>i3Md?b5&QvYsr?6^F;2Dm4~l7w3h2$8F9uM%2@B1%cI>)ML)~bG
zEa4d6hK&<<1`B)S#J5tMTP_v6FWhdGqCZ&eRBOr{Q7rzLOT=Z-wdj9Io*!^5lUw6T
zGK6%V@br|T-8e|{YUGR3`ZZZzQ;q#E4=nf8EXc3$wbZNd^;Q}+`q(cLmXBR{{0kd|
zmr(D__{`U7c#%3%I*0g+`tO2Ses2Sbta|KdcpB4%`huvjCW5ZW6OphdE88MK
z1LvdGua&cKu)Zk7km4P)%%x>|SeHJX@Z?5==w426y2KpmcB<2cEEHJXB8?3%lew^8
z0S7PjJD(@4@^w~n>+=R1GGI*}hV{@k8z@$hd|*=w>?38Doh<%`?G-2Jv-3evkq;2(
zIzIN_%!}jsZIc98!6LHVH}n>L9tl0q?niP2!=K3Z{8
z9jWF2Y*1EFqRmv8*gwulze?G1W@T*^GWuJ76#POP7p1YoH&o3+AeM*u
zN%|F;u!zBifi^FGMUwtFbE5rCNi0XBjf1p~JHI3U$*ba(Qf=iyZduOpWw94BL22yQ
zis9Pft+w|#>PXe*0OonM^u>0i_QNTZmFZesBJ5^(#v4L!VdXWOdsE3FIa>`uVlB8S
z^I>1_AhS6synbx|?mV)_2lMN%qsQ6!|BQAA5VyC79)mN&Aej&d4@M*K(;yAmhL(EU
zq-XlQF*z?7hj-RVTdekb0y?D_{PFNj(tR3F^7ffH4jHq|+_LM^?#xEc|1dJflZ-(h
zR%2uHyN}PZ7D{mFI+5A-N3SRwPa`Ur9KFEoF$JJ9wRS`%daMUc@@R<8b~j$k%!4aL
zt{gZ~o!yGkWd-A@EBIIMt)O3Py))TwwFsH!h=o?ahOiwUx3+{?95ea}o$?S?`k@jZ
zs1x-5%57-s?l1%;$99F~wVSA9_trpkD(Pfban{|Bs>bvR!jxD~{Zj2g$cy2mYTM6z
z9&=9qlx}7r=AVmu>|M?E9*eoCZkpOiM--NGL{OXumRN+NbO-afDGz_y5UEyk@oxVgy>=NGVjsfaL|ACH)e@;~WRXu@CSqhbocFEiL
zH&a?*@USPf*r*i
zsx)ZtBzOnM9efC46jA9nR_LznRsQN4H!ZBlYRd;r}i_OFpz)vY~Knb;mGCW=WpjsG0Y4WYnI+F4@!DCoW92`r2nLc^ycG>8`1Sj^JPP%_qS05
zx&6%4ve6Fr`bJHOIM}OzI9{Gp6<{D5xZim(Ex3zJ>zWXAAkTTX+
zR-VRAAS0kephQz}&)3B^COdOfdnl5Qgm_m>0aU
zt>UCUPTZOsf_s}cg`l^MQ$3zjb7+L@BO=os+F|^?;8mrK93q@agl>?rb#{*hsKy{D
zSqJC+)V;~=KTdlm5<1JbW{5a>A`E9=LS2!{4%=UaGN_!E-v_F}oT1Pgxd=ZU12Q+$
z`M6idFkEq?>xqyAzbP{Xm$5A6`V-b57=GQJ`{QEDB8pybL1r9_3u#e_z63U>b5YFz
zUuyEAN;2W?d73X~&$v1=P$U6J1kc=fSL#Di&=lh7#;uy<&14fMBNYAE@I}+9u@y30apAjj6O?n;fGs^tX@5c
z50_uK)@Q2V$=MYOC^2}FJD^hxUx-IeVGA9f!aK!0*GDiwcBO-`@$NG{
z@1GO0ZHhD+>Alm@IMDX&@d#J>Ve)tbIchTp)Z_{^7wUa|R3I2C3YVRpL^|BH8p(;5
zz>DhIId?3@_WR*MW1@vd5Dx4uk_r-Y;@^xK)lywRFRXCEx~f{lw=cG2Uok_P;y8a;
zWTrm0zc5FJk(N&*UvQACr1<`hCyDn+Do!Kvx8_)(rRVK2lJlznNb`Z#7X=SkUT;>k
z7U3-8;B&i@oJZlhZ^I6LmGFb>W{R_^^DNJEEb5F{b)e
z_%JfTd~7>Cal;>+%Yvl!Tc($XS9L4T(zmvG3Mo5yFbd?@anToA96rkxvJ9|umr9H=
zAXb$dUqoia{Wk`17lQGZaM8}U+fNFmxZgY&W6sbTOM9q|pu7JUVdvmnY1DQ3P6r*^
z=yYt{*hVMmSU0xav2Av2+qT`YZQYn1Ywr6^eKj>TQ#1d&9SoT>^Gx?H
z91gPa&Hl5{G@ba0xW&*P?pI#)YhHv8=`96%6{U0%%3P}KybnSdW19cjo(vHLZ(ikM
zx8GYMi*~~GB~_5Hq?nF?83*B7w*a5A7Z}Q@Q@>;)Tal(9$e-Pgq}}efltFNh0ssIJ
z#2;tvY)F|`&((1#dT5vuD2fdFh}MVu-6%X^x#wMJxHntE96T=lL%b0(SF_2fbw)$)
zHOeCQ1S}2`v_|uMJ3r)MjxcJG2X?w48beh!AN(8l0fq4%o^tb5`!SU82XYfeyMT9=
z7!CQo%KOY)-3Xm3H(%WuQ&Y8SI06yjaTBX29bDL>vu;~n3jzzefx$hnEylq4S(4_h%kLxvJo)EP_1xgv%o+@JrX{NMv^bOsVsAeAh6_;8(BwWJEN*1me+9|6gC;5XoR_)7Z9I?GzUeSw%Ul;
zP@mN>L_H2w6mdm-fK%GZDF|FFfY)4;&UUP{RALS0xXG+Zm)9Mi7%K-6v*vF7_HM|J
zz}wp>xyM!LlGRdSy%?&u$l0xcCa5)f6?`(Y`#Q#E8&aMYjcF66@eC6Z{RGufA>da7
z@(Jp&Jm}i;v=EyLXCT=i(LWhYh65OLcv8zUHw^cMt{xt+MJfr<{=tO8|Hgxx$vaLh
z$cne6-@5?~YQAj;x$g0;28ek6J#;zlyat&ojE5YuN^;yVe`q#-UC$r#v}zGs{!XOG
zx)PAk(-rh8;T@5M6|#5Nrl;?O!T3dgpe%U*dgUA*SMZLzu>(0>`X^s75>?N6>{O)Y
zewS9vzTKv|*Ha>`K2c`Q4eS=2J8{D09F9G5WkFEt;XS5?g9sb8$&|Zd-lS0xG^?aG
z)tb5_eGHYqM3vG--}^6suwDt*h`v<_*6TdVzTaUjmoni>>eOBnNa`0TU-Ear$a|K^0lb3_hicZK+6aYchi
zL(`0ep{A+$(|D1lmW`B3p`H6F=MBfEu7HKqKrVY)=)OgQeBI{?B+Cs!Vxy;C2Ou8v
zmjsY+;J1wO$ZKsquuc1w`l}iuvbe?GpmYCN7=m9&YSobp0wLGxs6@r$4-GFSAHncK
z=9x6543gcsd5nWS3Z&I80->s`IaE!2w*f3#A)JCckR*`@+TdnT;1dYhOcl
zQ&~WcQRZIW`10Xqb%faw5JV$}&Iv8Gx63IrFXfDe;lO)C6fY$U9v?g{6!$|i!mkgf
zlyVdd#z`SjhBCh#CJ^q$j}sl+QS3DE22{UB@Fq+cxhui8W@)h0WKykZZv+4KVq5>B
zBi#C7s5-|w=dkfx6Rj~EPio36Sdijl@n22En#fFur8YrH8o9xbfPiEO*cU@YiKl$H
z!}x^CW21*0$#kDLD_RgX*;{#^6yAhqCXEog7FX5jC&56lf>9xC(=M)J-cm2NNJt1PtdNoYpnRyFDK*E1o78mu_InF#Q
z$EMmMH*7$+>hdF7+8@(it%UA7Y*LhVX|7*ts<+gH(j`Dtm|0@x5+*neH#(P-KGiJH
znFOyx;RfsVPpc}BSP9%>3_lP@+ZAZ-9}%Zv9W=;noW7#}G$#1O%(M)y3FDiHV~Zp3Yv4Q(;lyTD=MS
zRlqC|Y)&-nxF1#E#h%;q=|hXkS^b@3JfhH7J*mHB5UAS;UmR!d
zcIL3KjO=SK+rR1WXs8uzq`T}YL9nzea5ylOjdHW0Z`Ukjg~W4fX^HYXja||I(iCJh%10L8#d!xJAUM{H
z)s|*&R$2%-noj-SRe=D7%mXTYRj$k06AfLFhq)d>gRO`C6*il%rr*n=M~bIM20O|J
z1aI`b*mf8@J9#j>uC8E>c5K|B^!xIpyl32R7@my&I*zt)SX=zZr9)-QB|JAL
zL_nb0m^X;VnKHKkGQ~Jzk%}Tl+!P1Ok{xReBfU!s@e<0)9CXTT#(20E@L5Ts{D>(
z!czr5S^1Vq5O_AdzS+X*M3e|{MK*V3kSU!4rF2stcPi9P`jfTExa0N{9fILzZr0cnBRxm_{lMM!Z#tkav*Q4A#18%j0C<%$8ZDxe@
za6AO3*_xzUhW89_W5eWtF=F%Bb;MtTTPt?F7CrSj+B2Ig=U^l50vPbr$j!(_2^q9&
z_4*CQ6$e-Mc9|Zu!t@lfze)!$LqZB$^vBdy}9u*%Zj$W3fsL|CLaaF7XBJ@8$mI
zW4+`d9ks@-JHS?I(qfxHH^J|o3OA}B?x^f*?=KIsJ(@0EOtMy@Q3ARZ68cC2{jwGeVIj-Ojcs_
zozm3&E;$wrzIGUp#hJ!A^VQ@8HU2xId9htkdb=ewy542L*9WgTOoyw}tM8WQ{xg0t
zQ<7T<9h}4snS+XFQYIG~ST1a)rpb6~0ZtrhyY6V#n4n;;l9LpVmriPUdb%SgLLXf<
zkyZ?KC~Gy`AMjIjaZxISZzrPkY%#;F-Ku%uc*r=B$IiPb=iVl}@7^H+-f6?``Pd`C
zb`Sln!vjHXn;8bd)!Too$jQ>W4`H7oH~=Pb>SkyVTn_MKeg&Qm1jOQS$WXQheTja2
zhHAj9jYkipu;=^sKG*}cyAOHpOj5A`X;l$7^ih5kZ0#{~!<{e{m0#8Lx1zcArUn4A
z7*QTKvlUSE^xtmw&E{*V_+?@B+^b}6`$JmyB^}0$r{>Jq>B52UlddanMJ(ta{?7PG1)WK>^TKI~iz=sl%e%euR`VmFk-W=pFEkK56zvHR#qMUq-VD7XiX>EnnRJ0xn4G}
zmy!i1$UfIE?x!?9*sSKTofw;YdJ2G;LTXZO>v=*B)^UlX`0MoR7Q0^K=SHLsuX0|+
zEYfC;Qtek+uG6M7f8Byh$t#H>C#9IepDX!~2HP6h6`D{`c(rzYd~X<6ZWoqg`5L3&
z>9rHiy)rrZe2E|q`X@P^OX7-_41*zkQHad!rqvvv4%~o@{iXjS2#6+%-T89b%ynVWZ7)|z7N6ukz3P
zj8HG=QM=(!oTvyLtY8?Zu6%saIwj^Z|wdONM;8I
z{!YO|2PYc^4Ap}36@f#ajl15@TEk_BOuk0sNMGIn7XH@)S;k4?4LErL2}T4h#&JqH
zgT8JCeG5d3w-c4S{C_H#>ZCOEol9Jj$#*>~?g^l3ra^5LhzhXw5?_JU%tfzI@t}l)dr=x6@opKc^7Z@AM^!C&@GrdCfa0N$)>RytWg1tY&xkki}_
zI4indFu{Z)l59X>S1pu#VfMhPtqs|Eq;sRbNu#gYdbQ#eU(D(HY9-XP#F;0(CrAI`
z18@4)O%VEE-QVMhfx46nD2KAr=l;j=ofv=wG5ry+O(_uJ*#WiFvRwS5deX)m_wp5{
zb%D)=rOF*i{u0-ZDccg!H`T%E5~KC(;|gv0LF3np7vYOaPn6)c`mU4Y!8loQdl}opqz2qXto8KX>f=sbf6l=*
zJSxDEZ#VO;U#(-7tAUT3iNuP#`3n=GFJ?k>=-$#d;$eQ-`>;s5KwFfJP!5Ct5231=
zo-t`f&SCflT;Fcz@-V1EB_LNdIN5L+yd
zC19K7@;dk$(B2g~)Dtrx>u!#}rpFV1neBLL*+9j3qDndFhq?RB?e3}15-A1)v*s=H
ztuvdvFFEYuqyu%Ie$ga0p#6h=R&4u=N6%Xz)+Obp2e#Bv^xoSl)%UUyjYuJHGAM#I
zgS#UBR^b*n9>MD3olwW*a&y!*2-7oPZ#E7)lDlg6XT2QQhE!(ix6HYsMfL$Dq4J-M-0a=RH2R8QE$~=T0CO_(1BQ6?P!r(FYc!oswqhuD)-GnW^F)xil6H4
zed)xIkz~i?lB2ZEFbY(8oolctY?8Ar$SM0VUf9K_M*!mfb&|$v5pSAo9TH58_c%qD-^~n_O!u%wed4Ls0$Ou
zlo#kR7@6Vd;033vOYh6^WaXg2^;%>=1m?*Ghor!y%G?xs{B6=eg{TlAnJTTW(9u+-
zKH^qiVHdR=2-hHU7!T9USM@ad}^ttsUABeF*DRy3)FwOT0>2~I5wwo3X
z&mr1)1OqHYjsNC8!p1r1tlRy!hjTL*v-Nb;GzE*fX7=jYl0T4P;bw4;-?{3=>yv6A
zG5yU>B3Mb^MJ6^(_2*F9#i#ys(NDIM8FbZ5Rp}#{&|06Z61Vq~dN<#{yvHht^K`8Q
zc&hDjacZT<=%A_Vfa{}39!~0RJm;e~Jkz)?_IAoYn9w{vr=48Me^U-dPa^zO4O~@d
zS7d^nZgpZ{5kS}fJHVs0wx$lqzso)Gvj+0+yafQ-AWChwBNEHBo?wtqAQ
zQD1Mo@kQc>#?Nt(vU|6<6-Fotk}wneSXn`a(P0NUsp;an*PCVPEQ2^-3xoint*E0J=kDc{F#Q-lf2!SqZd2(|xRjn0u02a&*95XsQNk
zMxnPHO8(03hOJPy5jx!z+`V$;!5)V=usy)H3~obRLFiGYcP234?1
zkKgXao2&FW_NhY1xJ}j04Or53?y7>|aDM(Dwqx<#>PMv#Fx`zs-~8eG}J6lzvzj%tL{Q0+9z?c)Z|0lQ|q
z73jFPMn%YPd1jd#o<|bplda)FM>fDXctAp)!3pnha_sf%?JWv+Zku}0ihzWfC=fH0
zGxXhgV=Y)#$Mfc^+HkhI?sViW45VgIXn}<}S3slY5OxW|pOn;xmZ#TmSy%a~43Vft
zW>BXY5k33shD*ls0}d5ryyNU)HCqGsa
zYCTy%Yak0><$^koJ=o6V7nbCnZPmJx-C><}d
z?)T%E3NNS}oVw;`O
zo@%EsTFzd%itUiCs_oG9t=d~kxQ8CmwA;XI#0pl6Ml`c;%Zukp=gjKpYH~jg@kZ6uH^GpV{u+{yiX}sDW
zjhIP|ScytTQ=U_{yuitk9p;K&n{=7PyiJ+o2HliaN_pmrS{OQKKl4`>|c~_kQWOC;g@hUh;N`%kF%(c>S`k
zu|f{h`wmo?dh~P$LIW(Q@MM3R?DdcQMr6^dHKPrwu^?z*M8&xxUNyuit|C7ppz+Na
zJMystbAIRK&_t#l#g)Qq!Kl*jv9ipMg2Z{0iVt50$*=|3S2j6IFLZkOSi#UnPDG9@Xl)+3byj6Vk}1OZ3W>L1E*CSH<-n7|WCLzES{Bj`Pl>hQ^EBVUXVeKykT`0q$6R&Id#w
zCbt#~jj=rd608}@YrH*XHs;&%Z6iK?5@M)hxcs6vl@XrC>N{BTnJrO{jpcVN#kGE~XR1E}GNd3I?4POv8%&{nvh9XTRU>blARmW`ThESrS>UyCSAhQg
zM{_3A6Z;O|Ea%}h1iaXtj%h4cbTrYb4Rd#sir|H}X!BJ!JkGEr4aTQs6~_<%sN76W
zkKbj+pxU)P-6HeH3&q@w~a9Ovi;i&BAehA
zmGX)qR=_Byw@Vg017ifakiq0R2SJx$_e7D93HD9hKJJ`^@{qDT<9=kb5>w&xNB_+c
zjAlIa=4Eww*YDe9x|CS2lR>px&J%gON#)*Gb?PZLP$zZq;)e;$euBGEhtnW1RPTvJ
z8|V@G-2<2U@5sJ!!Yoa8O;4_mCOj%ELQ^D@)Y?G7&(*|`raF!0!HJb;Dq$nJ8v*V?
z>=2M*WKmfY>+#W)$1q&4v8K1i#GxD^v+5@{PXeMNEV+jTG^4M}ibP2qo!Jq|OAYU>
zn3DquLD(`uuPUWM(i%q{YGKT@(YL=fBU9&Hk*TO?0v^JibTqcnia2vMfVk-)A$`ZL
z8fmN<>D`i@dt7#JO$OA&OXAjh4KL4TYdZH!%T5dg?pa_~2pWmGQPwy$S~M4Wl5>)R
zq&mi-b4*)CkdI{0HNZl~-NrL8t|UucChUY@Hiz!7LuiHH49c|n=nE!?_FBhR!O6_<
z6G6uQrO&xRERv~Um=?p{9iL;B6I>tO1*CZadXvM!BO7AkdX%0%{UmApEpJ_yI%C~1E$Fq9dx9$tPVTdJg10Ib5sUzF2f$~L2Zwk
z_OG>&m7sf(L0vRyOZCU>W9+Y
z6jy_g*&Q2c9FcYwg_DC&p%e1b9o3vNTCQ(P<+g%3qF?oQv=C`O9m_SC7RD8@#)Wz8
zhD*ZZttT9rJFIJMlkb?DU!if||K-(2M^wfuzdQW6)lbBYFOYXRP4%Nw8?Pl9-LVDB
zICvzPS~u5}tq^I(&Mv*eRIXI&Z2dRGYOBI}`ZtYe-aj
z5Py=S;pxUSpwb|@n7A0pt!aMpY2_M!sTx0aslIwek}Ecc_&l-C+J!C*NM2R?xUAxL25T~L*IYGn6P!x0W6#PuFx0AM>q3vGb}l0r7MNTh
z;=|qTm{^RciI+~Qbk=={XkljV%iWUV61uM%f0ECI!ttZ8Q8p7pU0MyPJw3vW{bd=;
zNV%`R=QSR^j*kPMMr1U;9K!!tCy#XO)bYC?JT2IEu1X}(8_q5qkI+{GShAU8qEkXK
zpaxa_1lZwB^XP&!n@ENq3^PslX#!bOD^<^#-cfZS8#{i{D*&IrHX{eqgbI8#oUsc9cHPRKeW==7GCPP4`-py$X0k8j-&UW=1h#jM7d
zIGX7X|7!y_^J1U4IOwKW1@OWkjm+!@I+#^NK&LAm%#(7QfM!eik39`|ARR>Ss$1`m
z7>^g)5ZayF(AY(Yif3F-pyUc@EfXDbf({uA{CBj0>br0w2|cwYUfCfbfveb4IC&Ts
z9cGbd_CV$t^9cXMkOaM~NT+$ALZ`0$D=Cu-NaXJ?gaK(ZnF7%PP4rDq8T4e48;@OA
zrg8^5TW}%xWrZrYAif)i1ruXNzfYG+(zNr$G;MJGo>1F$j_UP3&WXEtd|I8<%(`v#
zGjbOAd*5uO+YK{dDh*AQK^m70fsqIpz0Cofbsr=uJ>=b_sU|>A<_}*7ZH2jG*!H53
zh3-~)1)Vp4ZiWg>+XxS+n;9HEX~8$zl3Xwmjm2vQCK(Y1wzn}9D5Hxct7o^%@p>b;
zss|oG#X|DVT}p3~&00@8e9nx#NQj}v!NSq5A
z)@v+{oM7iTMQiJ-aAU`KbH;PXE1+eJjO?jb>&|0!qk>x`l0++N!}vxT`@?`($ppj(
zZ4P@Tha6|Kv<1rrs-?(Hh@OLwnZL4{lm6=(3+`z5@%(F~`H5_M_DRVxsA;f#te7WmhlnSDIPh)L-}F$M^<03#P+7ev
zToT7zalnmmowKqpy^6{~BU7e*RMUL@(A4Q<1``bcS9;$+e(YLw(EZci)!_%jgl6Qa
zc;7QkH*-#6j76b+{^oAnE4q`?aoCK#k`T4xl#^t5Ex8?6B0grxfPo6$TduH)`7L*>j4cr>Y(-*l2Ye%k6
zbfALHA1bgu@lVh4{e5+gBUTJje83zzzih1U3St#XhejM^6yBWilb;cFJJKMnDn06e
zD4Rr+-;Hhb`|%ijDb0Fvy<-S<%)G9eo4NY`X!X$YKAw5!#L?9nW2df{@C-|V@0^Eb8gGWa
zYoa5@R9SUiMFe)9vjTT%l;l{XsvbWl_|&ap$CqYv%
z4sqP?UtH%J0fFbF2lF=Nj{VGa0B~%W$a#@CacvEaeT1kD|
zIAdL1c*#>05D*bS82ewQR(aLLdI3j4%P7zcL`3S{H6_sWyM$Mz%mvKD!F0jJ^Mmeg
zZ3IfJF&}UAFoMijp_cq%(;-^5JUi@f*Y-;>HuEg?`4gL-=#94pwg_IgM$a9Ekrc8@6==zBa(o
zs^{()(y%?^jZ9STjho7&_dByHO=t~rd6*z6bv45WL|>rpYzsy4R&%?h%uNCbF`|_z
zN3IYSEFC=C;LQm&uT6Y6D8S_)Avr`R9BzzoZGrx9IB(6#37I$Mce{6gGuIG3qO@A_
z>T2eo=Y#&2Kt?`!uRNa9{EK)}Q~AidiDU|S9XWLvhJn-kdOeE5Vyx*$>Qe9k=2IMJ
zk{=4Pmg
zc6(yYzv&LA#G1xF;2>Ypn(C*8J7aP9I16qpOVrhzlrn2h{G>%HuSw*?!p6s$GRVhe_(|DL4*rv6^6<>*D(5Ewfy?u%
zcpB^6Z|?(M=P{d67Q_>Me|b=L&$x5+^;e_ayw6G5+@5U^Epf})d^LGY__BRHNf?{F
z;Preyzh9PD^_>*e=pTp0)=^k~$?jH;Jf@A^PpGqTSp3C1PU7`>9r)C=3pL?=d0LDD
z-rjD-xbu0!2|@OrDNrraer!1G{j`51yK!@WCNQVkG9ITj?%)rk(d!PdO?C$6tRcHL
zGd^WXt!>Arf2-hWc2a74lN468v2cLZm@)%NNM_{MaMhUt>jdAKk%7o&5fLouuqInp
z8a8-`4hH(ybThfiUSw-uFZFzIzVAAb=*LeG1f$f;^C>-M)snnOh_p6gxE#QM9HB#3
zPkr|ocduk{661CI8Tni{a4d=freLTBz@Tvl1^Q&*-xTuI8
z`cOQe@HEK=SA@e>GdF$KwAXak^ciWy)@AfY+7LRE|3x_!!C^w%C`r`cX*g_||1(Dg
z`>YCBeLm2Cc%TPV!KypK^l8OLF|ONJHBrEqg9MQ!slC~rI`b{QH$FQW*_i|>Y$uwp7Up{GK@iUhF9moC7emKD8BhOi_iz!
z5h5@z%w42e#%jQ4EAQq(KMY-4d{v6Kce@P>NGbGF%`3n_KgO+i;?|~+6vh8!5+GoQ
zWg
zn!f5DFV`@=Y&w7AS)5S&G~xBP3*876MR|*fwRJY0v;p|;UgSOQZuS@&pp=KK56cd?
zGT~GPrP^mwm)+~T)3QQpt=JY0kr=0Wn^-ss2O5d*79JwH6kY|>3>5MqU$fGtshjE}
z>@zD<+w&n}62#ICsg&3D5!(Jwp9n$EifnGkxvKw20@Je3aNZJLJ-n}2x@Sm?Z1~*#
z3s&)+IzDHesJ#KT99l+_N(PoEItG?-g^!Cbi9P`|p~QUdz?k2L(MqtpsC43|D1}fhKwr{Q3zEvLnH_PT}72nO)Ox
zaF6KJAzdB9c1ir}#R-L{>mReGafKJ_P?os0<&QD-7$?)mdiow^2i)#dUOcj_@hjB3
zGH)(4ZQ}dQl}h>eZ{fUvyLTyIyCcEJ+Z{Ec>e!Fx1w8VId7x2c&=LR3UUj$OnKvuG
z`0V8xDk})$pUyhYc=1DEdPxIecEAVGcHqg)NvdKmrCiWM%JMHM9h^$J+!U3>o-0NY
z8OaQ9+E#h4WK58P`(Nyu2~X9I5zE0Im=EWfpL!@ff?2njboRdt{4$kg-1sbf7wcwB
z!H;yfS155DE0K8sSmaSr@AI#jUB%@B$^j<+W6Wlf%w2UPakR+r^0#zAN?^Mh+qT!%{?t}P_
zsOFsvPjWU$*Z@g)^zK)d;~QJhGTk>NCBS4D?61v+&95gAJ|+5^CptiY8elw_%@d}R
zv7|o~(bTH&={}+H#Np9uk>_=p-=7G$VaxY3F$#!3)m2kisF!3UP+1d$3Pz8n0&ULN
zmn-UpKMH*d4HS&Ul8NT#Y-og5Qjz5FH2ZCbUQ3Iaf8MK;>h9tcg^SuT97mZs;;zG;
zIL>+_48e;WDEW@kVTT)2=GfDFZM_+Z)`GtO(i6jKLp-=Ik0|+S)xIUY5q>!6Lx^A(
z_JZcH!gmPZ`zMO}{kt!3tqGiu&J%$eQAFTkwLfD$WBYv@f%of!XioF&Q|*;k
zqOyk?teNrWqnoRgD&LEBAGl+2R$LLl6c1jelL2f3kV;EMlJf)ip*p6q&5Y<0HiNQG
zUhVrm)}9d+R9yq(y^)rnxN~T@q_2Td7n#*@=|`~ND+EG@upZDN_8S;E(1U&ZFuOH=H%u9Mpi^(-GgC
zAD>lK%8*l&>j?JS{}Qqewke{YMhJ6$yw8yvhoAH=yB&?tJ{@PRdutD_$?bMXZV3%L
z>h1ZMX=tkbD)}7dWD<V%i4gI)~f_bcDtbe$!RYV?NN9ffuz4k+Ckt$)e&R
z@PJR2m~%r9bxWcc*g}?3Bq}o1G`ZwPvb3PJQeOtYK2zsj;77rSthft99yUElQKTK*
z#|uNL&0d%csqAny8N2^Dy8HA3Vye~^R3pK9?eRr@|DGOw$B^CI$W6SPttn1)NC6#s
z))CV;YA1TVHz5xwHD!6I(u~sc(FqFf`S8~Ph*-mexCDu|x
zBV;nUs@ZPIY#CJ>;rNz#kBELA4vSs0IHxW;bgkb42j2XsC?vP$iPdo@)^?%dO%|=F
z+o~aJUsSEIOt8NdV<}%~^g5ig}Z
zsy1k|Fj$g5(C=;h2u6?ifrHt6v}BRy?_3pF&N<*c?H(2+%LOc^yyT1xXZZ0xo9b9D}WeNUZCR&R$XBWV3B)cIJB)8;}W31HEZjO}dcr?iBvRBAJ4U;#Qf0mz)kCtMV|4jN}1CU
zH&O&)K7Tj|Ole=-Q?PVdDoqWqqaB~Yd41e9+bisPIId0?b`GK#;e4*1<8L}`@s(lg
z%|tJf-F@RWceabio}^2iM8r`~*iC}Zgzs4<(#!f=0dP#VVpk|5`37j>=g&d?-1ZZO
zZm)bRbCmjc&q#1P!E{4b7XBmg5O5q20E9pxfE5EYRYf~F648%jS`Rv><`Y89;`E>-
z)Hio^#IlPGrOpLkHXkB<{ToH$T6d%G&bT9aS&aMId@!mp
zT3(pF8nun{9-)_#IUG$aJAe_aQ0HW6h%pyL4Pq
zruDTs&2_)?J^wa!%7_1R&AhS(#aD2-n#nX0uhr~cW2vZ}zBzJit-9%B?kBX
zjP@)Ge`wqB+h4T`)S6kYxbsxyGp6Azt^Bb9SvRpduDdO2Lsg)y7d-`D2l~E2%cV#{
z@4HAX_aH${$w3qkHPxUsp3xRjHr~W8z(gpZd3k+2%Kf4~GugjHIXorN9jvuxUTP;(
zQHzL)*@za&JsIPI!!-l_U+ozSaXVvc-6;;_hMou
zqrRq~LyWg^82D>bN;)RpBqn|j;#bJzh9|>U!dKDbx3f?Niu|$*K&o1pGqejjl22P^
z63C&Gi7c(HpzF|YH~uLLSZTR~jVC+JbASKM90u^-jv5K_%kp=e%m(}Of%H@L;4FfbKO&lGQ-^a?d%xwik_1daDK
zV(#5j-Z%#9rBm_pGLG!W-qrndoC+-`!i?e_R~Jk5y@`g|qBm>+*E(;J30-CzN!8CR
z*m>^c@;s1lpC;1Bi(Dy=Chs(*dgYiw4x~s=Gx^vn0=zeL(=%QTgCyLIhBf=u1bxDE
z#O$$%WW$5oc!Y<3dyi6p{hmPi(W-wy{ZvwjK+n9%eP7yT@SUwVo
zRf}I#Z!IJ)14-55u9*bzIKJ#B59N6!aF}m43RtQmL&onrTwV)4S=l*9>3uln<@W@2
z6xQOQn=s>0RdbrD15J0B1jw}fU+eu2Qwb;^T7(g6*)r)pZ+2_QXuj|OSVF_@Qie%1
zxYC@`z2H=P%P9uS|J<39$WSGTEG@cw1}b+U?4vUxZ<({t*0bUA@*`j;EAywbF<#pe-Llu=TFp>&&3|scpRBuL
zx7bi-))J+Rv(NPQ!`f2Xe>wAP&2>}|U?-eA)l&2u*3PAiL>!o|RDe$>WTN+Dk2`?;
zvFGu|%W`z1B-9@Lf5@l1UI#w7VS9Z02r2x6O*2~K5z6X(!PsPsyZ7&`fWojkdM50;
z@D;ubQF@f`Q8?c?+M9)y_jKR>^|^JvFR~DZHSk~uE*&crpLDtH7NFc%Kc3%}!6Um$
z8H|OghW-;F@mW%y(Z6+6G6tWH$pEFTlM5~$y)050`SU<&Pk{0q$H&iY
zt}Y&5#V?xR?$vwljJe^IOFK)C@76i*po?}!+;ioX4mpcB>$x6ZH4aVN^4;IBec5vd
zW__!0_(^Sx^OZwEy|u_qdk9CoGGfzq;O_xZt>zM$0=St8&tzBcHE?uP0TW}7@^-^!
z^mx@1&7{L{r@5;eXd^ySsA_Q3^%WwDX87c^uiQk}ut9X{`pXv~FYQy3Ly&aDcry&)
zlr_4mRa`z4*;m|fs;2aVD<0w%$=AAQR=k02u@6I3{&xB&CEln*p!lg>6=|+=)z-PX
zIl?Z2v(L@Q&vwTT8!E(ur-*r9R%D3cT;x#E1Qe(topBrU(>{u<@Ow3ux`I
zR-g02Z~#A;52g1{EKPqZwJT)NMD&c8o#+xwjDll|)Ut_#?mDT~B_-70fMyIqu*Oy(
zMf${XJ0qAN70da(p9!hN=u}^F^x-YO?$~7-#HHQMB%pf@nQfg_lu&l~Mc`r+=aqQH
z&+dgzO>_@!ukK^g=lFg~apT`9V#<>LB>V*vYi)M5v;w3Q;$^a!ixgJ9wPDGWuxm=QP=Rw6kY6)Y8VfA`Bw$J{EI_C<6tQSj6;RlV<~=@{lyJ_EzXCG1dvL{o!Dn~E!^U6gZwoK!7%Su5Y2c<)!mZY-U~
z*yr51k(o5vuAQpnT`e;kJX}ZdMrwW(h`z*8an!>K1zmT!awq>~E3=k3BjK_Ch@KAl
z7C|VdH~Tx9TZ~0WuvT+5HD(chIhd{YB-q(r$fN64!Bl>vPZzJ|9qmpsVDEdT#74E$
zcp|&;dp+f+?IzM@yORWnk+Z;Ry`6f$*aBq(i;Zxiza3urA?nG^C`B=!=N0@8vF6u!
zoX!s*k2U6SwlIv|aWd5nVxxTC-b(@MFU%NKsc(Pg&Mkl`&2SD6P(_MkNbM1=tmR#9
z6jwL?MVjYYwmUVA?`SU7Ql`KVzN_DXV
z#Bjnk`sDRbn-|b06f9zT6D^$s!3!a6PG|&sjHY9UOS@>t-&F;spEHNVEl}$;0ATFK
zpmoT5<{jP-9)4-G2AoQ)*&pPXlOh8OWX#9q4w)_~-FUGmB|~mn0YM)Sppe@_g95I|
z&2l!+we)To_b$Di3)ltBo}%11MlBH}Jp>&j)nA3x@3<&loiKGJ$Yj;pIx15F13v}=
zA|>Z%?m`d$2U}+uQ&-e=>lQ2S?(XhRad&t3gS!-Wch{oD5AN>n9Hda(p*R$~=Y5l#
zd^g|yv3K^*m8@j#x#oPv7~VacT%L`D?+}oVxp5S&HiZ0cx(1y1VHyt6N<3bYnM(N!
zW%km`iaK#rvhQp)vFD+*UeHLD)!C?2F$6W*a+g`QH{8qE?~_62yAy|3LHP!F@0zQ(
z=;R{=Pr%ZHU61kch7=}7GIKfc5Bal{=@%>d!#9eB-}5Ukv|c*o%ArGXNo{6W?np>|YBmf>bYYn
z`-_s(W)PSCn%QMeAMoEb6R*oUe3umDkFwpYj(0yqXJATPS1=Krq2b@GV{twg)b5tN
zkYqz8=cl6=&yIl~QwZB9^-MtSL-d^xaWW1VO#KU!i&Lc)fu%>YzK4dpg_SL0X9V)*
z7aqc0zPa;^(NzLz>sgMyJkHE9Ibb?Y0trWFIQ4^5_i``G>cb~SV9(;eGb1<``y1ar
zVQ*`7AHh~<4C;qw(>rCihWa;$_TvnGXQn608_MQBz_fN2GDtF59grn!HeXrF%oU>r
zW4D67cj0DXNVb7rorX+f#haOp_L*~XUQ!pxv^(FGQL8bgrav0^e7&egX7m(ey*yVU
zS=5Ubf^<2olm5icBsl_0~K`LV;C6-jpwY5HJJ@sDYz67YF88F
zlJU$G7&_+E3`%tqDA*LE))|RvSaisN6lG|$r2+{)=|NvCuK*=UD!5?&)6qo_vEre@
ztb!5}s0&?VSKT<9{5z!;0c?|&RuyIx{5yS3=xe=0Ak#>*HG`5j0Jq^T0RSM(@E63`
zO%#q!kc<<5$%vYP)_-FI=ykyde*A;9%Dx&i7DH4DtUmaf+{AhZaH&z74R?vVj(l@;iKP-={*B0e=Fdy!r
z^)Zr1Gg`|j@kI~5y|=iUj*6vYsk1u&I2$Bb^kxjKVxZQ_2`*#d>>@Wy_1w#43uX1Z
zLJQmGM(bnohM$3Co>zwlqeOROC?EwdOW^Yh+~D;?6a{Jg=fFHmv#9f;awG09>-JQENh$5)gprEdlNsrY
zSJQ3tRYs$buKcCq23KJL^0raxN0WQWB<^N{^(2HpEv)TfV4~f$ucs$7rc)yMz>SNQ
zR^o)jk?Zo6Nw>CdTXC=s8Qbt9h!v}tp6i`IzZXWW|3SxT%qRi*vBG=ggc)u&^YDO&
zhzANj%Px$Bh0%#oItWT19k?_EkCJP0K9x(QBhQbSFt}{pt@8HJt~d>{!PRc3-5qgR
zIBYCwm^b;jyN^^lT_iTkJ|k~Z(bt16dfC!UFb*(c@B(ww8K&wvCA<<<;tjVMWioZn
z)j))w?$FmXb6?W<=f*cxGW1~o;+PknTaH+UJDYGM4dSb9PK0|qZt*t<9t&;h(|n2k
zhO?V`TsRVDNrSq0un9@(s`S4D`Bd-+!39
z{Kj^A+QB5#%PdSB=%pOnsrcU60pM$nk79H=_S>N&sG-@L|F{>0(w%OwX$EQ6(I;rt
zCd?|vRHSX_v+B>mE<8MGo_~~YP&80=d`&3VL`x(bqn4;pKbmb#h?BTvADHa8oRA13
zHwnLN2D!H8zokvf+}=$`>4pr)Su8)**QRay#r$N{*T!sj>NdLsmHK#}6a7g@WUzzfsk#;
zf;(zp=cVDx-*XO#Gdm8N0*D&6r&Y({?TatRFy{~JvI(ylhjUsi2#7NN7#_s%6$q?4
z=>&ubYV2wpl2s{&ErwE!tgapT(HHjXO=9hy=G0
zV8Mth9vE>|8eXmJy4lL|WX;=-5YfJh1P(#ses18pr<{-bLU$k08VMOr3FYmoV|3nO^DqxD|-tiT%$6M7GOtESE@zbZoAtfsx5wE
zna?-^jhBI=%Gt-0FLnPFI*d%!O~F0C`!gzdMkC|8WeEarYGNuXZf-(IJ(B0VDP%+YIqpYtq>)SVC5=fs9gkvp|Ch
z8EtNuq8}aX?Xmz{YJDI=>l=8stBCUBAQ0oxOns6V_dQN7ltwv5z+>k3;C&`7JYsE3|yAq
z&65HdK^a}B<0$f8j=NB3u@oR(!L{?4ycIh5m6N{PdBGx~kdenNa>=A=y=bLMYUY3Y
z?xdO=V*T#~bS4*@K%w8$(Su3Hg%$bISUr7GXbaGn>ZYq@X0m(qS(kjT`202tGRDSe
zt!FwKsO;<)F{rZKPGNjg-O?hv^G!p}4ZwKZCP8(xY!lhBlq?%Ud5;=Xd(4k1heLYW
z!(Ssvk)+%i3o*b9bNq@(+mY06P3{|hqJUl&(_1@{ARYfbW#%S%x3k<{vcL(xjnZb#aWLAn9@3YqJh2*H__Q4NuSji?HP~@;pr!y2o
z3u1(GG5jTsBEONm^hLp&y;G0UxYw1nlQw1;9$#1L_L-vMQw+5ng-Cf3t|c%R>kGPt(U7#eR?RU-uFQV`h=(72V9MGq_SKbdc%vX2zh2^^ytDu>Im?
zWDfysr39F#`%Na)BV1JKKGbm~izk~c#ATa4+R|{@DW4Dm
z6-|+QV!7n{d`_B|@Ct5|dos1&$?=${S{8aA@J8q<@-A}mm|DAHUC7H#2)0sA?~o{_
z9r_NcYF*HH^&9T~7xZtScOaV0nJek6gWn*~7cjm-eA@ybp4;xU4d)7$)s5c9*E+v4
zNjkNiyt|o=n7nQiv5=cStE7pvBinx(#dGB=?1v{2Sk}St^6f3Hyvpzf&rC3UPQo
zXbthBemeug)y-KS<}q6l3-;K!1kk#nI&f*mRfP4?O7B
z7qt26#$m{2<1OUD-Xct6R@=!sHyAg#Dq&;xYGO4ACh`S(+MX3o+W&y`BOgbyr*8a;
z$RUFnyUlIJL6Dnc<>sRcZE0x0J<8BXrZ`&Xqs|)m4#W6-#gkXZ2d{ptG};iM`_zSG
z9pH!m112Ci7ShV}&MTCRAx0kChy{3KtJJo}k#^w94~QClu<^-nXBRS&ot&I@9MUya
zxx6Yenw5Sr2O4UudMa{VW4^x0Y4LCt2}8U&q3_jY+~LhRZ|6nqnEkvl&XM(*?nvvI
z)RyQmdE$)wIS$svAq()pV|U^L9?h}LB8AaIL%3Z;MAr6QJx1wEWG~@XL8J|B5YOsR
zjZ@66_fuzv5f1z5&aU|YC11UUy&6A;z0-LakpV%ZONnlU>Eb2Rus
zZ~M>oq}x@{S9*MaQ|@4qr;nvid9`7Y?&>f=CK)JcewZuH_SU7+-MJ?%xbbZF)$b~i
zht(bA2SOEwGG_UQO>s7btxN^?J7JJhoe@64%3{zn^E+vjBY9VuUU-nJ=J$H;TIuG?
z?z;u7Njvfd3fAkEijKmW)e;${*Qcro!GHkYG^7r+WYNnJHrR)gkA`TG)~iKmpy4zk
zODjCWSgq<|x}_fYLn^dS%VoL)I2n2uKckgPZs60$8ho3@EAW^%{ar$i+?Q}0S$wo5
zbn6j*?8d33gw{G%tdt63_Xstt{2Mt4@V!X(TgiOS2Q@Yk4q|{@?828OdiU^3#bsAF
z!EzASX;=tT(3jo6Rq%S|tv;P2WS+IHyp;52o|)bUAZ`3UlQ3L?!7AH`-_gH7`R;fw
zr`i{|Q*C(NHq66-+MKX+#?lXt=(>T&+#cry57$55efHPxT)|4p(69olnu^G=83A&_t6^{7I%zfmyee8pG5PTQI)>W_JTDpdj1
zdO-@ETDpD)#PPZf9PAB(Y8X8w=u1A86|CThaBb!f(Q+wIu*
zjG;*bkUnG=Zy^)^n`0h{K6>rTUFhB4mluQl9Mc0G(u81r@ZgH@BkP_*?VnI+mtCdS
zU_?`2Uxew9NWX~b`^|b@{a+b|Ii@nw2|TMo1BY@QOj?`CbQ3(bOCkhuAze37wAUolSWr3>M)RGC<1d@krQjV^@eS`OAr!bDw~Mk3r@;ySQKI?C+9
zCYeA@nV2!AF|VN=$57ea1jIf>+}aPZv~^(9AY4sd%h-#0$A0E+Bx;i?AefdS%rT6L
zaQ9SfD;p9utz6n92iq5iIU148svMy>*NT(=aBBL1y~K|%7R04~f}K=j#-qN)&(Sj~
z7G+q?^c8$YB&YO>3%Wv8w@au;0U;qCYSHXl%#B^iTMc6(`C@O7R?p@&qI72?6DGAB
z30YUDA+#*PKAL&0ACdGCwNBZK<#_vSMR3`h&4%GCR0*#O3Z4xsZ@e8e@
zs0mE(`07W1e%;TFc8_HY?;^Lu>Tm~P+Ped_TIS3?AA9gMAnL;m>NZ?TX|fT*OvH6Q
z`OcV%J}v@m%NefXkp@9CxF=RZ)UD2Z#?j{faH?3A;*a
z+*$~N5>XjiB!55xWwWvJA`^?uFv%TN#`0
zbEq8NLc0qAQ8o|-)F1%Qz=gsn6lcyNHMqO_AZ*q5BrEx_313~n9{1)CKdFz8^3^(3
z35QA$t>;kKc)Sb>R-4#c8D2ODtGA%#&+8QPo0xo%_(@LKU-=b*qf=_N3p+#^RI^Ms
zZ$mO^;FKa-w{|B-wb>dyz(;Mv*OHj~6&EAkxTw)W-P=Sz4Cb5p`3Ucun=mp>UVM)R
zJ$_tFRxK80nB~H;3-?Nxy9h&wyB=b#F4BXzcnqV-MBplJjqIy*he(w_NtU{xzM6vE
z&zh{g4u6Ro&W*{`5OO|;T76o-LNJJI
zj=3M`E)*xV%Nl+^SP{tFzaO8@g9(eQ0`6Kvu*f(8j!K))j-y$GGu!6qEFmktgevjA
z`>-%L_Y8iOyi6L8Ux&@9-$x
zUULQlvb)7y0{}ee{0iC$$Ke`ISR9B>TE+DbnRk6%B^rCnZvqI$G6m#N7cG{f+Vf;G
zI|^|Q4!iYCiKwQ5p<(Yml%tFpYrZih6B9b<1o2Z^k)R^hQ}tqWdRS9U6<$tffGMO5
zH9D`9+XiNMbr*`QUZ?qoy&1g<12aC65=od;9#LWjXSsnVGF>+5;AJh-w
zN7~;RF&bocL}F_l>v`ln;B#iP?lK?`D=?S0FuZ?;-qS~ZfWQt}O)l%*ff?4243;Ta
zDvQo@1n6Sn5kF~QXyt+N7WQS$05TYt4=jqQxRx9iv!{~HbS=b^X~x59qIr@+Ma);Y7quBfUb<@AC+Q=*Dnw8zZ#t6OzC38I(~Re#FRcCR&tS=$%`Ll6DktI}X_
z_PVWPl^pxp8cN|p`)ll0eX6JHu$vcBT&oW^s_VG)=o#HT@lYoj4
z!jOF>N(U<-W^FO4Oas*&v}$-sVuhkJ)5i>~%Iw!^n`3F!|J>w-qz~qrb6jdvtyjg(
zT(xS_+d%7cd!;7l64Md!T>1vtSFD^&8>@kn&lk$=iQf((1C~?s&
zwB_vCfrAnGqBQfxfKpebj!=gqL+~`c{F%7Ksa^f%L5R<~aNbLuJ||@a_mr%#;FzZg
zc#k0z@k07oP#HYhabM1Q{f;=aJ^!Z+&wPQr(C4QSlNr;i$WV~=tU2iX2U`0+LHEB&
z_dJ_7!GO0@hpi(3lkNYo)2UXz%cGk%AoM`C2Ef%aWD+nA?MGi--kF>hsYw70mc$>I
zs#Q8EV%(zIFzW~zYA{m(+`E`5m}v#$DYs7c;hklnBk`6eXVyR6#E;dol_?z&a`>D%
z_2kgWHL0-)D
zZecmI-Qa=T1{Z#{OJ2Zn3V4b6Y#Wdfitua&eKJNCVLhIOUIyWICc}3~HEc8km?3#)O=T80CyT^uT
zcGIK1`1~%5|5EY3u8zO+f0*}3@fJ=I1!!^;yKvFV9mk;cUnn`q4lWva?zy$N&!ztL
zE&593PTu{j;Ksn&4d50PmDrUlIYEuMNXaTrbM@J*D~c-1nxSteD%>oeHspx4k4RL%
z_@&EUZPrA^Pr!8e7SBs*<Zhx=~hx^WgVb)%CqazB%$d?rWwYl^}y3B&>
z1@qu6tS|Ui(44fI^-d)gL#y_$zn7+gG}J4@TD(43rn`-C?;b<(=p!JbF5Y}WQCtb4%E{|m(RsDFPy6eo%7
zL+t?{KrH%qJCHA=)MN}Xh#vPQXeV=WB7wRW;bP#iDnmNqbTK67u^>h
z{VlYEy`cCEHSjMS^-~9ekOuL0_&rqVv0ASLVgQT;Y%o4`SXGYhp@Eh*0h13TW|Vu7
zdN@ToV`~VkP9vKGgOZMN3D`E=opbxZzx7$Aq?O9l#!Vax<#^xFUPTmA&y4AlY6J{k
zjEO04grqn_MK`9Q9M|9!iy~YeEiR|a1vNCbqVs=i_ULsDU%nTLl7Cr$85b>!B4$0~
zM=-Ty`I~i!?Csx1IlNzwa2y`*47GgrE%HyoEf(s}sjkk5Oo1`h7;Pcs42^m)YSqoGtkIm%l=YQP4}Rw-IbDqOln)!vZ+{^ix{
ziW}eol)Vi)0_hmBL+v%)t9wA10kB<*PjreDuh84Y3g3zh9}X6I!h*lSgbJPRyKTA7
zbzVY~+9FGFetxk&*OoMCt@5-#^CA4*K{PfqbF;Uo>D?y3jb=lF3S9gDI0_x<%T3qo
zW+g-&4#mrLI6Ar=2qMj~Fk|%xk1sv^gAOlTNQG%GHEC75Oc{@rxlB5_yi0A`vuRFa
z3_G(NMi?q(woFZNdH8vtyC4=S6Bwz2g!U=+PN=*3g@;@37RFsn!^J^*@WMxQ_VDmm
zAYJxZ`M(G-yLs*3>Rk*pr}LF{H=03G>ig4iXy=A#)!;28OQzph^ln)+qW-lf1zy-%
zOF9bplh$n`L0@UtxGa}po}r-&9dSBgrXcKOiiGY7@-3Ha*c)Roq>7Gexw&U+t)QU~
zXU6Ic+Yi}VA>qhhRFBCwy}r_2c^0Lmm{^i06se#&-%1{)mBfS7c@cDZ;^W9NY3R%-
zownEj|H#E={?JiBT!Od^>kz7hB~4fV>p3P$(=;jRoN~#!_&Up?G3svk#fzJ0|
zZUVl{IG9Hc`Ox*z#aDOYZtlVuN**(eAk4{H<;&S2P3NgJRS~)wMpJS)_jiP;4QCKz
zT5IQJ7Bqen^uYCJjy?}wBZJJA%&u_6*dI?>7D@YV=3$7{%+u=G>&j)&NGa@*{M-wp
z0q>`}#s0kijM)7V(-oYdNVtaNf&+Y9lspu+MZ>O3Lm>!HnnB>wR}=}Bg-|A`7}kgk
zvV^3`pNZN2yc7jdNw4jeVsklnCZjPZ1=ggkO5xf6luoZ07VhCMxv&TGRL3wCB|uk&
ze0KFM`hKf6UXHGW_`Ov!ccR}pwdEEkYKz4@>u7o9(krxqr5bY1ANN@9do2h#)NQV2LB5bxXdVn%+Uch8n4F%ivw__~5u!R5BMW7&UhhDM-Gq$Cni
zd)k%O5zzzbcd#My$Ltnh8j4Kb--k8{h0<*|{j+{(v41az<+E@i46JI|_3s=imw0|H
z@p_S~L`(CkhlGcSOjRJG15_diXLqUK^X50lzEkEsA|hM~_8NF`%by%I2+;KMI@y|*vd#hV8DO&_v|`v52D5be
zO-xPBc~Mhy2b~NvD;OI}j#G
zg$}M!u>-|qk^VaE^IgasbT1}$Wu$1mD!m@=2_syBtq-
z*~>2MU_
zetH{`drAMlt3xL$!hv+>`Q+R$Jl%g6E#uw+xb&NeCX*OD#gAjxCDK+d?u1s`YlL?K
zZ_zJ_t60qDSCw~-2D&&}viq@LCv!UkGBn&fispll0!o_pHpM%|hgr`j`Gc^lr8IGR|d(B2F5u5
z4r?cTz}GPk+g&$ridYrKlS6Da5Gj9TL18MGgxwK0M%-5Kg7<^sUw%zu3DF}%y#~`=
zvo{y|1U3UTu|Q24@QK+N`2Baip{e+WRbFF(q>lrpBiu=;p!8?1x}jh#g3Y%f1R%iB
zw0MN+(Zk%Kg6$Ky$##+ZE`#C}vON@0q}>%!=?$O`km*Bc+sOQLT$2Cy^eM4OR`_A`
z3cy`YBQ+QkbA^
zmSOOnyHBt&dO{V=XwP0qCS#&dlQclNkaYM9#d!f{gZn@9OIcnfnEBsF?>goiQId2F
zf88d{YB*~Lua6`)k4RJK#_%
z@)k7WQ%nV#aLYFEIC5{JiJVcUUIJsCXd!n(z8_s`>qRhMtbkGf{h@n0
z|F<@+*^iz&awFW+8iNbgW*|9pfTVWE8JV7xOHmERJILT35lz;
zIF!Fub9b0oMJ9+aNXUuxHwG_dh(9WP)!#{q%YAathN>~75!Ro}hza`b;Svpwd9SYY
zbH#4(WDqM65g#ujF-?5Mi3r1j6?sDDJjKa#`fJF0)Cfr5qoP*;@#eQXhL7Jj<^Hjj
zN7F}ttH~?|j*6RiP`2L9mybnV5;hOgtmP<+TMNS(Znyx6!e`xKKguEhD*vWw6`HTO
zK;QH{No|X9;6Rl9B&T%@;QlJlo`iNsF!^m+Ovi#LY)oPhtQ&gDG23RI^%8;@aj`!D
z5VmrGacWlg)i7RM?G$A
zmaC#V)%Qa*Qy-)Cg*{Q8=PsYGtEo%f%i1c_?=H-AcKYIKI`8hs^{cScJKu-#NZ-xtJj%_O
z^2)H0yV+C^&!2r`AH)bJ!NjLP56?m?WuZTk%XpjGYpiQ?zymI
z6e)&T{m$T7AAp@G*aTO*8vY!}(j7;_BDWosjR5@hJ5$r$xqW3&4X!d5q9$F4U-Jo
z#+u;S=3+5*Y0O-6q&)Y%JH|#&ldUS0BD8aM09Wyk7s0T}3!4Yxe)-5iii;6cyIOVU
z$MJfE>~vo9Uk4`+^HK&t>9J(kNh};11OC;PJ+?$2;q>FQu}S-!DE+*-kNvlwgR`fE
zH^!iq7W^e+*q<`z!x?vJ$?BSq=PJf*y8I4~n~g+@0eu%SGs}+#W<7x%o-j{E68d2X
zU@PK+czFiw)gjxbxo6V)!iO7$Kx)-=v!8^z4k2U1Checv!Y4q2g`9zng
zVXWtUrq6I8?e5^QI+E1JiKPZAV3^l+?DPKTR)mPqKW604m64y}y`E&f?ytswW9_Y|
z^2vtl(`#{()?$8@y1F2T$|QH{THFPZX~0DHFH@G&Vs#3WY8MsH@taFUxvn$HJ2WpF
z_R%{c7vHCF>5=2gYED$~Pg3K6H=N