From 7a554e7ef1caedd10481712efd14a926c9de2ae8 Mon Sep 17 00:00:00 2001 From: Gwilym-Rutherford Date: Thu, 31 Jul 2025 17:24:40 +0100 Subject: [PATCH] Add metrics and metric data helper data class --- metrics_manager.py | 73 ++++++++++++++----- .../all_metrics/acceleration_metric.py | 27 +++++++ srunner/metrics/all_metrics/basic_metric.py | 48 ++++++++++++ .../metrics/all_metrics/collision_metric.py | 15 ++++ .../all_metrics/driving_score_metric.py | 23 ++++++ .../percentage_over_line_metric.py | 56 ++++++++++++++ .../all_metrics/route_completion_metric.py | 34 +++++++++ srunner/metrics/examples/criteria_filter.py | 4 +- .../examples/distance_between_vehicles.py | 2 +- .../examples/distance_to_lane_center.py | 2 +- srunner/metrics/helper/metric_data.py | 6 ++ 11 files changed, 267 insertions(+), 23 deletions(-) create mode 100644 srunner/metrics/all_metrics/acceleration_metric.py create mode 100644 srunner/metrics/all_metrics/basic_metric.py create mode 100644 srunner/metrics/all_metrics/collision_metric.py create mode 100644 srunner/metrics/all_metrics/driving_score_metric.py create mode 100644 srunner/metrics/all_metrics/percentage_over_line_metric.py create mode 100644 srunner/metrics/all_metrics/route_completion_metric.py create mode 100644 srunner/metrics/helper/metric_data.py diff --git a/metrics_manager.py b/metrics_manager.py index 3cdea4c..f461fc3 100644 --- a/metrics_manager.py +++ b/metrics_manager.py @@ -22,10 +22,12 @@ import inspect import json import argparse +import yaml from argparse import RawTextHelpFormatter import carla from srunner.metrics.tools.metrics_log import MetricsLog +from srunner.metrics.helper.metric_data import MetricData class MetricsManager(object): @@ -53,9 +55,25 @@ def __init__(self, args): # Instanciate the MetricsLog, used to querry the needed information log = MetricsLog(recorder_str) + # Parse defintino json input + with open(args.definition, "r", encoding="UTF-8") as raw_json: + MetricData.definition = json.loads(raw_json.read()) + + with open("config.yaml", "r") as stream: + MetricData.config = yaml.safe_load(stream) + + metrics = [ + "percentage_over_line_metric", + "collision_metric", + "acceleration_metric" + ] + # Read and run the metric class metric_class = self._get_metric_class(self._args.metric) - metric_class(town_map, log, criteria_dict) + if metric_class.__name__ == "driving_score_metric": + metric_class(town_map, log, metrics, criteria_dict) + else: + metric_class(town_map, log, criteria_dict) def _get_recorder(self, log): """ @@ -64,10 +82,10 @@ def _get_recorder(self, log): # Get the log information. self._client = carla.Client(self._args.host, int(self._args.port)) - recorder_file = "{}/{}".format(os.getenv('SCENARIO_RUNNER_ROOT', "./"), log) + recorder_file = "{}/{}".format(os.getenv("SCENARIO_RUNNER_ROOT", "./"), log) # Check that the file is correct - if recorder_file[-4:] != '.log': + if recorder_file[-4:] != ".log": print("ERROR: The log argument has to point to a .log file") sys.exit(-1) if not os.path.exists(recorder_file): @@ -99,7 +117,7 @@ def _get_metric_class(self, metric_file): metric_file (str): path to the metric's file. """ # Get their module - module_name = os.path.basename(metric_file).split('.')[0] + module_name = os.path.basename(metric_file).split(".")[0] sys.path.insert(0, os.path.dirname(metric_file)) metric_module = importlib.import_module(module_name) @@ -107,7 +125,7 @@ def _get_metric_class(self, metric_file): for member in inspect.getmembers(metric_module, inspect.isclass): # Get the first one with parent BasicMetrics member_parent = member[1].__bases__[0] - if 'BasicMetric' in str(member_parent): + if "BasicMetric" in str(member_parent): return member[1] print("No child class of BasicMetric was found ... Exiting") @@ -130,25 +148,42 @@ def main(): """ # pylint: disable=line-too-long - description = ("Scenario Runner's metrics module. Evaluate the execution of a specific scenario by developing your own metric.\n") - - parser = argparse.ArgumentParser(description=description, - formatter_class=RawTextHelpFormatter) - parser.add_argument('--host', default='127.0.0.1', - help='IP of the host server (default: localhost)') - parser.add_argument('--port', '-p', default=2000, - help='TCP port to listen to (default: 2000)') - parser.add_argument('--log', required=True, - help='Path to the CARLA recorder .log file (relative to SCENARIO_RUNNER_ROOT).\nThis file is created by the record functionality at ScenarioRunner') - parser.add_argument('--metric', required=True, - help='Path to the .py file defining the used metric.\nSome examples at srunner/metrics') - parser.add_argument('--criteria', default="", - help='Path to the .json file with the criteria information.\nThis file is created by the record functionality at ScenarioRunner') + description = "Scenario Runner's metrics module. Evaluate the execution of a specific scenario by developing your own metric.\n" + + parser = argparse.ArgumentParser( + description=description, formatter_class=RawTextHelpFormatter + ) + parser.add_argument( + "--host", default="127.0.0.1", help="IP of the host server (default: localhost)" + ) + parser.add_argument( + "--port", "-p", default=2000, help="TCP port to listen to (default: 2000)" + ) + parser.add_argument( + "--log", + required=True, + help="Path to the CARLA recorder .log file (relative to SCENARIO_RUNNER_ROOT).\nThis file is created by the record functionality at ScenarioRunner", + ) + parser.add_argument( + "--metric", + required=True, + default="srunner/metrics/all_metrics/driving_score_metric.py", + help="Path to the .py file defining the used metric.\nSome examples at srunner/metrics", + ) + parser.add_argument( + "--criteria", + default="", + help="Path to the .json file with the criteria information.\nThis file is created by the record functionality at ScenarioRunner", + ) + parser.add_argument( + "--definition", required=True, help="path to json scenario definition" + ) # pylint: enable=line-too-long args = parser.parse_args() MetricsManager(args) + if __name__ == "__main__": sys.exit(main()) diff --git a/srunner/metrics/all_metrics/acceleration_metric.py b/srunner/metrics/all_metrics/acceleration_metric.py new file mode 100644 index 0000000..bb0bce7 --- /dev/null +++ b/srunner/metrics/all_metrics/acceleration_metric.py @@ -0,0 +1,27 @@ +from .basic_metric import BasicMetric + +class AccelerationMetric(BasicMetric): + def __init__(self, town_map, log, criteria=None): + super().__init__(town_map, log, criteria) + self.metric_value = 0 + + def _create_metric(self, town_map, log, criteria): + ego_id = log.log.get_ego_vehicle_id() + start, end = log.get_actor_alive_frames(ego_id) + + biggest_difference = 0 + prev_frame = start + for frame in range(start, end + 1): + prev_accel = log.get_actor_acceleration(ego_id, prev_frame) + curr_accel = log.get_actor_acceleration(ego_id, frame) + accel_diff = abs(curr_accel - prev_accel) + + if accel_diff > biggest_difference: + biggest_difference = accel_diff + + + self.metric_value = biggest_difference + + + def get_value(self): + return self.metric_value \ No newline at end of file diff --git a/srunner/metrics/all_metrics/basic_metric.py b/srunner/metrics/all_metrics/basic_metric.py new file mode 100644 index 0000000..07d336f --- /dev/null +++ b/srunner/metrics/all_metrics/basic_metric.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright (c) 2020 Computer Vision Center (CVC) at the Universitat Autonoma de +# Barcelona (UAB). +# +# This work is licensed under the terms of the MIT license. +# For a copy, see . + +""" +This module provide BasicMetric, the basic class of all the metrics. +""" + +class BasicMetric(object): + """ + Base class of all the metrics. + """ + + def __init__(self, town_map, log, criteria=None): + """ + Initialization of the metric class. This calls the metrics log and creates the metrics + + Args: + town_map (carla.Map): Map of the simulation. Used to access the Waypoint API. + log (srunner.metrics.tools.Metricslog): instance of a class used to access the recorder information + criteria (dict): list of dictionaries with all the criteria information + """ + + # Create the metrics of the simulation. This part is left to the user + self._create_metric(town_map, log, criteria) + + def _create_metric(self, town_map, log, criteria): + """ + Pure virtual function to setup the metrics by the user. + + Args: + town_map (carla.Map): Map of the simulation. Used to access the Waypoint API. + log (srunner.metrics.tools.Metricslog): instance of a class used to access the recorder information + criteria (dict): dictionaries with all the criteria information + """ + raise NotImplementedError( + "This function should be re-implemented by all metrics" + "If this error becomes visible the class hierarchy is somehow broken") + + def get_value(self): + + raise NotImplementedError( + "This function should be implemented to return the specific metric value" + ) \ No newline at end of file diff --git a/srunner/metrics/all_metrics/collision_metric.py b/srunner/metrics/all_metrics/collision_metric.py new file mode 100644 index 0000000..32f91ab --- /dev/null +++ b/srunner/metrics/all_metrics/collision_metric.py @@ -0,0 +1,15 @@ +from .basic_metric import BasicMetric + +class CollisionMetric(BasicMetric): + + def __init__(self, town_map, log, criteria=None): + super().__init__(town_map, log, criteria) + self.metric_value = 0 + + def _create_metric(self, town_map, log, criteria): + ego_id = log.get_ego_vehicle_id() + collisions = log.get_actor_collisions(ego_id) + self.metric_value = len(collisions) + + def get_value(self): + return self.metric_value \ No newline at end of file diff --git a/srunner/metrics/all_metrics/driving_score_metric.py b/srunner/metrics/all_metrics/driving_score_metric.py new file mode 100644 index 0000000..d8c63cc --- /dev/null +++ b/srunner/metrics/all_metrics/driving_score_metric.py @@ -0,0 +1,23 @@ +import importlib +from .basic_metric import BasicMetric + +class DrivingScore(BasicMetric): + def __init__(self, town_map, log, metrics, criteria=None): + super().__init__(town_map, log, criteria) + self.metric_value = 0 + self.metrics = metrics + + def _create_metric(self, town_map, log, criteria): + metric_values = [] + module = importlib.import_module('srunner.metrics.all_metrics') + + for metric in self.metrics: + instance = getattr(module, metric)(town_map, log, criteria) + metric_values.append(instance.get_value()) + + self.metric_value = sum(metric_values) + + def get_value(self): + return self.metric_value + + \ No newline at end of file diff --git a/srunner/metrics/all_metrics/percentage_over_line_metric.py b/srunner/metrics/all_metrics/percentage_over_line_metric.py new file mode 100644 index 0000000..88fe645 --- /dev/null +++ b/srunner/metrics/all_metrics/percentage_over_line_metric.py @@ -0,0 +1,56 @@ +import math + +from .basic_metric import BasicMetric + + +class PercentageOverLineMetric(BasicMetric): + + def __init__(self, town_map, log, criteria=None): + super().__init__(town_map, log, criteria) + self.metric_value = 0 + + def _create_metric(self, town_map, log, criteria): + + # Get ego vehicle id + ego_id = log.get_ego_vehicle_id() + + # Get the frames the ego actor was alive and its transforms + start, end = log.get_actor_alive_frames(ego_id) + + actor_over_line_counter = 0 + + # Get the projected distance vector to the center of the lane + for i in range(start, end + 1): + + ego_location = log.get_actor_transform(ego_id, i).location + ego_waypoint = town_map.get_waypoint(ego_location) + + # Get the distance vector and project it + a = ego_location - ego_waypoint.transform.location # Ego to waypoint vector + b = ego_waypoint.transform.get_right_vector() # Waypoint perpendicular vector + b_norm = math.sqrt(b.x * b.x + b.y * b.y + b.z * b.z) + + ab_dot = a.x * b.x + a.y * b.y + a.z * b.z + dist_v = ab_dot/(b_norm*b_norm)*b + dist = math.sqrt(dist_v.x * dist_v.x + dist_v.y * dist_v.y + dist_v.z * dist_v.z) + + # Get the sign of the distance (left side is positive) + c = ego_waypoint.transform.get_forward_vector() # Waypoint forward vector + ac_cross = c.x * a.y - c.y * a.x + if ac_cross < 0: + dist *= -1 + + if dist <= 0: + actor_over_line_counter += 1 + + total_frames = end - start + percentage = (total_frames - actor_over_line_counter/actor_over_line_counter) * 100 + self.metric_value = 1 - percentage + + def get_value(self): + return self.metric_value + + + + + diff --git a/srunner/metrics/all_metrics/route_completion_metric.py b/srunner/metrics/all_metrics/route_completion_metric.py new file mode 100644 index 0000000..37212f8 --- /dev/null +++ b/srunner/metrics/all_metrics/route_completion_metric.py @@ -0,0 +1,34 @@ +from math import sqrt, pow +from .basic_metric import BasicMetric +from srunner.metrics.helper.metric_data import MetricData + + +class RouteCompletionMetric(BasicMetric): + + def __init__(self, town_map, log, criteria=None): + super().__init__(town_map, log, criteria) + self.metric_value = 0 + + def _create_metric(self, town_map, log, criteria): + ego_id = log.get_ego_vehicle_id() + start, end = log.get_actor_alive_frames(ego_id) + + waypoint_index = 0 + ego_waypoints = MetricData.definition['routes'][0]['waypoints'] + for i in range(start, end + 1): + curr_waypoint = ego_waypoints[waypoint_index]['position'] + ego_location = log.get_actor_transform(ego_id, i).location + + if self._euclidean_distance(curr_waypoint, ego_location) < 0.1: + waypoint_index += 1 + self.metric_value = waypoint_index/len(ego_waypoints) + + def _euclidean_distance(self, curr_waypoint, ego_location): + return sqrt( + pow(curr_waypoint['x'] - ego_location.x, 2) + + pow(curr_waypoint['y'] - ego_location.y, 2) + + pow(curr_waypoint['z'] - ego_location.z, 2) + ) + + def get_value(self): + return self.metric_value \ No newline at end of file diff --git a/srunner/metrics/examples/criteria_filter.py b/srunner/metrics/examples/criteria_filter.py index e1eacf2..dcc7927 100644 --- a/srunner/metrics/examples/criteria_filter.py +++ b/srunner/metrics/examples/criteria_filter.py @@ -15,7 +15,7 @@ import json -from srunner.metrics.examples.basic_metric import BasicMetric +from srunner.metrics.all_metrics.basic_metric import BasicMetric class CriteriaFilter(BasicMetric): @@ -23,7 +23,7 @@ class CriteriaFilter(BasicMetric): Metric class CriteriaFilter """ - def _create_metric(self, town_map, log, criteria): + def _create_metric(self, town_map, log, definition, criteria): """ Implementation of the metric. This is an example to show how to use the criteria """ diff --git a/srunner/metrics/examples/distance_between_vehicles.py b/srunner/metrics/examples/distance_between_vehicles.py index 5716c8f..6ac4744 100644 --- a/srunner/metrics/examples/distance_between_vehicles.py +++ b/srunner/metrics/examples/distance_between_vehicles.py @@ -17,7 +17,7 @@ import math import matplotlib.pyplot as plt -from srunner.metrics.examples.basic_metric import BasicMetric +from srunner.metrics.all_metrics.basic_metric import BasicMetric class DistanceBetweenVehicles(BasicMetric): diff --git a/srunner/metrics/examples/distance_to_lane_center.py b/srunner/metrics/examples/distance_to_lane_center.py index 3916096..222d443 100644 --- a/srunner/metrics/examples/distance_to_lane_center.py +++ b/srunner/metrics/examples/distance_to_lane_center.py @@ -16,7 +16,7 @@ import math import json -from srunner.metrics.examples.basic_metric import BasicMetric +from srunner.metrics.all_metrics.basic_metric import BasicMetric class DistanceToLaneCenter(BasicMetric): diff --git a/srunner/metrics/helper/metric_data.py b/srunner/metrics/helper/metric_data.py new file mode 100644 index 0000000..d44fe79 --- /dev/null +++ b/srunner/metrics/helper/metric_data.py @@ -0,0 +1,6 @@ +from dataclasses import dataclass + +@dataclass +class MetricData: + definition: dict + config: dict \ No newline at end of file