Add metrics and metric data helper data class

metrics_manager.py

@@ -22,10 +22,12 @@
 import inspect
 import json
 import argparse
+import yaml
 from argparse import RawTextHelpFormatter
 
 import carla
 from srunner.metrics.tools.metrics_log import MetricsLog
+from srunner.metrics.helper.metric_data import MetricData
 
 
 class MetricsManager(object):
@@ -53,9 +55,25 @@ def __init__(self, args):
         # Instanciate the MetricsLog, used to querry the needed information
         log = MetricsLog(recorder_str)
 
+        # Parse defintino json input
+        with open(args.definition, "r", encoding="UTF-8") as raw_json:
+            MetricData.definition = json.loads(raw_json.read())
+
+        with open("config.yaml", "r") as stream:
+            MetricData.config = yaml.safe_load(stream)
+
+        metrics = [
+            "percentage_over_line_metric",
+            "collision_metric",
+            "acceleration_metric"
+        ]
+
         # Read and run the metric class
         metric_class = self._get_metric_class(self._args.metric)
-        metric_class(town_map, log, criteria_dict)
+        if metric_class.__name__ == "driving_score_metric":
+            metric_class(town_map, log, metrics, criteria_dict)
+        else:
+            metric_class(town_map, log, criteria_dict)
 
     def _get_recorder(self, log):
         """
@@ -64,10 +82,10 @@ def _get_recorder(self, log):
 
         # Get the log information.
         self._client = carla.Client(self._args.host, int(self._args.port))
-        recorder_file = "{}/{}".format(os.getenv('SCENARIO_RUNNER_ROOT', "./"), log)
+        recorder_file = "{}/{}".format(os.getenv("SCENARIO_RUNNER_ROOT", "./"), log)
 
         # Check that the file is correct
-        if recorder_file[-4:] != '.log':
+        if recorder_file[-4:] != ".log":
             print("ERROR: The log argument has to point to a .log file")
             sys.exit(-1)
         if not os.path.exists(recorder_file):
@@ -99,15 +117,15 @@ def _get_metric_class(self, metric_file):
             metric_file (str): path to the metric's file.
         """
         # Get their module
-        module_name = os.path.basename(metric_file).split('.')[0]
+        module_name = os.path.basename(metric_file).split(".")[0]
         sys.path.insert(0, os.path.dirname(metric_file))
         metric_module = importlib.import_module(module_name)
 
         # And their members of type class
         for member in inspect.getmembers(metric_module, inspect.isclass):
             # Get the first one with parent BasicMetrics
             member_parent = member[1].__bases__[0]
-            if 'BasicMetric' in str(member_parent):
+            if "BasicMetric" in str(member_parent):
                 return member[1]
 
         print("No child class of BasicMetric was found ... Exiting")
@@ -130,25 +148,42 @@ def main():
     """
 
     # pylint: disable=line-too-long
-    description = ("Scenario Runner's metrics module. Evaluate the execution of a specific scenario by developing your own metric.\n")
-
-    parser = argparse.ArgumentParser(description=description,
-                                    formatter_class=RawTextHelpFormatter)
-    parser.add_argument('--host', default='127.0.0.1',
-                        help='IP of the host server (default: localhost)')
-    parser.add_argument('--port', '-p', default=2000,
-                        help='TCP port to listen to (default: 2000)')
-    parser.add_argument('--log', required=True,
-                        help='Path to the CARLA recorder .log file (relative to SCENARIO_RUNNER_ROOT).\nThis file is created by the record functionality at ScenarioRunner')
-    parser.add_argument('--metric', required=True,
-                        help='Path to the .py file defining the used metric.\nSome examples at srunner/metrics')
-    parser.add_argument('--criteria', default="",
-                        help='Path to the .json file with the criteria information.\nThis file is created by the record functionality at ScenarioRunner')
+    description = "Scenario Runner's metrics module. Evaluate the execution of a specific scenario by developing your own metric.\n"
+
+    parser = argparse.ArgumentParser(
+        description=description, formatter_class=RawTextHelpFormatter
+    )
+    parser.add_argument(
+        "--host", default="127.0.0.1", help="IP of the host server (default: localhost)"
+    )
+    parser.add_argument(
+        "--port", "-p", default=2000, help="TCP port to listen to (default: 2000)"
+    )
+    parser.add_argument(
+        "--log",
+        required=True,
+        help="Path to the CARLA recorder .log file (relative to SCENARIO_RUNNER_ROOT).\nThis file is created by the record functionality at ScenarioRunner",
+    )
+    parser.add_argument(
+        "--metric",
+        required=True,
+        default="srunner/metrics/all_metrics/driving_score_metric.py",
+        help="Path to the .py file defining the used metric.\nSome examples at srunner/metrics",
+    )
+    parser.add_argument(
+        "--criteria",
+        default="",
+        help="Path to the .json file with the criteria information.\nThis file is created by the record functionality at ScenarioRunner",
+    )
+    parser.add_argument(
+        "--definition", required=True, help="path to json scenario definition"
+    )
     # pylint: enable=line-too-long
 
     args = parser.parse_args()
 
     MetricsManager(args)
 
+
 if __name__ == "__main__":
     sys.exit(main())

srunner/metrics/all_metrics/acceleration_metric.py

@@ -0,0 +1,27 @@
+from .basic_metric import BasicMetric
+
+class AccelerationMetric(BasicMetric):
+    def __init__(self, town_map, log, criteria=None):
+        super().__init__(town_map, log, criteria)
+        self.metric_value = 0
+
+    def _create_metric(self, town_map, log, criteria):
+        ego_id = log.log.get_ego_vehicle_id()
+        start, end = log.get_actor_alive_frames(ego_id)
+
+        biggest_difference = 0
+        prev_frame = start
+        for frame in range(start, end + 1):
+            prev_accel = log.get_actor_acceleration(ego_id, prev_frame)
+            curr_accel = log.get_actor_acceleration(ego_id, frame)
+            accel_diff = abs(curr_accel - prev_accel)
+
+            if accel_diff > biggest_difference:
+                biggest_difference = accel_diff
+
+
+        self.metric_value = biggest_difference
+
+
+    def get_value(self):
+        return self.metric_value

srunner/metrics/all_metrics/basic_metric.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Barcelona (UAB).
+#
+# This work is licensed under the terms of the MIT license.
+# For a copy, see <https://opensource.org/licenses/MIT>.
+
+"""
+This module provide BasicMetric, the basic class of all the metrics.
+"""
+
+class BasicMetric(object):
+    """
+    Base class of all the metrics.
+    """
+
+    def __init__(self, town_map, log, criteria=None):
+        """
+        Initialization of the metric class. This calls the metrics log and creates the metrics
+
+        Args:
+            town_map (carla.Map): Map of the simulation. Used to access the Waypoint API.
+            log (srunner.metrics.tools.Metricslog): instance of a class used to access the recorder information
+            criteria (dict): list of dictionaries with all the criteria information
+        """
+
+        # Create the metrics of the simulation. This part is left to the user
+        self._create_metric(town_map, log, criteria)
+
+    def _create_metric(self, town_map, log, criteria):
+        """
+        Pure virtual function to setup the metrics by the user.
+
+        Args:
+            town_map (carla.Map): Map of the simulation. Used to access the Waypoint API.
+            log (srunner.metrics.tools.Metricslog): instance of a class used to access the recorder information
+            criteria (dict): dictionaries with all the criteria information
+        """
+        raise NotImplementedError(
+            "This function should be re-implemented by all metrics"
+            "If this error becomes visible the class hierarchy is somehow broken")
+
+    def get_value(self):
+
+        raise NotImplementedError(
+            "This function should be implemented to return the specific metric value"
+        )

srunner/metrics/all_metrics/collision_metric.py

@@ -0,0 +1,15 @@
+from .basic_metric import BasicMetric
+
+class CollisionMetric(BasicMetric):
+
+    def __init__(self, town_map, log, criteria=None):
+        super().__init__(town_map, log, criteria)
+        self.metric_value = 0
+
+    def _create_metric(self, town_map, log, criteria):
+        ego_id = log.get_ego_vehicle_id()
+        collisions = log.get_actor_collisions(ego_id)
+        self.metric_value = len(collisions)
+
+    def get_value(self):
+        return self.metric_value

srunner/metrics/all_metrics/driving_score_metric.py

@@ -0,0 +1,23 @@
+import importlib
+from .basic_metric import BasicMetric
+
+class DrivingScore(BasicMetric):
+    def __init__(self, town_map, log, metrics, criteria=None):
+        super().__init__(town_map, log, criteria)
+        self.metric_value = 0       
+        self.metrics = metrics
+
+    def _create_metric(self, town_map, log, criteria):
+        metric_values = []
+        module = importlib.import_module('srunner.metrics.all_metrics')
+
+        for metric in self.metrics:
+            instance = getattr(module, metric)(town_map, log, criteria)
+            metric_values.append(instance.get_value())
+
+        self.metric_value = sum(metric_values)
+
+    def get_value(self):
+        return self.metric_value
+
+

srunner/metrics/all_metrics/percentage_over_line_metric.py

@@ -0,0 +1,56 @@
+import math
+
+from .basic_metric import BasicMetric
+
+
+class PercentageOverLineMetric(BasicMetric):
+
+    def __init__(self, town_map, log, criteria=None):
+        super().__init__(town_map, log, criteria)
+        self.metric_value = 0
+
+    def _create_metric(self, town_map, log, criteria):
+
+        # Get ego vehicle id
+        ego_id = log.get_ego_vehicle_id()
+
+        # Get the frames the ego actor was alive and its transforms
+        start, end = log.get_actor_alive_frames(ego_id)
+
+        actor_over_line_counter = 0
+
+        # Get the projected distance vector to the center of the lane
+        for i in range(start, end + 1):
+
+            ego_location = log.get_actor_transform(ego_id, i).location
+            ego_waypoint = town_map.get_waypoint(ego_location)
+
+            # Get the distance vector and project it
+            a = ego_location - ego_waypoint.transform.location      # Ego to waypoint vector
+            b = ego_waypoint.transform.get_right_vector()           # Waypoint perpendicular vector
+            b_norm = math.sqrt(b.x * b.x + b.y * b.y + b.z * b.z)
+
+            ab_dot = a.x * b.x + a.y * b.y + a.z * b.z
+            dist_v = ab_dot/(b_norm*b_norm)*b
+            dist = math.sqrt(dist_v.x * dist_v.x + dist_v.y * dist_v.y + dist_v.z * dist_v.z)
+
+            # Get the sign of the distance (left side is positive)
+            c = ego_waypoint.transform.get_forward_vector()         # Waypoint forward vector
+            ac_cross = c.x * a.y - c.y * a.x
+            if ac_cross < 0:
+                dist *= -1
+
+            if dist <= 0:
+                actor_over_line_counter += 1
+
+        total_frames = end - start
+        percentage = (total_frames - actor_over_line_counter/actor_over_line_counter) * 100
+        self.metric_value = 1 - percentage
+
+    def get_value(self):
+        return self.metric_value
+
+
+
+
+

srunner/metrics/all_metrics/route_completion_metric.py

@@ -0,0 +1,34 @@
+from math import sqrt, pow
+from .basic_metric import BasicMetric
+from srunner.metrics.helper.metric_data import MetricData
+
+
+class RouteCompletionMetric(BasicMetric):
+
+    def __init__(self, town_map, log, criteria=None):
+        super().__init__(town_map, log, criteria)
+        self.metric_value = 0
+
+    def _create_metric(self, town_map, log, criteria):
+        ego_id = log.get_ego_vehicle_id()
+        start, end = log.get_actor_alive_frames(ego_id)
+
+        waypoint_index = 0
+        ego_waypoints = MetricData.definition['routes'][0]['waypoints']
+        for i in range(start, end + 1):
+            curr_waypoint = ego_waypoints[waypoint_index]['position']
+            ego_location = log.get_actor_transform(ego_id, i).location
+
+            if self._euclidean_distance(curr_waypoint, ego_location) < 0.1:
+                waypoint_index += 1
+                self.metric_value = waypoint_index/len(ego_waypoints)
+
+    def _euclidean_distance(self, curr_waypoint, ego_location):
+        return sqrt(
+            pow(curr_waypoint['x'] - ego_location.x, 2) +
+            pow(curr_waypoint['y'] - ego_location.y, 2) +
+            pow(curr_waypoint['z'] - ego_location.z, 2)
+        )
+
+    def get_value(self):
+        return self.metric_value

srunner/metrics/examples/criteria_filter.py

@@ -15,15 +15,15 @@
 
 import json
 
-from srunner.metrics.examples.basic_metric import BasicMetric
+from srunner.metrics.all_metrics.basic_metric import BasicMetric
 
 
 class CriteriaFilter(BasicMetric):
     """
     Metric class CriteriaFilter
     """
 
-    def _create_metric(self, town_map, log, criteria):
+    def _create_metric(self, town_map, log, definition, criteria):
         """
         Implementation of the metric. This is an example to show how to use the criteria
         """

srunner/metrics/examples/distance_between_vehicles.py

@@ -17,7 +17,7 @@
 import math
 import matplotlib.pyplot as plt
 
-from srunner.metrics.examples.basic_metric import BasicMetric
+from srunner.metrics.all_metrics.basic_metric import BasicMetric
 
 
 class DistanceBetweenVehicles(BasicMetric):

srunner/metrics/examples/distance_to_lane_center.py

@@ -16,7 +16,7 @@
 import math
 import json
 
-from srunner.metrics.examples.basic_metric import BasicMetric
+from srunner.metrics.all_metrics.basic_metric import BasicMetric
 
 
 class DistanceToLaneCenter(BasicMetric):

srunner/metrics/helper/metric_data.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+@dataclass
+class MetricData:
+    definition: dict
+    config: dict