references.bib

@article{2020SciPy-NMeth,
  author  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and
             Haberland, Matt and Reddy, Tyler and Cournapeau, David and
             Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and
             Bright, Jonathan and {van der Walt}, St{\'e}fan J. and
             Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and
             Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and
             Kern, Robert and Larson, Eric and Carey, C J and
             Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and
             {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and
             Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and
             Harris, Charles R. and Archibald, Anne M. and
             Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and
             {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},
  title   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific
             Computing in Python}},
  journal = {Nature Methods},
  year    = {2020},
  volume  = {17},
  pages   = {261--272},
  adsurl  = {https://rdcu.be/b08Wh},
  doi     = {10.1038/s41592-019-0686-2}
}

@inproceedings{8793740,
  author    = {Abeysirigoonawardena, Yasasa and Shkurti, Florian and Dudek, Gregory},
  booktitle = {2019 International Conference on Robotics and Automation (ICRA)},
  title     = {Generating Adversarial Driving Scenarios in High-Fidelity Simulators},
  year      = {2019},
  volume    = {},
  number    = {},
  pages     = {8271-8277},
  keywords  = {Optimization;Accidents;Rendering (computer graphics);Bayes methods;Reinforcement learning;Roads;Trajectory},
  doi       = {10.1109/ICRA.2019.8793740}
}

@article{9829243,
  author   = {Li, Quanyi and Peng, Zhenghao and Feng, Lan and Zhang, Qihang and Xue, Zhenghai and Zhou, Bolei},
  journal  = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
  title    = {MetaDrive: Composing Diverse Driving Scenarios for Generalizable Reinforcement Learning},
  year     = {2023},
  volume   = {45},
  number   = {3},
  pages    = {3461-3475},
  keywords = {Task analysis;Roads;Reinforcement learning;Benchmark testing;Training;Safety;Autonomous vehicles;Reinforcement learning;autonomous driving;simulation},
  doi      = {10.1109/TPAMI.2022.3190471}
}

@inproceedings{abdessalem_testing_2018,
  location  = {New York, {NY}, {USA}},
  title     = {Testing vision-based control systems using learnable evolutionary algorithms},
  isbn      = {978-1-4503-5638-1},
  url       = {https://doi.org/10.1145/3180155.3180160},
  doi       = {10.1145/3180155.3180160},
  series    = {{ICSE} '18},
  abstract  = {Vision-based control systems are key enablers of many autonomous vehicular systems, including self-driving cars. Testing such systems is complicated by complex and multidimensional input spaces. We propose an automated testing algorithm that builds on learnable evolutionary algorithms. These algorithms rely on machine learning or a combination of machine learning and Darwinian genetic operators to guide the generation of new solutions (test scenarios in our context). Our approach combines multiobjective population-based search algorithms and decision tree classification models to achieve the following goals: First, classification models guide the search-based generation of tests faster towards critical test scenarios (i.e., test scenarios leading to failures). Second, search algorithms refine classification models so that the models can accurately characterize critical regions (i.e., the regions of a test input space that are likely to contain most critical test scenarios). Our evaluation performed on an industrial automotive automotive system shows that: (1) Our algorithm outperforms a baseline evolutionary search algorithm and generates 78\% more distinct, critical test scenarios compared to the baseline algorithm. (2) Our algorithm accurately characterizes critical regions of the system under test, thus identifying the conditions that are likely to lead to system failures.},
  pages     = {1016--1026},
  booktitle = {Proceedings of the 40th International Conference on Software Engineering},
  publisher = {Association for Computing Machinery},
  author    = {Abdessalem, Raja Ben and Nejati, Shiva and Briand, Lionel C. and Stifter, Thomas},
  urldate   = {2023-11-20},
  date      = {2018-05-27},
  keywords  = {automotive software systems, evolutionary algorithms, search-based software engineering, software testing},
  file      = {Full Text:/home/olek/Zotero/storage/JALE8266/Abdessalem et al. - 2018 - Testing vision-based control systems using learnab.pdf:application/pdf}
}


@inproceedings{abeysirigoonawardena2019generating,
  title     = {Generating adversarial driving scenarios in high-fidelity simulators},
  author    = {Abeysirigoonawardena, Yasasa and Shkurti, Florian and Dudek, Gregory},
  booktitle = {ICRA},
  year      = {2019}
}

@inproceedings{amini2022vista,
  title        = {Vista 2.0: An open, data-driven simulator for multimodal sensing and policy learning for autonomous vehicles},
  author       = {Amini, Alexander and Wang, Tsun-Hsuan and Gilitschenski, Igor and Schwarting, Wilko and Liu, Zhijian and Han, Song and Karaman, Sertac and Rus, Daniela},
  booktitle    = {2022 International Conference on Robotics and Automation (ICRA)},
  pages        = {2419--2426},
  year         = {2022},
  organization = {IEEE}
}

@inproceedings{amini2022vista,
  title        = {Vista 2.0: An open, data-driven simulator for multimodal sensing and policy learning for autonomous vehicles},
  author       = {Amini, Alexander and Wang, Tsun-Hsuan and Gilitschenski, Igor and Schwarting, Wilko and Liu, Zhijian and Han, Song and Karaman, Sertac and Rus, Daniela},
  booktitle    = {2022 International Conference on Robotics and Automation (ICRA)},
  pages        = {2419--2426},
  year         = {2022},
  organization = {IEEE}
}

@article{amini2024evaluating,
  title     = {Evaluating the impact of flaky simulators on testing autonomous driving systems},
  author    = {Amini, Mohammad Hossein and Naseri, Shervin and Nejati, Shiva},
  journal   = {Empirical Software Engineering},
  volume    = {29},
  number    = {2},
  pages     = {47},
  year      = {2024},
  publisher = {Springer}
}

@book{ammann2016introduction,
  title     = {Introduction to software testing},
  author    = {Ammann, Paul and Offutt, Jeff},
  year      = {2016},
  publisher = {Cambridge University Press}
}


@inproceedings{arcaini2022less,
  title        = {Less is more: Simplification of test scenarios for autonomous driving system testing},
  author       = {Arcaini, Paolo and Zhang, Xiao-Yi and Ishikawa, Fuyuki},
  booktitle    = {2022 IEEE Conference on Software Testing, Verification and Validation (ICST)},
  pages        = {279--290},
  year         = {2022},
  organization = {IEEE}
}

@article{arcuri2014hitchhiker,
  title     = {A hitchhiker's guide to statistical tests for assessing randomized algorithms in software engineering},
  author    = {Arcuri, Andrea and Briand, Lionel},
  journal   = {Software Testing, Verification and Reliability},
  volume    = {24},
  number    = {3},
  pages     = {219--250},
  year      = {2014},
  publisher = {Wiley Online Library}
}

@article{autofuzz,
  author   = {Zhong, Ziyuan and Kaiser, Gail and Ray, Baishakhi},
  journal  = {IEEE Transactions on Software Engineering},
  title    = {Neural Network Guided Evolutionary Fuzzing for Finding Traffic Violations of Autonomous Vehicles},
  year     = {2023},
  volume   = {49},
  number   = {4},
  pages    = {1860-1875},
  keywords = {Automobiles;Testing;Fuzzing;Vehicle crash testing;Grammar;Artificial neural networks;Roads;Search-based software engineering;evolutionary algorithms;neural networks;software testing;test generation;autonomous vehicles},
  doi      = {10.1109/TSE.2022.3195640}
}


@misc{autoware_docs,
  author       = {{The Autoware Foundation}},
  title        = {{Autoware Documentation}},
  howpublished = {\url{https://autowarefoundation.github.io/autoware-documentation/main/home/}},
  year         = {2026},
  note         = {Accessed: June 9, 2026}
}

@online{av_market_raport,
  title    = {Autonomous Vehicle Market - Size, Share, Forecast \& Growth},
  url      = {https://www.mordorintelligence.com/industry-reports/autonomous-driverless-cars-market-potential-estimation},
  abstract = {The Autonomous (Driverless) Car Market is expected to reach {USD} 41.10 billion in 2024 and grow at a {CAGR} of 22.75\% to reach {USD} 114.54 billion by 2029. Volkswagen {AG}, Toyota Motor Corporation, General Motors Company, Daimler {AG} and Nissan Motor Co., Ltd. are the major companies operating in this market.},
  urldate  = {2024-04-11},
  langid   = {english},
  file     = {Snapshot:/home/olek/Zotero/storage/KPIXAQA3/autonomous-driverless-cars-market-potential-estimation.html:text/html}
}


@inproceedings{av-fuzzer,
  author    = {Li, Guanpeng and Li, Yiran and Jha, Saurabh and Tsai, Timothy and Sullivan, Michael and Hari, Siva Kumar Sastry and Kalbarczyk, Zbigniew and Iyer, Ravishankar},
  booktitle = {2020 IEEE 31st International Symposium on Software Reliability Engineering (ISSRE)},
  title     = {AV-FUZZER: Finding Safety Violations in Autonomous Driving Systems},
  year      = {2020},
  volume    = {},
  number    = {},
  pages     = {25-36},
  keywords  = {Perturbation methods;Web and internet services;Safety;Trajectory;Vehicle dynamics;Autonomous vehicles;Testing;Autonomous vehicles;safety-critical applications},
  doi       = {10.1109/ISSRE5003.2020.00012}
}

@inproceedings{bagschik2018ontology,
  title        = {Ontology based scene creation for the development of automated vehicles},
  author       = {Bagschik, Gerrit and Menzel, Till and Maurer, Markus},
  booktitle    = {2018 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {1813--1820},
  year         = {2018},
  organization = {IEEE}
}

@article{Bai2024-domain-gap,
  author     = {Bai, Xiangyu and Luo, Yedi and Jiang, Le and Gupta, Aniket and Kaveti, Pushyami and Singh, Hanumant and Ostadabbas, Sarah},
  title      = {Bridging the Domain Gap between Synthetic and Real-World Data for Autonomous Driving},
  year       = {2024},
  issue_date = {June 2024},
  publisher  = {Association for Computing Machinery},
  address    = {New York, NY, USA},
  volume     = {1},
  number     = {2},
  url        = {https://doi.org/10.1145/3633463},
  doi        = {10.1145/3633463},
  abstract   = {Modern autonomous systems require extensive testing to ensure reliability and build trust in ground vehicles. However, testing these systems in the real-world is challenging due to the lack of large and diverse datasets, especially in edge cases. Therefore, simulations are necessary for their development and evaluation. However, existing open-source simulators often exhibit a significant gap between synthetic and real-world domains, leading to deteriorated mobility performance and reduced platform reliability when using simulation data. To address this issue, our Scoping Autonomous Vehicle Simulation (SAVeS) platform benchmarks the performance of simulated environments for autonomous ground vehicle testing between synthetic and real-world domains. Our platform aims to quantify the domain gap and enable researchers to develop and test autonomous systems in a controlled environment. Additionally, we propose using domain adaptation technologies to address the domain gap between synthetic and real-world data with our SAVeS+ extension. Our results demonstrate that SAVeS+ is effective in helping to close the gap between synthetic and real-world domains and yields comparable performance for models trained with processed synthetic datasets to those trained on real-world datasets of same scale. Finally, we introduce two new autonomy driving datasets with complex scenes, essential sensor data, ground truth and improved imagery. The data is generated using both open-source and commercial simulators and processed through our SAVeS+ domain adaptation pipeline. This paper highlights our efforts to quantify and address the domain gap between synthetic and real-world data for autonomy simulation. By enabling researchers to develop and test autonomous systems in a controlled environment, we hope to bring autonomy simulation one step closer to realization.1},
  journal    = {ACM J. Auton. Transport. Syst.},
  month      = {Apr},
  articleno  = {9},
  numpages   = {15},
  keywords   = {Autonomous ground vehicles, machine learning inference, simulation, simultaneous localization and mapping (SLAM) algorithms, synthetic environments, domain adaption}
}

@inproceedings{balasubramanian2021traffic,
  title        = {Traffic scenario clustering by iterative optimisation of self-supervised networks using a random forest activation pattern similarity},
  author       = {Balasubramanian, Lakshman and Wurst, Jonas and Botsch, Michael and Deng, Ke},
  booktitle    = {2021 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {682--689},
  year         = {2021},
  organization = {IEEE}
}

@inproceedings{balasubramanian2021traffic,
  title        = {Traffic scenario clustering by iterative optimisation of self-supervised networks using a random forest activation pattern similarity},
  author       = {Balasubramanian, Lakshman and Wurst, Jonas and Botsch, Michael and Deng, Ke},
  booktitle    = {2021 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {682--689},
  year         = {2021},
  organization = {IEEE}
}

@inproceedings{bashetty_deepcrashtest_2020,
  title      = {{DeepCrashTest}: Turning Dashcam Videos into Virtual Crash Tests for Automated Driving Systems},
  url        = {https://ieeexplore.ieee.org/abstract/document/9197053},
  doi        = {10.1109/ICRA40945.2020.9197053},
  shorttitle = {{DeepCrashTest}},
  abstract   = {The goal of this paper is to generate simulations with real-world collision scenarios for training and testing autonomous vehicles. We use numerous dashcam crash videos uploaded on the internet to extract valuable collision data and recreate the crash scenarios in a simulator. We tackle the problem of extracting 3D vehicle trajectories from videos recorded by an unknown and uncalibrated monocular camera source using a modular approach. A working architecture and demonstration videos along with the open-source implementation are provided with the paper.},
  eventtitle = {2020 {IEEE} International Conference on Robotics and Automation ({ICRA})},
  pages      = {11353--11360},
  booktitle  = {2020 {IEEE} International Conference on Robotics and Automation ({ICRA})},
  author     = {Bashetty, Sai Krishna and Ben Amor, Heni and Fainekos, Georgios},
  urldate    = {2023-12-07},
  date       = {2020-05},
  year       = {2020},
  note       = {{ISSN}: 2577-087X},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/2Y5KBETD/Bashetty et al. - 2020 - DeepCrashTest Turning Dashcam Videos into Virtual.pdf:application/pdf}
}

@inproceedings{bashetty_deepcrashtest_2020,
  title      = {{DeepCrashTest}: Turning Dashcam Videos into Virtual Crash Tests for Automated Driving Systems},
  url        = {https://ieeexplore.ieee.org/abstract/document/9197053},
  doi        = {10.1109/ICRA40945.2020.9197053},
  shorttitle = {{DeepCrashTest}},
  abstract   = {The goal of this paper is to generate simulations with real-world collision scenarios for training and testing autonomous vehicles. We use numerous dashcam crash videos uploaded on the internet to extract valuable collision data and recreate the crash scenarios in a simulator. We tackle the problem of extracting 3D vehicle trajectories from videos recorded by an unknown and uncalibrated monocular camera source using a modular approach. A working architecture and demonstration videos along with the open-source implementation are provided with the paper.},
  eventtitle = {2020 {IEEE} International Conference on Robotics and Automation ({ICRA})},
  pages      = {11353--11360},
  booktitle  = {2020 {IEEE} International Conference on Robotics and Automation ({ICRA})},
  author     = {Bashetty, Sai Krishna and Ben Amor, Heni and Fainekos, Georgios},
  urldate    = {2023-12-07},
  date       = {2020-05},
  year       = {2020},
  note       = {{ISSN}: 2577-087X},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/2Y5KBETD/Bashetty et al. - 2020 - DeepCrashTest Turning Dashcam Videos into Virtual.pdf:application/pdf}
}

@misc{beamngtech,
  title  = {BeamNG.tech Technical Paper},
  author = {Pascale Maul and Marc Mueller and Fabian Enkler and Eva Pigova and Thomas Fischer and Lefteris Stamatogiannakis},
  year   = {2021}
}

@book{beizer2003software,
  title     = {Software testing techniques},
  author    = {Beizer, Boris},
  year      = {2003},
  publisher = {Dreamtech Press}
}


@book{beizer2003software,
  title     = {Software testing techniques},
  author    = {Beizer, Boris},
  year      = {2003},
  publisher = {Dreamtech Press}
}

@book{boehm2002software,
  title     = {Software engineering economics},
  author    = {Boehm, Barry W},
  year      = {2002},
  publisher = {Springer}
}


@book{boehm2002software,
  title     = {Software engineering economics},
  author    = {Boehm, Barry W},
  year      = {2002},
  publisher = {Springer}
}



@inproceedings{bussler2020application,
  title        = {Application of evolutionary algorithms and criticality metrics for the verification and validation of automated driving systems at urban intersections},
  author       = {Bussler, Andreas and Hartjen, Lukas and Philipp, Robin and Schuldt, Fabian},
  booktitle    = {2020 IEEE intelligent vehicles symposium (IV)},
  pages        = {128--135},
  year         = {2020},
  organization = {IEEE}
}

@inproceedings{bussler2020application,
  title        = {Application of evolutionary algorithms and criticality metrics for the verification and validation of automated driving systems at urban intersections},
  author       = {Bussler, Andreas and Hartjen, Lukas and Philipp, Robin and Schuldt, Fabian},
  booktitle    = {2020 IEEE intelligent vehicles symposium (IV)},
  pages        = {128--135},
  year         = {2020},
  organization = {IEEE}
}


@article{caesar2021nuplan,
  title   = {nuplan: A closed-loop ml-based planning benchmark for autonomous vehicles},
  author  = {Caesar, Holger and Kabzan, Juraj and Tan, Kok Seang and Fong, Whye Kit and Wolff, Eric and Lang, Alex and Fletcher, Luke and Beijbom, Oscar and Omari, Sammy},
  journal = {arXiv preprint arXiv:2106.11810},
  year    = {2021}
}


@article{caesar2021nuplan,
  title   = {nuplan: A closed-loop ml-based planning benchmark for autonomous vehicles},
  author  = {Caesar, Holger and Kabzan, Juraj and Tan, Kok Seang and Fong, Whye Kit and Wolff, Eric and Lang, Alex and Fletcher, Luke and Beijbom, Oscar and Omari, Sammy},
  journal = {arXiv preprint arXiv:2106.11810},
  year    = {2021}
}


@misc{caesar2022nuplan,
  title         = {NuPlan: A closed-loop ML-based planning benchmark for autonomous vehicles},
  author        = {Holger Caesar and Juraj Kabzan and Kok Seang Tan and Whye Kit Fong and Eric Wolff and Alex Lang and Luke Fletcher and Oscar Beijbom and Sammy Omari},
  year          = {2022},
  eprint        = {2106.11810},
  archiveprefix = {arXiv},
  primaryclass  = {cs.CV}
}


@misc{caesar2022nuplan,
  title         = {NuPlan: A closed-loop ML-based planning benchmark for autonomous vehicles},
  author        = {Holger Caesar and Juraj Kabzan and Kok Seang Tan and Whye Kit Fong and Eric Wolff and Alex Lang and Luke Fletcher and Oscar Beijbom and Sammy Omari},
  year          = {2022},
  eprint        = {2106.11810},
  archiveprefix = {arXiv},
  primaryclass  = {cs.CV}
}

@article{cai_survey_2022,
  author         = {Cai, Jinkang and Deng, Weiwen and Guang, Haoran and Wang, Ying and Li, Jiangkun and Ding, Juan},
  title          = {A Survey on Data-Driven Scenario Generation for Automated Vehicle Testing},
  journal        = {Machines},
  volume         = {10},
  year           = {2022},
  number         = {11},
  article-number = {1101},
  url            = {https://www.mdpi.com/2075-1702/10/11/1101},
  issn           = {2075-1702},
  abstract       = {Automated driving is a promising tool for reducing traffic accidents. While some companies claim that many cutting-edge automated driving functions have been developed, how to evaluate the safety of automated vehicles remains an open question, which has become a crucial bottleneck. Scenario-based testing has been introduced to test automated vehicles, and much progress has been achieved. While data-driven and knowledge-based approaches are hot research topics, this survey is mainly about Data-Driven Scenario Generation (DDSG) for automated vehicle testing. Rather than describe the contributions of every study respectively, in this survey, methodologies from various studies are anatomized as solutions for several significant problems and compared with each other. This way, scholars and engineers can quickly find state-of-the-art approaches to the issues they might encounter. Furthermore, several critical challenges that might hinder DDSG are described, and responding solutions are presented at the end of this survey.},
  doi            = {10.3390/machines10111101}
}

@article{cai_survey_2022,
  author         = {Cai, Jinkang and Deng, Weiwen and Guang, Haoran and Wang, Ying and Li, Jiangkun and Ding, Juan},
  title          = {A Survey on Data-Driven Scenario Generation for Automated Vehicle Testing},
  journal        = {Machines},
  volume         = {10},
  year           = {2022},
  number         = {11},
  article-number = {1101},
  url            = {https://www.mdpi.com/2075-1702/10/11/1101},
  issn           = {2075-1702},
  abstract       = {Automated driving is a promising tool for reducing traffic accidents. While some companies claim that many cutting-edge automated driving functions have been developed, how to evaluate the safety of automated vehicles remains an open question, which has become a crucial bottleneck. Scenario-based testing has been introduced to test automated vehicles, and much progress has been achieved. While data-driven and knowledge-based approaches are hot research topics, this survey is mainly about Data-Driven Scenario Generation (DDSG) for automated vehicle testing. Rather than describe the contributions of every study respectively, in this survey, methodologies from various studies are anatomized as solutions for several significant problems and compared with each other. This way, scholars and engineers can quickly find state-of-the-art approaches to the issues they might encounter. Furthermore, several critical challenges that might hinder DDSG are described, and responding solutions are presented at the end of this survey.},
  doi            = {10.3390/machines10111101}
}

@inproceedings{cai2020summit,
  title        = {Summit: A simulator for urban driving in massive mixed traffic},
  author       = {Cai, Panpan and Lee, Yiyuan and Luo, Yuanfu and Hsu, David},
  booktitle    = {2020 IEEE International Conference on Robotics and Automation (ICRA)},
  pages        = {4023--4029},
  year         = {2020},
  organization = {IEEE}
}


@inproceedings{cai2020summit,
  title        = {Summit: A simulator for urban driving in massive mixed traffic},
  author       = {Cai, Panpan and Lee, Yiyuan and Luo, Yuanfu and Hsu, David},
  booktitle    = {2020 IEEE International Conference on Robotics and Automation (ICRA)},
  pages        = {4023--4029},
  year         = {2020},
  organization = {IEEE}
}


@misc{carla_docs,
  author       = {{CARLA Contributors}},
  title        = {{CARLA} Simulator documentation},
  howpublished = {\url{https://carla.readthedocs.io/en/0.9.10/}},
  note         = {Online; accessed 27 October 2024},
  year         = 2024
}

@misc{carla_leaderboard,
  title      = {Get started with Leaderboard 2.0},
  author     = {{CARLA} Team},
  year       = {2024},
  url        = {https://leaderboard.carla.org/get_started_v2_0/},
  abstract   = {Information about how to get started with the {CARLA} Leaderboard version 2.0 and its rules.},
  titleaddon = {{CARLA} Autonomous Driving Leaderboard},
  note       = {Online; accessed 09 November 2025},
  langid     = {english}
}

@inproceedings{carla_sim,
  title     = { {CARLA}: {An} Open Urban Driving Simulator},
  author    = {Alexey Dosovitskiy and German Ros and Felipe Codevilla and Antonio Lopez and Vladlen Koltun},
  booktitle = {Proceedings of the 1st Annual Conference on Robot Learning},
  pages     = {1--16},
  year      = {2017}
}

@online{carmaker,
  title   = {{CarMaker} {\textbar} {IPG} Automotive},
  url     = {https://www.ipg-automotive.com/en/products-solutions/software/carmaker/},
  urldate = {2024-04-15},
  file    = {CarMaker | IPG Automotive:/home/olek/Zotero/storage/FWTUNP4T/carmaker.html:text/html}
}

@online{carmaker,
  title   = {{CarMaker} {\textbar} {IPG} Automotive},
  url     = {https://www.ipg-automotive.com/en/products-solutions/software/carmaker/},
  urldate = {2024-04-15},
  file    = {CarMaker | IPG Automotive:/home/olek/Zotero/storage/FWTUNP4T/carmaker.html:text/html}
}

@article{chance2022determinism,
  title     = {On determinism of game engines used for simulation-based autonomous vehicle verification},
  author    = {Chance, Greg and Ghobrial, Abanoub and McAreavey, Kevin and Lemaignan, S{\'e}verin and Pipe, Tony and Eder, Kerstin},
  journal   = {IEEE Transactions on Intelligent Transportation Systems},
  volume    = {23},
  number    = {11},
  pages     = {20538--20552},
  year      = {2022},
  publisher = {IEEE}
}


@misc{chen_end_to_end_2023,
  title      = {End-to-end Autonomous Driving: Challenges and Frontiers},
  url        = {http://arxiv.org/abs/2306.16927},
  doi        = {10.48550/arXiv.2306.16927},
  shorttitle = {End-to-end Autonomous Driving},
  abstract   = {The autonomous driving community has witnessed a rapid growth in approaches that embrace an end-to-end algorithm framework, utilizing raw sensor input to generate vehicle motion plans, instead of concentrating on individual tasks such as detection and motion prediction. End-to-end systems, in comparison to modular pipelines, benefit from joint feature optimization for perception and planning. This field has flourished due to the availability of large-scale datasets, closed-loop evaluation, and the increasing need for autonomous driving algorithms to perform effectively in challenging scenarios. In this survey, we provide a comprehensive analysis of more than 250 papers, covering the motivation, roadmap, methodology, challenges, and future trends in end-to-end autonomous driving. We delve into several critical challenges, including multi-modality, interpretability, causal confusion, robustness, and world models, amongst others. Additionally, we discuss current advancements in foundation models and visual pre-training, as well as how to incorporate these techniques within the end-to-end driving framework. To facilitate future research, we maintain an active repository that contains up-to-date links to relevant literature and open-source projects at https://github.com/{OpenDriveLab}/End-to-end-Autonomous-Driving.},
  number     = {{arXiv}:2306.16927},
  publisher  = {{arXiv}},
  author     = {Chen, Li and Wu, Penghao and Chitta, Kashyap and Jaeger, Bernhard and Geiger, Andreas and Li, Hongyang},
  urldate    = {2024-03-24},
  date       = {2023-06-29},
  eprinttype = {arxiv},
  eprint     = {2306.16927 [cs]},
  keywords   = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Computer Science - Robotics, Computer Science - Computer Vision and Pattern Recognition},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/EUI388QS/Chen et al. - 2023 - End-to-end Autonomous Driving Challenges and Fron.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/6MSPPL96/2306.html:text/html}
}


@misc{chen_end_to_end_2023,
  title      = {End-to-end Autonomous Driving: Challenges and Frontiers},
  url        = {http://arxiv.org/abs/2306.16927},
  doi        = {10.48550/arXiv.2306.16927},
  shorttitle = {End-to-end Autonomous Driving},
  abstract   = {The autonomous driving community has witnessed a rapid growth in approaches that embrace an end-to-end algorithm framework, utilizing raw sensor input to generate vehicle motion plans, instead of concentrating on individual tasks such as detection and motion prediction. End-to-end systems, in comparison to modular pipelines, benefit from joint feature optimization for perception and planning. This field has flourished due to the availability of large-scale datasets, closed-loop evaluation, and the increasing need for autonomous driving algorithms to perform effectively in challenging scenarios. In this survey, we provide a comprehensive analysis of more than 250 papers, covering the motivation, roadmap, methodology, challenges, and future trends in end-to-end autonomous driving. We delve into several critical challenges, including multi-modality, interpretability, causal confusion, robustness, and world models, amongst others. Additionally, we discuss current advancements in foundation models and visual pre-training, as well as how to incorporate these techniques within the end-to-end driving framework. To facilitate future research, we maintain an active repository that contains up-to-date links to relevant literature and open-source projects at https://github.com/{OpenDriveLab}/End-to-end-Autonomous-Driving.},
  number     = {{arXiv}:2306.16927},
  publisher  = {{arXiv}},
  author     = {Chen, Li and Wu, Penghao and Chitta, Kashyap and Jaeger, Bernhard and Geiger, Andreas and Li, Hongyang},
  urldate    = {2024-03-24},
  date       = {2023-06-29},
  eprinttype = {arxiv},
  eprint     = {2306.16927 [cs]},
  keywords   = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Computer Science - Robotics, Computer Science - Computer Vision and Pattern Recognition},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/EUI388QS/Chen et al. - 2023 - End-to-end Autonomous Driving Challenges and Fron.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/6MSPPL96/2306.html:text/html}
}

@article{chen2017multimodel,
  title     = {Multimodel fusion based sequential optimization},
  author    = {Chen, Shishi and Jiang, Zhen and Yang, Shuxing and Chen, Wei},
  journal   = {AIAA journal},
  volume    = {55},
  number    = {1},
  pages     = {241--254},
  year      = {2017},
  publisher = {American Institute of Aeronautics and Astronautics}
}

@inproceedings{chen2022lav,
  title     = {Learning from all vehicles},
  author    = {Chen, Dian and Kr{\"a}henb{\"u}hl, Philipp},
  booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages     = {17222--17231},
  year      = {2022}
}

@inproceedings{chen2022learning,
  title     = {Learning from all vehicles},
  author    = {Chen, Dian and Kr{\"a}henb{\"u}hl, Philipp},
  booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages     = {17222--17231},
  year      = {2022}
}

@article{chitta2022transfuser,
  title     = {Transfuser: Imitation with transformer-based sensor fusion for autonomous driving},
  author    = {Chitta, Kashyap and Prakash, Aditya and Jaeger, Bernhard and Yu, Zehao and Renz, Katrin and Geiger, Andreas},
  journal   = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
  volume    = {45},
  number    = {11},
  pages     = {12878--12895},
  year      = {2022},
  publisher = {IEEE}
}

@article{corso_survey_2021,
  title        = {A Survey of Algorithms for Black-Box Safety Validation of Cyber-Physical Systems},
  volume       = {72},
  issn         = {1076-9757},
  url          = {https://jair.org/index.php/jair/article/view/12716},
  doi          = {10.1613/jair.1.12716},
  abstract     = {Autonomous cyber-physical systems ({CPS}) can improve safety and efficiency for safetycritical applications, but require rigorous testing before deployment. The complexity of these systems often precludes the use of formal verification and real-world testing can be too dangerous during development. Therefore, simulation-based techniques have been developed that treat the system under test as a black box operating in a simulated environment. Safety validation tasks include finding disturbances in the environment that cause the system to fail (falsification), finding the most-likely failure, and estimating the probability that the system fails. Motivated by the prevalence of safety-critical artificial intelligence, this work provides a survey of state-of-the-art safety validation techniques for {CPS} with a focus on applied algorithms and their modifications for the safety validation problem. We present and discuss algorithms in the domains of optimization, path planning, reinforcement learning, and importance sampling. Problem decomposition techniques are presented to help scale algorithms to large state spaces, which are common for {CPS}. A brief overview of safety-critical applications is given, including autonomous vehicles and aircraft collision avoidance systems. Finally, we present a survey of existing academic and commercially available safety validation tools.},
  journaltitle = {Journal of Artificial Intelligence Research},
  shortjournal = {jair},
  author       = {Corso, Anthony and Moss, Robert and Koren, Mark and Lee, Ritchie and Kochenderfer, Mykel},
  urldate      = {2023-04-09},
  date         = {2021-10-13},
  langid       = {english},
  file         = {Corso et al. - 2021 - A Survey of Algorithms for Black-Box Safety Valida.pdf:/home/olek/Zotero/storage/F4NKKCZU/Corso et al. - 2021 - A Survey of Algorithms for Black-Box Safety Valida.pdf:application/pdf}
}

@article{corso_survey_2021,
  title        = {A Survey of Algorithms for Black-Box Safety Validation of Cyber-Physical Systems},
  volume       = {72},
  issn         = {1076-9757},
  url          = {https://jair.org/index.php/jair/article/view/12716},
  doi          = {10.1613/jair.1.12716},
  abstract     = {Autonomous cyber-physical systems ({CPS}) can improve safety and efficiency for safetycritical applications, but require rigorous testing before deployment. The complexity of these systems often precludes the use of formal verification and real-world testing can be too dangerous during development. Therefore, simulation-based techniques have been developed that treat the system under test as a black box operating in a simulated environment. Safety validation tasks include finding disturbances in the environment that cause the system to fail (falsification), finding the most-likely failure, and estimating the probability that the system fails. Motivated by the prevalence of safety-critical artificial intelligence, this work provides a survey of state-of-the-art safety validation techniques for {CPS} with a focus on applied algorithms and their modifications for the safety validation problem. We present and discuss algorithms in the domains of optimization, path planning, reinforcement learning, and importance sampling. Problem decomposition techniques are presented to help scale algorithms to large state spaces, which are common for {CPS}. A brief overview of safety-critical applications is given, including autonomous vehicles and aircraft collision avoidance systems. Finally, we present a survey of existing academic and commercially available safety validation tools.},
  journaltitle = {Journal of Artificial Intelligence Research},
  shortjournal = {jair},
  author       = {Corso, Anthony and Moss, Robert and Koren, Mark and Lee, Ritchie and Kochenderfer, Mykel},
  urldate      = {2023-04-09},
  date         = {2021-10-13},
  langid       = {english},
  file         = {Corso et al. - 2021 - A Survey of Algorithms for Black-Box Safety Valida.pdf:/home/olek/Zotero/storage/F4NKKCZU/Corso et al. - 2021 - A Survey of Algorithms for Black-Box Safety Valida.pdf:application/pdf}
}

@software{craig_quiter_2020_3871907,
  author    = {Craig Quiter},
  title     = {Deepdrive Zero},
  month     = jun,
  year      = 2020,
  publisher = {Zenodo},
  version   = {alpha},
  doi       = {10.5281/zenodo.3871907},
  url       = {https://doi.org/10.5281/zenodo.3871907}
}

@software{craig_quiter_2020_3871907,
  author    = {Craig Quiter},
  title     = {Deepdrive Zero},
  month     = jun,
  year      = 2020,
  publisher = {Zenodo},
  version   = {alpha},
  doi       = {10.5281/zenodo.3871907},
  url       = {https://doi.org/10.5281/zenodo.3871907}
}

@article{damm2017traffic,
  title   = {Traffic sequence charts-from visualization to semantics},
  author  = {Damm, Werner and Kemper, Stephanie and M{\"o}hlmann, Eike and Peikenkamp, Thomas and Rakow, Astrid},
  journal = {AVACS Technical Report 117},
  year    = {2017}
}

@article{damm2017traffic,
  title   = {Traffic sequence charts-from visualization to semantics},
  author  = {Damm, Werner and Kemper, Stephanie and M{\"o}hlmann, Eike and Peikenkamp, Thomas and Rakow, Astrid},
  journal = {AVACS Technical Report 117},
  year    = {2017}
}


@article{dilhara_understanding_2021,
  title        = {Understanding Software-2.0: A Study of Machine Learning Library Usage and Evolution},
  volume       = {30},
  issn         = {1049-331X},
  url          = {https://dl.acm.org/doi/10.1145/3453478},
  doi          = {10.1145/3453478},
  shorttitle   = {Understanding Software-2.0},
  abstract     = {Enabled by a rich ecosystem of Machine Learning ({ML}) libraries, programming using learned models, i.e., Software-2.0, has gained substantial adoption. However, we do not know what challenges developers encounter when they use {ML} libraries. With this knowledge gap, researchers miss opportunities to contribute to new research directions, tool builders do not invest resources where automation is most needed, library designers cannot make informed decisions when releasing {ML} library versions, and developers fail to use common practices when using {ML} libraries.We present the first large-scale quantitative and qualitative empirical study to shed light on how developers in Software-2.0 use {ML} libraries, and how this evolution affects their code. Particularly, using static analysis we perform a longitudinal study of 3,340 top-rated open-source projects with 46,110 contributors. To further understand the challenges of {ML} library evolution, we survey 109 developers who introduce and evolve {ML} libraries. Using this rich dataset we reveal several novel findings.Among others, we found an increasing trend of using {ML} libraries: The ratio of new Python projects that use {ML} libraries increased from 2\% in 2013 to 50\% in 2018. We identify several usage patterns including the following: (i) 36\% of the projects use multiple {ML} libraries to implement various stages of the {ML} workflows, (ii) developers update {ML} libraries more often than the traditional libraries, (iii) strict upgrades are the most popular for {ML} libraries among other update kinds, (iv) {ML} library updates often result in cascading library updates, and (v) {ML} libraries are often downgraded (22.04\% of cases). We also observed unique challenges when evolving and maintaining Software-2.0 such as (i) binary incompatibility of trained {ML} models and (ii) benchmarking {ML} models. Finally, we present actionable implications of our findings for researchers, tool builders, developers, educators, library vendors, and hardware vendors.},
  pages        = {55:1--55:42},
  number       = {4},
  journaltitle = {{ACM} Trans. Softw. Eng. Methodol.},
  author       = {Dilhara, Malinda and Ketkar, Ameya and Dig, Danny},
  urldate      = {2024-09-24},
  date         = {2021-07-23},
  file         = {Full Text PDF:/home/olek/Zotero/storage/5KSP3Q33/Dilhara et al. - 2021 - Understanding Software-2.0 A Study of Machine Learning Library Usage and Evolution.pdf:application/pdf}
}


@article{Ding2023,
  author   = {Ding, Wenhao and Xu, Chejian and Arief, Mansur and Lin, Haohong and Li, Bo and Zhao, Ding},
  journal  = {IEEE Transactions on Intelligent Transportation Systems},
  title    = {A Survey on Safety-Critical Driving Scenario Generation—A Methodological Perspective},
  year     = {2023},
  volume   = {24},
  number   = {7},
  pages    = {6971-6988},
  keywords = {Measurement;Safety;Autonomous vehicles;Vehicle dynamics;Roads;Heuristic algorithms;Trajectory;Autonomous vehicles;safety;robustness;deep generative models},
  doi      = {10.1109/TITS.2023.3259322}
}

@article{ding2023survey,
  title     = {A survey on safety-critical driving scenario generation—A methodological perspective},
  author    = {Ding, Wenhao and Xu, Chejian and Arief, Mansur and Lin, Haohong and Li, Bo and Zhao, Ding},
  journal   = {IEEE Transactions on Intelligent Transportation Systems},
  year      = {2023},
  publisher = {IEEE}
}
  
@article{do2023multifidelity,
  title     = {Multifidelity Bayesian Optimization: A Review},
  author    = {Do, Bach and Zhang, Ruda},
  journal   = {AIAA Journal},
  pages     = {1--37},
  year      = {2023},
  publisher = {American Institute of Aeronautics and Astronautics}
}

@inproceedings{eck2019understanding,
  title     = {Understanding flaky tests: The developer’s perspective},
  author    = {Eck, Moritz and Palomba, Fabio and Castelluccio, Marco and Bacchelli, Alberto},
  booktitle = {Proceedings of the 2019 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering},
  pages     = {830--840},
  year      = {2019}
}

@inproceedings{erbsmehl_simulation_2009,
  title     = {Simulation of real crashes as a method for estimating the potential benefits of advanced safety technologies},
  url       = {https://www-esv.nhtsa.dot.gov/Proceedings/21/09-0162.pdf},
  booktitle = {Technical Conference on the Enhanced Safety of Vehicles},
  author    = {Erbsmehl, Christian},
  urldate   = {2023-12-07},
  date      = {2009},
  note      = {Issue: 09-0162},
  file      = {Available Version (via Google Scholar):/home/olek/Zotero/storage/LWFCYQSV/Erbsmehl - 2009 - Simulation of real crashes as a method for estimat.pdf:application/pdf}
}

@inproceedings{erbsmehl_simulation_2009,
  title     = {Simulation of real crashes as a method for estimating the potential benefits of advanced safety technologies},
  url       = {https://www-esv.nhtsa.dot.gov/Proceedings/21/09-0162.pdf},
  booktitle = {Technical Conference on the Enhanced Safety of Vehicles},
  author    = {Erbsmehl, Christian},
  urldate   = {2023-12-07},
  date      = {2009},
  note      = {Issue: 09-0162},
  file      = {Available Version (via Google Scholar):/home/olek/Zotero/storage/LWFCYQSV/Erbsmehl - 2009 - Simulation of real crashes as a method for estimat.pdf:application/pdf}
}

@inproceedings{esenturk_analyzing_2021,
  title      = {Analyzing Real-world Accidents for Test Scenario Generation for Automated Vehicles},
  url        = {https://ieeexplore.ieee.org/abstract/document/9576007},
  doi        = {10.1109/IV48863.2021.9576007},
  abstract   = {Identification of test scenarios for Automated Driving Systems ({ADSs}) remains a key challenge for the Verification \& Validation of {ADSs}. Various approaches including data based approaches and knowledge based approaches have been proposed for scenario generation. Identifying the conditions that lead to high severity traffic accidents can help us not only identify test scenarios for {ADSs}, but also implement measures to save lives and infrastructure resources. Taking a data based approach, in this paper, we introduce a novel accident data analysis method for generating test scenarios where we analyze {UK}'s Stats19 accident data to identify trends in high severity accidents for test scenario generation. This paper first focuses on the severity of the accidents with the goal of relating it to static and time-dependent internal and external factors in a comprehensive way taking into account Operational Design Domain ({ODD}) properties, e.g. road, environmental conditions, and vehicle properties and driver characteristics. For this purpose, the paper utilizes a data grouping strategy (coarse-graining) and builds a logistic regression approach, derived from conventional regression models, in which emerging features become more pronounced, while uninteresting features and noise weaken. The approach makes the relationship between the factors and outcome variable more visible and hence well suited for the severity analysis. The method shows superior performance as compared to ordinary logistic models measured by goodness of fit and accounting for model variance (R{\textasciicircum}2=0.05 for the ordinary model, R{\textasciicircum}2=0.85 for the current model). The model is then used to solve the inverse problem of constructing high-risk pre-crash conditions as test scenarios for simulation based testing of {ADSs}.},
  eventtitle = {2021 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {288--295},
  booktitle  = {2021 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Esenturk, E. and Khastgir, S. and Wallace, A. and Jennings, P.},
  urldate    = {2024-01-12},
  date       = {2021-07},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/TB7P2C9R/9576007.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/X66BC99S/Esenturk et al. - 2021 - Analyzing Real-world Accidents for Test Scenario G.pdf:application/pdf}
}

@inproceedings{esenturk_analyzing_2021,
  title      = {Analyzing Real-world Accidents for Test Scenario Generation for Automated Vehicles},
  url        = {https://ieeexplore.ieee.org/abstract/document/9576007},
  doi        = {10.1109/IV48863.2021.9576007},
  abstract   = {Identification of test scenarios for Automated Driving Systems ({ADSs}) remains a key challenge for the Verification \& Validation of {ADSs}. Various approaches including data based approaches and knowledge based approaches have been proposed for scenario generation. Identifying the conditions that lead to high severity traffic accidents can help us not only identify test scenarios for {ADSs}, but also implement measures to save lives and infrastructure resources. Taking a data based approach, in this paper, we introduce a novel accident data analysis method for generating test scenarios where we analyze {UK}'s Stats19 accident data to identify trends in high severity accidents for test scenario generation. This paper first focuses on the severity of the accidents with the goal of relating it to static and time-dependent internal and external factors in a comprehensive way taking into account Operational Design Domain ({ODD}) properties, e.g. road, environmental conditions, and vehicle properties and driver characteristics. For this purpose, the paper utilizes a data grouping strategy (coarse-graining) and builds a logistic regression approach, derived from conventional regression models, in which emerging features become more pronounced, while uninteresting features and noise weaken. The approach makes the relationship between the factors and outcome variable more visible and hence well suited for the severity analysis. The method shows superior performance as compared to ordinary logistic models measured by goodness of fit and accounting for model variance (R{\textasciicircum}2=0.05 for the ordinary model, R{\textasciicircum}2=0.85 for the current model). The model is then used to solve the inverse problem of constructing high-risk pre-crash conditions as test scenarios for simulation based testing of {ADSs}.},
  eventtitle = {2021 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {288--295},
  booktitle  = {2021 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Esenturk, E. and Khastgir, S. and Wallace, A. and Jennings, P.},
  urldate    = {2024-01-12},
  date       = {2021-07},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/TB7P2C9R/9576007.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/X66BC99S/Esenturk et al. - 2021 - Analyzing Real-world Accidents for Test Scenario G.pdf:application/pdf}
}

@inproceedings{feng2023trafficgen,
  title        = {Trafficgen: Learning to generate diverse and realistic traffic scenarios},
  author       = {Feng, Lan and Li, Quanyi and Peng, Zhenghao and Tan, Shuhan and Zhou, Bolei},
  booktitle    = {2023 IEEE International Conference on Robotics and Automation (ICRA)},
  pages        = {3567--3575},
  year         = {2023},
  organization = {IEEE}
}

@inproceedings{feng2023trafficgen,
  title        = {Trafficgen: Learning to generate diverse and realistic traffic scenarios},
  author       = {Feng, Lan and Li, Quanyi and Peng, Zhenghao and Tan, Shuhan and Zhou, Bolei},
  booktitle    = {2023 IEEE International Conference on Robotics and Automation (ICRA)},
  pages        = {3567--3575},
  year         = {2023},
  organization = {IEEE}
}

@misc{frazier_tutorial_2018,
  title      = {A Tutorial on Bayesian Optimization},
  url        = {http://arxiv.org/abs/1807.02811},
  abstract   = {Bayesian optimization is an approach to optimizing objective functions that take a long time (minutes or hours) to evaluate. It is best-suited for optimization over continuous domains of less than 20 dimensions, and tolerates stochastic noise in function evaluations. It builds a surrogate for the objective and quantifies the uncertainty in that surrogate using a Bayesian machine learning technique, Gaussian process regression, and then uses an acquisition function defined from this surrogate to decide where to sample. In this tutorial, we describe how Bayesian optimization works, including Gaussian process regression and three common acquisition functions: expected improvement, entropy search, and knowledge gradient. We then discuss more advanced techniques, including running multiple function evaluations in parallel, multi-fidelity and multi-information source optimization, expensive-to-evaluate constraints, random environmental conditions, multi-task Bayesian optimization, and the inclusion of derivative information. We conclude with a discussion of Bayesian optimization software and future research directions in the field. Within our tutorial material we provide a generalization of expected improvement to noisy evaluations, beyond the noise-free setting where it is more commonly applied. This generalization is justified by a formal decision-theoretic argument, standing in contrast to previous ad hoc modifications.},
  number     = {{arXiv}:1807.02811},
  publisher  = {{arXiv}},
  author     = {Frazier, Peter I.},
  urldate    = {2024-10-02},
  date       = {2018-07-08},
  langid     = {english},
  eprinttype = {arxiv},
  eprint     = {1807.02811 [cs, math, stat]},
  keywords   = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
  file       = {PDF:/home/olek/Zotero/storage/IJ2HSBJJ/Frazier - 2018 - A Tutorial on Bayesian Optimization.pdf:application/pdf}
}

@inproceedings{gambi_generating_2019,
  location  = {New York, {NY}, {USA}},
  title     = {Generating effective test cases for self-driving cars from police reports},
  isbn      = {978-1-4503-5572-8},
  url       = {https://dl.acm.org/doi/10.1145/3338906.3338942},
  doi       = {10.1145/3338906.3338942},
  series    = {{ESEC}/{FSE} 2019},
  abstract  = {Autonomous driving carries the promise to drastically reduce the number of car accidents; however, recently reported fatal crashes involving self-driving cars show that such an important goal is not yet achieved. This calls for better testing of the software controlling self-driving cars, which is difficult because it requires producing challenging driving scenarios. To better test self-driving car soft- ware, we propose to specifically test car crash scenarios, which are critical par excellence. Since real car crashes are difficult to test in field operation, we recreate them as physically accurate simulations in an environment that can be used for testing self-driving car software. To cope with the scarcity of sensory data collected during real car crashes which does not enable a full reproduction, we extract the information to recreate real car crashes from the police reports which document them. Our extensive evaluation, consisting of a user study involving 34 participants and a quantitative analysis of the quality of the generated tests, shows that we can generate accurate simulations of car crashes in a matter of minutes. Compared to tests which implement non critical driving scenarios, our tests effectively stressed the test subject in different ways and exposed several shortcomings in its implementation.},
  pages     = {257--267},
  booktitle = {Proceedings of the 2019 27th {ACM} Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering},
  publisher = {Association for Computing Machinery},
  author    = {Gambi, Alessio and Huynh, Tri and Fraser, Gordon},
  urldate   = {2023-11-10},
  date      = {2019-08-12},
  year      = {2019},
  keywords  = {self-driving cars, automatic test generation, natural language processing, procedural content generation},
  file      = {Full Text PDF:/home/olek/Zotero/storage/B5EP2UM4/Gambi et al. - 2019 - Generating effective test cases for self-driving c.pdf:application/pdf}
}

@inproceedings{gambi_generating_2019,
  location  = {New York, {NY}, {USA}},
  title     = {Generating effective test cases for self-driving cars from police reports},
  isbn      = {978-1-4503-5572-8},
  url       = {https://dl.acm.org/doi/10.1145/3338906.3338942},
  doi       = {10.1145/3338906.3338942},
  series    = {{ESEC}/{FSE} 2019},
  abstract  = {Autonomous driving carries the promise to drastically reduce the number of car accidents; however, recently reported fatal crashes involving self-driving cars show that such an important goal is not yet achieved. This calls for better testing of the software controlling self-driving cars, which is difficult because it requires producing challenging driving scenarios. To better test self-driving car soft- ware, we propose to specifically test car crash scenarios, which are critical par excellence. Since real car crashes are difficult to test in field operation, we recreate them as physically accurate simulations in an environment that can be used for testing self-driving car software. To cope with the scarcity of sensory data collected during real car crashes which does not enable a full reproduction, we extract the information to recreate real car crashes from the police reports which document them. Our extensive evaluation, consisting of a user study involving 34 participants and a quantitative analysis of the quality of the generated tests, shows that we can generate accurate simulations of car crashes in a matter of minutes. Compared to tests which implement non critical driving scenarios, our tests effectively stressed the test subject in different ways and exposed several shortcomings in its implementation.},
  pages     = {257--267},
  booktitle = {Proceedings of the 2019 27th {ACM} Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering},
  publisher = {Association for Computing Machinery},
  author    = {Gambi, Alessio and Huynh, Tri and Fraser, Gordon},
  urldate   = {2023-11-10},
  date      = {2019-08-12},
  year      = {2019},
  keywords  = {self-driving cars, automatic test generation, natural language processing, procedural content generation},
  file      = {Full Text PDF:/home/olek/Zotero/storage/B5EP2UM4/Gambi et al. - 2019 - Generating effective test cases for self-driving c.pdf:application/pdf}
}


@inproceedings{gambi_generating_2022,
  location   = {Newark, {CA}, {USA}},
  title      = {Generating Critical Driving Scenarios from Accident Sketches},
  isbn       = {978-1-66548-737-5},
  url        = {https://ieeexplore.ieee.org/document/9898136/},
  doi        = {10.1109/AITest55621.2022.00022},
  abstract   = {Artificial Intelligence ({AI}) technologies are increasingly deployed to perform safety-critical tasks in various systems, including driverless vehicles. Therefore, ensuring the high quality of these {AI}-based systems is paramount to avoiding accidents and fatalities. Software testing has been proven to be a costeffective quality assurance method for traditional software systems but requires adaptations to address the peculiarities of {AI} applications. For instance, thoroughly testing the software controlling autonomous vehicles requires the definition of relevant driving scenarios and their implementation in physically accurate driving simulators, which remains an open challenge. Recent work showed that simulations of critical driving scenarios such as car crashes are fundamental to generating effective test cases. However, generating such complex simulations is challenging, and state-of-the-art approaches based on natural language descriptions struggle to complete the task. Therefore, we propose {CRISCE}, the first approach to create accurate car crash simulations from accident sketches. Our extensive evaluation shows that {CRISCE} is efficient, effective, and generates accurate simulations, drastically improving state-of-art approaches based on natural language processing.},
  eventtitle = {2022 {IEEE} International Conference On Artificial Intelligence Testing ({AITest})},
  pages      = {95--102},
  booktitle  = {2022 {IEEE} International Conference On Artificial Intelligence Testing ({AITest})},
  publisher  = {{IEEE}},
  author     = {Gambi, Alessio and Nguyen, Vuong and Ahmed, Jasim and Fraser, Gordon},
  urldate    = {2023-11-16},
  date       = {2022-08},
  year       = {2022},
  langid     = {english},
  file       = {Gambi et al. - 2022 - Generating Critical Driving Scenarios from Acciden.pdf:/home/olek/Zotero/storage/DCS4GF8A/Gambi et al. - 2022 - Generating Critical Driving Scenarios from Acciden.pdf:application/pdf}
}

@inproceedings{gambi_generating_2022,
  location   = {Newark, {CA}, {USA}},
  title      = {Generating Critical Driving Scenarios from Accident Sketches},
  isbn       = {978-1-66548-737-5},
  url        = {https://ieeexplore.ieee.org/document/9898136/},
  doi        = {10.1109/AITest55621.2022.00022},
  abstract   = {Artificial Intelligence ({AI}) technologies are increasingly deployed to perform safety-critical tasks in various systems, including driverless vehicles. Therefore, ensuring the high quality of these {AI}-based systems is paramount to avoiding accidents and fatalities. Software testing has been proven to be a costeffective quality assurance method for traditional software systems but requires adaptations to address the peculiarities of {AI} applications. For instance, thoroughly testing the software controlling autonomous vehicles requires the definition of relevant driving scenarios and their implementation in physically accurate driving simulators, which remains an open challenge. Recent work showed that simulations of critical driving scenarios such as car crashes are fundamental to generating effective test cases. However, generating such complex simulations is challenging, and state-of-the-art approaches based on natural language descriptions struggle to complete the task. Therefore, we propose {CRISCE}, the first approach to create accurate car crash simulations from accident sketches. Our extensive evaluation shows that {CRISCE} is efficient, effective, and generates accurate simulations, drastically improving state-of-art approaches based on natural language processing.},
  eventtitle = {2022 {IEEE} International Conference On Artificial Intelligence Testing ({AITest})},
  pages      = {95--102},
  booktitle  = {2022 {IEEE} International Conference On Artificial Intelligence Testing ({AITest})},
  publisher  = {{IEEE}},
  author     = {Gambi, Alessio and Nguyen, Vuong and Ahmed, Jasim and Fraser, Gordon},
  urldate    = {2023-11-16},
  date       = {2022-08},
  year       = {2022},
  langid     = {english},
  file       = {Gambi et al. - 2022 - Generating Critical Driving Scenarios from Acciden.pdf:/home/olek/Zotero/storage/DCS4GF8A/Gambi et al. - 2022 - Generating Critical Driving Scenarios from Acciden.pdf:application/pdf}
}

@inproceedings{gambi2019automatically,
  title     = {Automatically testing self-driving cars with search-based procedural content generation},
  author    = {Gambi, Alessio and Mueller, Marc and Fraser, Gordon},
  booktitle = {Proceedings of the 28th ACM SIGSOFT International Symposium on Software Testing and Analysis},
  pages     = {318--328},
  year      = {2019}
}


@inproceedings{gangopadhyay2019identification,
  title        = {Identification of test cases for automated driving systems using bayesian optimization},
  author       = {Gangopadhyay, Briti and Khastgir, Siddartha and Dey, Sumanta and Dasgupta, Pallab and Montana, Giovanni and Jennings, Paul},
  booktitle    = {2019 IEEE Intelligent Transportation Systems Conference (ITSC)},
  pages        = {1961--1967},
  year         = {2019},
  organization = {IEEE}
}


@article{gao_scenario_2023,
  title        = {Scenario Generation for Autonomous Vehicles with Deep-Learning-Based Heterogeneous Driver Models: Implementation and Verification},
  volume       = {23},
  rights       = {http://creativecommons.org/licenses/by/3.0/},
  issn         = {1424-8220},
  url          = {https://www.mdpi.com/1424-8220/23/9/4570},
  doi          = {10.3390/s23094570},
  shorttitle   = {Scenario Generation for Autonomous Vehicles with Deep-Learning-Based Heterogeneous Driver Models},
  abstract     = {Virtual testing requires hazardous scenarios to effectively test autonomous vehicles ({AVs}). Existing studies have obtained rarer events by sampling methods in a fixed scenario space. In reality, heterogeneous drivers behave differently when facing the same situation. To generate more realistic and efficient scenarios, we propose a two-stage heterogeneous driver model to change the number of dangerous scenarios in the scenario space. We trained the driver model using the {HighD} dataset, and generated scenarios through simulation. Simulations were conducted in 20 experimental groups with heterogeneous driver models and 5 control groups with the original driver model. The results show that, by adjusting the number and position of aggressive drivers, the percentage of dangerous scenarios was significantly higher compared to that of models not accounting for driver heterogeneity. To further verify the effectiveness of our method, we evaluated two driving strategies: car-following and cut-in scenarios. The results verify the effectiveness of our approach. Cumulatively, the results indicate that our approach could accelerate the testing of {AVs}.},
  pages        = {4570},
  number       = {9},
  journaltitle = {Sensors},
  author       = {Gao, Li and Zhou, Rui and Zhang, Kai},
  urldate      = {2024-01-12},
  date         = {2023-01},
  langid       = {english},
  note         = {Number: 9
                  Publisher: Multidisciplinary Digital Publishing Institute},
  keywords     = {autonomous-driving testing, deep learning, heterogeneous driver model, scenario generation},
  file         = {Full Text PDF:/home/olek/Zotero/storage/2UINHMSX/Gao et al. - 2023 - Scenario Generation for Autonomous Vehicles with D.pdf:application/pdf}
}


@article{gao_scenario_2023,
  title        = {Scenario Generation for Autonomous Vehicles with Deep-Learning-Based Heterogeneous Driver Models: Implementation and Verification},
  volume       = {23},
  rights       = {http://creativecommons.org/licenses/by/3.0/},
  issn         = {1424-8220},
  url          = {https://www.mdpi.com/1424-8220/23/9/4570},
  doi          = {10.3390/s23094570},
  shorttitle   = {Scenario Generation for Autonomous Vehicles with Deep-Learning-Based Heterogeneous Driver Models},
  abstract     = {Virtual testing requires hazardous scenarios to effectively test autonomous vehicles ({AVs}). Existing studies have obtained rarer events by sampling methods in a fixed scenario space. In reality, heterogeneous drivers behave differently when facing the same situation. To generate more realistic and efficient scenarios, we propose a two-stage heterogeneous driver model to change the number of dangerous scenarios in the scenario space. We trained the driver model using the {HighD} dataset, and generated scenarios through simulation. Simulations were conducted in 20 experimental groups with heterogeneous driver models and 5 control groups with the original driver model. The results show that, by adjusting the number and position of aggressive drivers, the percentage of dangerous scenarios was significantly higher compared to that of models not accounting for driver heterogeneity. To further verify the effectiveness of our method, we evaluated two driving strategies: car-following and cut-in scenarios. The results verify the effectiveness of our approach. Cumulatively, the results indicate that our approach could accelerate the testing of {AVs}.},
  pages        = {4570},
  number       = {9},
  journaltitle = {Sensors},
  author       = {Gao, Li and Zhou, Rui and Zhang, Kai},
  urldate      = {2024-01-12},
  date         = {2023-01},
  langid       = {english},
  note         = {Number: 9
                  Publisher: Multidisciplinary Digital Publishing Institute},
  keywords     = {autonomous-driving testing, deep learning, heterogeneous driver model, scenario generation},
  file         = {Full Text PDF:/home/olek/Zotero/storage/2UINHMSX/Gao et al. - 2023 - Scenario Generation for Autonomous Vehicles with D.pdf:application/pdf}
}

@inproceedings{gladisch2019experience,
  title        = {Experience paper: Search-based testing in automated driving control applications},
  author       = {Gladisch, Christoph and Heinz, Thomas and Heinzemann, Christian and Oehlerking, Jens and von Vietinghoff, Anne and Pfitzer, Tim},
  booktitle    = {2019 34th IEEE/ACM International Conference on Automated Software Engineering (ASE)},
  pages        = {26--37},
  year         = {2019},
  organization = {IEEE}
}

@article{godino2023review,
  title   = {Review of multi-fidelity models},
  journal = {ACSE},
  author  = {Fernández-Godino, M. Giselle},
  year    = {2023}
}

@inproceedings{gruber2024automatic,
  title     = {Do Automatic Test Generation Tools Generate Flaky Tests?},
  author    = {Gruber, Martin and Roslan, Muhammad Firhard and Parry, Owain and Scharnb{\"o}ck, Fabian and McMinn, Phil and Fraser, Gordon},
  booktitle = {Proceedings of the 46th IEEE/ACM International Conference on Software Engineering},
  pages     = {1--12},
  year      = {2024}
}

@misc{gulino2023waymax,
  title         = {Waymax: An Accelerated, Data-Driven Simulator for Large-Scale Autonomous Driving Research},
  author        = {Cole Gulino and Justin Fu and Wenjie Luo and George Tucker and Eli Bronstein and Yiren Lu and Jean Harb and Xinlei Pan and Yan Wang and Xiangyu Chen and John D. Co-Reyes and Rishabh Agarwal and Rebecca Roelofs and Yao Lu and Nico Montali and Paul Mougin and Zoey Yang and Brandyn White and Aleksandra Faust and Rowan McAllister and Dragomir Anguelov and Benjamin Sapp},
  year          = {2023},
  eprint        = {2310.08710},
  archiveprefix = {arXiv},
  primaryclass  = {cs.RO}
}

@inproceedings{haq2022efficient,
  title     = {Efficient online testing for DNN-enabled systems using surrogate-assisted and many-objective optimization},
  author    = {Haq, Fitash Ul and Shin, Donghwan and Briand, Lionel},
  booktitle = {Proceedings of the 44th international conference on software engineering},
  pages     = {811--822},
  year      = {2022}
}

@inproceedings{haq2023many,
  title        = {Many-objective reinforcement learning for online testing of dnn-enabled systems},
  author       = {Haq, Fitash Ul and Shin, Donghwan and Briand, Lionel C},
  booktitle    = {2023 IEEE/ACM 45th International Conference on Software Engineering (ICSE)},
  pages        = {1814--1826},
  year         = {2023},
  organization = {IEEE}
}


@incollection{harman2008search,
  title     = {Search based software engineering: Techniques, taxonomy, tutorial},
  author    = {Harman, Mark and McMinn, Phil and De Souza, Jerffeson Teixeira and Yoo, Shin},
  booktitle = {LASER Summer School on Software Engineering},
  pages     = {1--59},
  year      = {2008},
  publisher = {Springer}
}



@online{hawkins_teslas_2024,
  title      = {Tesla’s Autopilot and Full Self-Driving linked to hundreds of crashes, dozens of deaths},
  url        = {https://www.theverge.com/2024/4/26/24141361/tesla-autopilot-fsd-nhtsa-investigation-report-crash-death},
  abstract   = {When it was time to react, often it was too late.},
  titleaddon = {The Verge},
  author     = {Hawkins, Andrew J.},
  urldate    = {2024-09-24},
  date       = {2024-04-26},
  langid     = {english},
  file       = {Snapshot:/home/olek/Zotero/storage/87TII7IT/tesla-autopilot-fsd-nhtsa-investigation-report-crash-death.html:text/html}
}

@article{hayward1972near,
  title     = {Near miss determination through use of a scale of danger},
  author    = {Hayward, John C},
  year      = {1972},
  publisher = {Pennsylvania State University University Park}
}

@article{hayward1972near,
  title     = {Near miss determination through use of a scale of danger},
  author    = {Hayward, John C},
  year      = {1972},
  publisher = {Pennsylvania State University University Park}
}

@inproceedings{he2017mask,
  title     = {Mask r-cnn},
  author    = {He, Kaiming and Gkioxari, Georgia and Doll{\'a}r, Piotr and Girshick, Ross},
  booktitle = {Proceedings of the IEEE international conference on computer vision},
  pages     = {2961--2969},
  year      = {2017}
}

@inproceedings{he2017mask,
  title     = {Mask r-cnn},
  author    = {He, Kaiming and Gkioxari, Georgia and Doll{\'a}r, Piotr and Girshick, Ross},
  booktitle = {Proceedings of the IEEE international conference on computer vision},
  pages     = {2961--2969},
  year      = {2017}
}

@inproceedings{hiddenBiases,
  title     = {Hidden Biases of End-to-End Driving Models},
  author    = {Bernhard Jaeger and Kashyap Chitta and Andreas Geiger},
  booktitle = {Proc. of the IEEE International Conf. on Computer Vision (ICCV)},
  year      = {2023}
}
@inproceedings{hiddenBiases,
  title     = {Hidden Biases of End-to-End Driving Models},
  author    = {Bernhard Jaeger and Kashyap Chitta and Andreas Geiger},
  booktitle = {Proc. of the IEEE International Conf. on Computer Vision (ICCV)},
  year      = {2023}
}

@inproceedings{highDdataset,
  title     = {The highD Dataset: A Drone Dataset of Naturalistic Vehicle Trajectories on German Highways for Validation of Highly Automated Driving Systems},
  author    = {Krajewski, Robert and Bock, Julian and Kloeker, Laurent and Eckstein, Lutz},
  booktitle = {2018 21st International Conference on Intelligent Transportation Systems (ITSC)},
  pages     = {2118-2125},
  year      = {2018},
  doi       = {10.1109/ITSC.2018.8569552}
}

@inproceedings{highDdataset,
  title     = {The highD Dataset: A Drone Dataset of Naturalistic Vehicle Trajectories on German Highways for Validation of Highly Automated Driving Systems},
  author    = {Krajewski, Robert and Bock, Julian and Kloeker, Laurent and Eckstein, Lutz},
  booktitle = {2018 21st International Conference on Intelligent Transportation Systems (ITSC)},
  pages     = {2118-2125},
  year      = {2018},
  doi       = {10.1109/ITSC.2018.8569552}
}

@misc{highway-env,
  author       = {Leurent, Edouard},
  title        = {An Environment for Autonomous Driving Decision-Making},
  year         = {2018},
  publisher    = {GitHub},
  journal      = {GitHub repository},
  howpublished = {\url{https://github.com/eleurent/highway-env}}
}

@misc{highway-env,
  author       = {Leurent, Edouard},
  title        = {An Environment for Autonomous Driving Decision-Making},
  year         = {2018},
  publisher    = {GitHub},
  journal      = {GitHub repository},
  howpublished = {\url{https://github.com/eleurent/highway-env}}
}

@inproceedings{hildebrandt_physcov_2023,
  location   = {New York, {NY}, {USA}},
  title      = {{PhysCov}: Physical Test Coverage for Autonomous Vehicles},
  isbn       = {9798400702211},
  url        = {https://dl.acm.org/doi/10.1145/3597926.3598069},
  doi        = {10.1145/3597926.3598069},
  series     = {{ISSTA} 2023},
  shorttitle = {{PhysCov}},
  abstract   = {Adequately exercising the behaviors of autonomous vehicles is fundamental to their validation. However, quantifying an autonomous vehicle’s testing adequacy is challenging as the system’s behavior is influenced both by its state as well as its physical environment. To address this challenge, our work builds on two insights. First, data sensed by an autonomous vehicle provides a unique spatial signature of the physical environment inputs. Second, given the vehicle’s current state, inputs residing outside the autonomous vehicle’s physically reachable regions are less relevant to its behavior. Building on those insights, we introduce an abstraction that enables the computation of a physical environment-state coverage metric, {PhysCov}. The abstraction combines the sensor readings with a physical reachability analysis based on the vehicle’s state and dynamics to determine the region of the environment that may affect the autonomous vehicle. It then characterizes that region through a parameterizable geometric approximation that can trade quality for cost. Tests with the same characterizations are deemed to have had similar internal states and exposed to similar environments and thus likely to exercise the same set of behaviors, while tests with distinct characterizations will increase {PhysCov}. A study on two simulated and one real system’s dataset examines {PhysCovs}’s ability to quantify an autonomous vehicle’s test suite, showcases its characterization cost and precision, investigates its correlation with failures found and potential for test selection, and assesses its ability to distinguish among real-world scenarios.},
  pages      = {449--461},
  booktitle  = {Proceedings of the 32nd {ACM} {SIGSOFT} International Symposium on Software Testing and Analysis},
  publisher  = {Association for Computing Machinery},
  author     = {Hildebrandt, Carl and von Stein, Meriel and Elbaum, Sebastian},
  urldate    = {2023-12-05},
  date       = {2023-07-13},
  keywords   = {Autonomous Systems, Coverage Metrics, Test Adequacy},
  file       = {Full Text PDF:/home/olek/Zotero/storage/G6GMIKX7/Hildebrandt et al. - 2023 - PhysCov Physical Test Coverage for Autonomous Veh.pdf:application/pdf}
}

@inproceedings{hildebrandt_physcov_2023,
  location   = {New York, {NY}, {USA}},
  title      = {{PhysCov}: Physical Test Coverage for Autonomous Vehicles},
  isbn       = {9798400702211},
  url        = {https://dl.acm.org/doi/10.1145/3597926.3598069},
  doi        = {10.1145/3597926.3598069},
  series     = {{ISSTA} 2023},
  shorttitle = {{PhysCov}},
  abstract   = {Adequately exercising the behaviors of autonomous vehicles is fundamental to their validation. However, quantifying an autonomous vehicle’s testing adequacy is challenging as the system’s behavior is influenced both by its state as well as its physical environment. To address this challenge, our work builds on two insights. First, data sensed by an autonomous vehicle provides a unique spatial signature of the physical environment inputs. Second, given the vehicle’s current state, inputs residing outside the autonomous vehicle’s physically reachable regions are less relevant to its behavior. Building on those insights, we introduce an abstraction that enables the computation of a physical environment-state coverage metric, {PhysCov}. The abstraction combines the sensor readings with a physical reachability analysis based on the vehicle’s state and dynamics to determine the region of the environment that may affect the autonomous vehicle. It then characterizes that region through a parameterizable geometric approximation that can trade quality for cost. Tests with the same characterizations are deemed to have had similar internal states and exposed to similar environments and thus likely to exercise the same set of behaviors, while tests with distinct characterizations will increase {PhysCov}. A study on two simulated and one real system’s dataset examines {PhysCovs}’s ability to quantify an autonomous vehicle’s test suite, showcases its characterization cost and precision, investigates its correlation with failures found and potential for test selection, and assesses its ability to distinguish among real-world scenarios.},
  pages      = {449--461},
  booktitle  = {Proceedings of the 32nd {ACM} {SIGSOFT} International Symposium on Software Testing and Analysis},
  publisher  = {Association for Computing Machinery},
  author     = {Hildebrandt, Carl and von Stein, Meriel and Elbaum, Sebastian},
  urldate    = {2023-12-05},
  date       = {2023-07-13},
  keywords   = {Autonomous Systems, Coverage Metrics, Test Adequacy},
  file       = {Full Text PDF:/home/olek/Zotero/storage/G6GMIKX7/Hildebrandt et al. - 2023 - PhysCov Physical Test Coverage for Autonomous Veh.pdf:application/pdf}
}

@article{Hu2024-survey,
  author   = {Hu, Xuemin and Li, Shen and Huang, Tingyu and Tang, Bo and Huai, Rouxing and Chen, Long},
  journal  = {IEEE Transactions on Intelligent Vehicles},
  title    = {How Simulation Helps Autonomous Driving: A Survey of Sim2real, Digital Twins, and Parallel Intelligence},
  year     = {2024},
  volume   = {9},
  number   = {1},
  pages    = {593-612},
  keywords = {Autonomous vehicles;Digital twins;Task analysis;Metalearning;Virtual reality;Vehicle dynamics;Solid modeling;Autonomous driving;sim2real;digital twins;parallel intelligence;reality gap},
  doi      = {10.1109/TIV.2023.3312777}
}


@article{huang2006sequential,
  title     = {Sequential kriging optimization using multiple-fidelity evaluations},
  author    = {Huang, Deng and Allen, Theodore T and Notz, William I and Miller, R Allen},
  journal   = {Structural and Multidisciplinary Optimization},
  volume    = {32},
  pages     = {369--382},
  year      = {2006},
  publisher = {Springer}
}

@inproceedings{hungar_concept_2020,
  location  = {Cham},
  title     = {A Concept of Scenario Space Exploration with Criticality Coverage Guarantees},
  isbn      = {978-3-030-61467-6},
  doi       = {10.1007/978-3-030-61467-6_19},
  series    = {Lecture Notes in Computer Science},
  abstract  = {Assuring the safety of an automated driving system is difficult, because a large, heterogeneous set of traffic situations has to be handled by the system. Systematic testing of the full system at the end of the development seems necessary to be able to reach the required level of assurance. In our approach, the set of potentially relevant, concrete test cases result by parameter instantiation from finitely many more abstract, so called logical scenarios. For nearly all interesting automation systems, even virtual testing via simulation can cover only a tiny fraction of this set of concrete test cases.},
  pages     = {293--306},
  booktitle = {Leveraging Applications of Formal Methods, Verification and Validation: Applications},
  publisher = {Springer International Publishing},
  author    = {Hungar, Hardi},
  editor    = {Margaria, Tiziana and Steffen, Bernhard},
  date      = {2020},
  langid    = {english},
  keywords  = {Automated driving system, Safety testing, Scenario exploration, Simulation},
  file      = {Full Text PDF:/home/olek/Zotero/storage/G6Y4GVUH/Hungar - 2020 - A Concept of Scenario Space Exploration with Criti.pdf:application/pdf}
}

@inproceedings{hungar_concept_2020,
  location  = {Cham},
  title     = {A Concept of Scenario Space Exploration with Criticality Coverage Guarantees},
  isbn      = {978-3-030-61467-6},
  doi       = {10.1007/978-3-030-61467-6_19},
  series    = {Lecture Notes in Computer Science},
  abstract  = {Assuring the safety of an automated driving system is difficult, because a large, heterogeneous set of traffic situations has to be handled by the system. Systematic testing of the full system at the end of the development seems necessary to be able to reach the required level of assurance. In our approach, the set of potentially relevant, concrete test cases result by parameter instantiation from finitely many more abstract, so called logical scenarios. For nearly all interesting automation systems, even virtual testing via simulation can cover only a tiny fraction of this set of concrete test cases.},
  pages     = {293--306},
  booktitle = {Leveraging Applications of Formal Methods, Verification and Validation: Applications},
  publisher = {Springer International Publishing},
  author    = {Hungar, Hardi},
  editor    = {Margaria, Tiziana and Steffen, Bernhard},
  date      = {2020},
  langid    = {english},
  keywords  = {Automated driving system, Safety testing, Scenario exploration, Simulation},
  file      = {Full Text PDF:/home/olek/Zotero/storage/G6Y4GVUH/Hungar - 2020 - A Concept of Scenario Space Exploration with Criti.pdf:application/pdf}
}

@inproceedings{itra2021,
  author    = {\'Scibior, Adam and Lioutas, Vasileios and Reda, Daniele and Bateni, Peyman and Wood, Frank},
  booktitle = {2021 IEEE International Intelligent Transportation Systems Conference (ITSC)},
  title     = {Imagining The Road Ahead: Multi-Agent Trajectory Prediction via Differentiable Simulation},
  year      = {2021},
  pages     = {720-725},
  doi       = {10.1109/ITSC48978.2021.9565113}
}

@inproceedings{itra2021,
  author    = {\'Scibior, Adam and Lioutas, Vasileios and Reda, Daniele and Bateni, Peyman and Wood, Frank},
  booktitle = {2021 IEEE International Intelligent Transportation Systems Conference (ITSC)},
  title     = {Imagining The Road Ahead: Multi-Agent Trajectory Prediction via Differentiable Simulation},
  year      = {2021},
  pages     = {720-725},
  doi       = {10.1109/ITSC48978.2021.9565113}
}

@inproceedings{jaeger2023hidden,
  title     = {Hidden biases of end-to-end driving models},
  author    = {Jaeger, Bernhard and Chitta, Kashyap and Geiger, Andreas},
  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages     = {8240--8249},
  year      = {2023}
}

@inproceedings{Jaeger2023ICCV,
  author    = {Jaeger, Bernhard and Chitta, Kashyap and Geiger, Andreas},
  title     = {Hidden Biases of End-to-End Driving Models},
  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
  month     = {October},
  year      = {2023},
  pages     = {8240-8249}
}

@inproceedings{Jaeger2023ICCV,
  author    = {Jaeger, Bernhard and Chitta, Kashyap and Geiger, Andreas},
  title     = {Hidden Biases of End-to-End Driving Models},
  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
  month     = {October},
  year      = {2023},
  pages     = {8240-8249}
}

@article{jia2024bench2drive,
  title   = {Bench2drive: Towards multi-ability benchmarking of closed-loop end-to-end autonomous driving},
  author  = {Jia, Xiaosong and Yang, Zhenjie and Li, Qifeng and Zhang, Zhiyuan and Yan, Junchi},
  journal = {Advances in Neural Information Processing Systems},
  volume  = {37},
  pages   = {819--844},
  year    = {2024}
}

@article{jodat2025automated,
  title   = {Automated Test Oracles for Flaky Cyber-Physical System Simulators: Approach and Evaluation},
  author  = {Jodat, Baharin A and Gaaloul, Khouloud and Sabetzadeh, Mehrdad and Nejati, Shiva},
  journal = {arXiv e-prints},
  pages   = {arXiv--2508},
  year    = {2025}
}

@article{jodat2026automated,
  title     = {Automated Test Validators for Flaky Cyber-Physical System Simulators: Approach and Evaluation},
  author    = {Jodat, Baharin A and Gaaloul, Khouloud and Sabetzadeh, Mehrdad and Nejati, Shiva},
  journal   = {IEEE Transactions on Software Engineering},
  year      = {2026},
  publisher = {IEEE}
}

@article{kalra2016driving,
  title     = {Driving to safety: How many miles of driving would it take to demonstrate autonomous vehicle reliability?},
  author    = {Kalra, Nidhi and Paddock, Susan M},
  journal   = {Transportation Research Part A: Policy and Practice},
  volume    = {94},
  pages     = {182--193},
  year      = {2016},
  publisher = {Elsevier}
}

@article{kalra2016driving,
  title     = {Driving to safety: How many miles of driving would it take to demonstrate autonomous vehicle reliability?},
  author    = {Kalra, Nidhi and Paddock, Susan M},
  journal   = {Transportation Research Part A: Policy and Practice},
  volume    = {94},
  pages     = {182--193},
  year      = {2016},
  publisher = {Elsevier}
}

@article{kandasamy2016gaussian,
  title   = {Gaussian process bandit optimisation with multi-fidelity evaluations},
  author  = {Kandasamy, Kirthevasan and Dasarathy, Gautam and Oliva, Junier B and Schneider, Jeff and P{\'o}czos, Barnab{\'a}s},
  journal = {Advances in neural information processing systems},
  volume  = {29},
  year    = {2016}
}

@inproceedings{kandasamy2017multi,
  title     = {Multi-fidelity {B}ayesian Optimisation with Continuous Approximations},
  author    = {Kirthevasan Kandasamy and Gautam Dasarathy and Jeff Schneider and Barnab{\'a}s P{\'o}czos},
  booktitle = {Proceedings of the 34th International Conference on Machine Learning},
  pages     = {1799--1808},
  year      = {2017},
  editor    = {Precup, Doina and Teh, Yee Whye},
  volume    = {70},
  series    = {Proceedings of Machine Learning Research},
  month     = {06--11 Aug},
  publisher = {PMLR},
  pdf       = {http://proceedings.mlr.press/v70/kandasamy17a/kandasamy17a.pdf},
  url       = {https://proceedings.mlr.press/v70/kandasamy17a.html},
  abstract  = {Bandit methods for black-box optimisation, such as Bayesian optimisation, are used in a variety of applications including hyper-parameter tuning and experiment design. Recently, <em>multi-fidelity</em> methods have garnered considerable attention since function evaluations have become increasingly expensive in such applications. Multi-fidelity methods use cheap approximations to the function of interest to speed up the overall optimisation process. However, most multi-fidelity methods assume only a finite number of approximations. On the other hand, in many practical applications, a continuous spectrum of approximations might be available. For instance, when tuning an expensive neural network, one might choose to approximate the cross validation performance using less data $N$ and/or few training iterations $T$. Here, the approximations are best viewed as arising out of a continuous two dimensional space $(N,T)$. In this work, we develop a Bayesian optimisation method, BOCA, for this setting. We characterise its theoretical properties and show that it achieves better regret than than strategies which ignore the approximations. BOCA outperforms several other baselines in synthetic and real experiments.}
}

@misc{karunakaran_automatic_2022,
  title      = {Automatic lane change scenario extraction and generation of scenarios in {OpenX} format from real-world data},
  url        = {http://arxiv.org/abs/2203.07521},
  doi        = {10.48550/arXiv.2203.07521},
  abstract   = {Autonomous Vehicles ({AV})'s wide-scale deployment appears imminent despite many safety challenges yet to be resolved. The modern autonomous vehicles will undoubtedly include machine learning and probabilistic techniques that add significant complexity to the traditional verification and validation methods. Road testing is essential before the deployment, but scenarios are repeatable, and it's hard to collect challenging events. Exploring numerous, diverse and crucial scenarios is a time-consuming and expensive approach. The research community and industry have widely accepted scenario-based testing in the last few years. As it is focused directly on the relevant critical road situations, it can reduce the effort required in testing. The scenario-based testing in simulation requires the realistic behaviour of the traffic participants to assess the System Under Test ({SUT}). It is essential to capture the scenarios from the real world to encode the behaviour of actual traffic participants. This paper proposes a novel scenario extraction method to capture the lane change scenarios using point-cloud data and object tracking information. This method enables fully automatic scenario extraction compared to similar approaches in this area. The generated scenarios are represented in {OpenX} format to reuse them in the {SUT} evaluation easily. The motivation of this framework is to build a validation dataset to generate many critical concrete scenarios. The code is available online at https://github.com/dkarunakaran/scenario\_extraction\_framework.},
  number     = {{arXiv}:2203.07521},
  publisher  = {{arXiv}},
  author     = {Karunakaran, Dhanoop and Berrio, Julie Stephany and Worrall, Stewart and Nebot, Eduardo},
  urldate    = {2024-01-12},
  date       = {2022-03-14},
  eprinttype = {arxiv},
  eprint     = {2203.07521 [cs]},
  keywords   = {Computer Science - Robotics, I.6.4},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/6NBCYYGK/Karunakaran et al. - 2022 - Automatic lane change scenario extraction and gene.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/IN3NA4G7/2203.html:text/html}
}

@misc{karunakaran_automatic_2022,
  title      = {Automatic lane change scenario extraction and generation of scenarios in {OpenX} format from real-world data},
  url        = {http://arxiv.org/abs/2203.07521},
  doi        = {10.48550/arXiv.2203.07521},
  abstract   = {Autonomous Vehicles ({AV})'s wide-scale deployment appears imminent despite many safety challenges yet to be resolved. The modern autonomous vehicles will undoubtedly include machine learning and probabilistic techniques that add significant complexity to the traditional verification and validation methods. Road testing is essential before the deployment, but scenarios are repeatable, and it's hard to collect challenging events. Exploring numerous, diverse and crucial scenarios is a time-consuming and expensive approach. The research community and industry have widely accepted scenario-based testing in the last few years. As it is focused directly on the relevant critical road situations, it can reduce the effort required in testing. The scenario-based testing in simulation requires the realistic behaviour of the traffic participants to assess the System Under Test ({SUT}). It is essential to capture the scenarios from the real world to encode the behaviour of actual traffic participants. This paper proposes a novel scenario extraction method to capture the lane change scenarios using point-cloud data and object tracking information. This method enables fully automatic scenario extraction compared to similar approaches in this area. The generated scenarios are represented in {OpenX} format to reuse them in the {SUT} evaluation easily. The motivation of this framework is to build a validation dataset to generate many critical concrete scenarios. The code is available online at https://github.com/dkarunakaran/scenario\_extraction\_framework.},
  number     = {{arXiv}:2203.07521},
  publisher  = {{arXiv}},
  author     = {Karunakaran, Dhanoop and Berrio, Julie Stephany and Worrall, Stewart and Nebot, Eduardo},
  urldate    = {2024-01-12},
  date       = {2022-03-14},
  eprinttype = {arxiv},
  eprint     = {2203.07521 [cs]},
  keywords   = {Computer Science - Robotics, I.6.4},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/6NBCYYGK/Karunakaran et al. - 2022 - Automatic lane change scenario extraction and gene.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/IN3NA4G7/2203.html:text/html}
}

@inproceedings{kerber2020clustering,
  title        = {Clustering of the scenario space for the assessment of automated driving},
  author       = {Kerber, Jonas and Wagner, Sebastian and Groh, Korbinian and Notz, Dominik and K{\"u}hbeck, Thomas and Watzenig, Daniel and Knoll, Alois},
  booktitle    = {2020 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {578--583},
  year         = {2020},
  organization = {IEEE}
}

@inproceedings{kerber2020clustering,
  title        = {Clustering of the scenario space for the assessment of automated driving},
  author       = {Kerber, Jonas and Wagner, Sebastian and Groh, Korbinian and Notz, Dominik and K{\"u}hbeck, Thomas and Watzenig, Daniel and Knoll, Alois},
  booktitle    = {2020 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {578--583},
  year         = {2020},
  organization = {IEEE}
}

@inproceedings{kim2019guiding,
  title        = {Guiding deep learning system testing using surprise adequacy},
  author       = {Kim, Jinhan and Feldt, Robert and Yoo, Shin},
  booktitle    = {2019 IEEE/ACM 41st International Conference on Software Engineering (ICSE)},
  pages        = {1039--1049},
  year         = {2019},
  organization = {IEEE}
}

@inproceedings{kim2019guiding,
  title        = {Guiding deep learning system testing using surprise adequacy},
  author       = {Kim, Jinhan and Feldt, Robert and Yoo, Shin},
  booktitle    = {2019 IEEE/ACM 41st International Conference on Software Engineering (ICSE)},
  pages        = {1039--1049},
  year         = {2019},
  organization = {IEEE}
}

@article{kitchenham_guidelines_2007,
  title    = {Guidelines for performing Systematic Literature Reviews in Software Engineering},
  volume   = {2},
  abstract = {The objective of this report is to propose comprehensive guidelines for systematic literature reviews appropriate for software engineering researchers, including {PhD} students. A systematic literature review is a means of evaluating and interpreting all available research relevant to a particular research question, topic area, or phenomenon of interest. 
              Systematic reviews aim to present a fair evaluation of a research topic by using a trustworthy, rigorous, and auditable methodology. The guidelines presented in this report were derived from three existing guidelines used by medical researchers, two books produced by researchers with social science backgrounds and discussions with researchers from other disciplines who are involved
              in evidence-based practice. The guidelines have been adapted to reflect the specific problems of software engineering research.
              The guidelines cover three phases of a systematic literature review: planning the review, conducting the review and reporting the review. They provide a relatively high level description. They do not consider the impact of the research questions on the review procedures, nor do they specify in detail the mechanisms needed to perform meta-analysis.},
  author   = {Kitchenham, Barbara and Charters, Stuart},
  date     = {2007-01-01},
  file     = {Full Text PDF:/home/olek/Zotero/storage/VFFUPN9P/Kitchenham and Charters - 2007 - Guidelines for performing Systematic Literature Re.pdf:application/pdf}
}

@article{kitchenham_guidelines_2007,
  title    = {Guidelines for performing Systematic Literature Reviews in Software Engineering},
  volume   = {2},
  abstract = {The objective of this report is to propose comprehensive guidelines for systematic literature reviews appropriate for software engineering researchers, including {PhD} students. A systematic literature review is a means of evaluating and interpreting all available research relevant to a particular research question, topic area, or phenomenon of interest. 
              Systematic reviews aim to present a fair evaluation of a research topic by using a trustworthy, rigorous, and auditable methodology. The guidelines presented in this report were derived from three existing guidelines used by medical researchers, two books produced by researchers with social science backgrounds and discussions with researchers from other disciplines who are involved
              in evidence-based practice. The guidelines have been adapted to reflect the specific problems of software engineering research.
              The guidelines cover three phases of a systematic literature review: planning the review, conducting the review and reporting the review. They provide a relatively high level description. They do not consider the impact of the research questions on the review procedures, nor do they specify in detail the mechanisms needed to perform meta-analysis.},
  author   = {Kitchenham, Barbara and Charters, Stuart},
  date     = {2007-01-01},
  file     = {Full Text PDF:/home/olek/Zotero/storage/VFFUPN9P/Kitchenham and Charters - 2007 - Guidelines for performing Systematic Literature Re.pdf:application/pdf}
}

@inproceedings{klein_fast_2017,
  year       = {2017},
  title      = {Fast Bayesian Optimization of Machine Learning Hyperparameters on Large Datasets},
  url        = {https://proceedings.mlr.press/v54/klein17a.html},
  abstract   = {Bayesian optimization has become a successful tool for hyperparameter optimization of machine learning algorithms, such as support vector machines or deep neural networks. Despite its success, for large datasets, training and validating a single configuration often takes hours, days, or even weeks, which limits the achievable performance. To accelerate hyperparameter optimization, we propose a generative model for the validation error as a function of training set size, which is learned during the optimization process and allows exploration of preliminary configurations on small subsets, by extrapolating to the full dataset.  We construct a Bayesian optimization procedure, dubbed {FABOLAS}, which models loss and training time as a function of dataset size and automatically trades off high information gain about the global optimum against computational cost. Experiments optimizing support vector machines and deep neural networks show that {FABOLAS} often finds high-quality solutions 10 to 100 times faster than other state-of-the-art Bayesian optimization methods or the recently proposed bandit strategy Hyperband.},
  eventtitle = {Artificial Intelligence and Statistics},
  pages      = {528--536},
  booktitle  = {Proceedings of the 20th International Conference on Artificial Intelligence and Statistics},
  publisher  = {{PMLR}},
  author     = {Klein, Aaron and Falkner, Stefan and Bartels, Simon and Hennig, Philipp and Hutter, Frank},
  urldate    = {2024-10-02},
  date       = {2017-04-10},
  langid     = {english},
  note       = {{ISSN}: 2640-3498},
  file       = {Full Text PDF:/home/olek/Zotero/storage/NEUKP3GD/Klein et al. - 2017 - Fast Bayesian Optimization of Machine Learning Hyperparameters on Large Datasets.pdf:application/pdf;Supplementary PDF:/home/olek/Zotero/storage/EGHX5Q3F/Klein et al. - 2017 - Fast Bayesian Optimization of Machine Learning Hyperparameters on Large Datasets.pdf:application/pdf}
}


@inproceedings{klischat_synthesizing_2020,
  location   = {Las Vegas, {NV}, {USA}},
  title      = {Synthesizing Traffic Scenarios from Formal Specifications for Testing Automated Vehicles},
  isbn       = {978-1-72816-673-5},
  url        = {https://ieeexplore.ieee.org/document/9304617/},
  doi        = {10.1109/IV47402.2020.9304617},
  abstract   = {Virtual testing plays an important role in the validation and verification of automated vehicles. State-of-theart approaches first generate a huge amount of test scenarios through simulations or test drives, which are later filtered to obtain relevant scenarios for a given set of specifications. However, only few works exist on synthesizing scenarios directly from specifications. In this work, we present an optimizationbased approach to synthesize scenarios only from formal specifications and a given map. To concretize the specifications, we formulate predicates, which are subsequently converted to a mixed-integer quadratic optimization problem. We demonstrate how our method can generate scenarios for maps featuring merging lanes and intersections given a variety of specifications.},
  eventtitle = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {2065--2072},
  booktitle  = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  publisher  = {{IEEE}},
  author     = {Klischat, Moritz and Althoff, Matthias},
  urldate    = {2023-11-16},
  date       = {2020-10-19},
  langid     = {english},
  file       = {Klischat and Althoff - 2020 - Synthesizing Traffic Scenarios from Formal Specifi.pdf:/home/olek/Zotero/storage/RL8BVK9Z/Klischat and Althoff - 2020 - Synthesizing Traffic Scenarios from Formal Specifi.pdf:application/pdf}
}

@inproceedings{klischat_synthesizing_2020,
  location   = {Las Vegas, {NV}, {USA}},
  title      = {Synthesizing Traffic Scenarios from Formal Specifications for Testing Automated Vehicles},
  isbn       = {978-1-72816-673-5},
  url        = {https://ieeexplore.ieee.org/document/9304617/},
  doi        = {10.1109/IV47402.2020.9304617},
  abstract   = {Virtual testing plays an important role in the validation and verification of automated vehicles. State-of-theart approaches first generate a huge amount of test scenarios through simulations or test drives, which are later filtered to obtain relevant scenarios for a given set of specifications. However, only few works exist on synthesizing scenarios directly from specifications. In this work, we present an optimizationbased approach to synthesize scenarios only from formal specifications and a given map. To concretize the specifications, we formulate predicates, which are subsequently converted to a mixed-integer quadratic optimization problem. We demonstrate how our method can generate scenarios for maps featuring merging lanes and intersections given a variety of specifications.},
  eventtitle = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {2065--2072},
  booktitle  = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  publisher  = {{IEEE}},
  author     = {Klischat, Moritz and Althoff, Matthias},
  urldate    = {2023-11-16},
  date       = {2020-10-19},
  langid     = {english},
  file       = {Klischat and Althoff - 2020 - Synthesizing Traffic Scenarios from Formal Specifi.pdf:/home/olek/Zotero/storage/RL8BVK9Z/Klischat and Althoff - 2020 - Synthesizing Traffic Scenarios from Formal Specifi.pdf:application/pdf}
}

@inproceedings{klischat2019generating,
  title        = {Generating critical test scenarios for automated vehicles with evolutionary algorithms},
  author       = {Klischat, Moritz and Althoff, Matthias},
  booktitle    = {2019 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {2352--2358},
  year         = {2019},
  organization = {IEEE}
}

@article{kothari2021drivergym,
  title   = {Drivergym: Democratising reinforcement learning for autonomous driving},
  author  = {Kothari, Parth and Perone, Christian and Bergamini, Luca and Alahi, Alexandre and Ondruska, Peter},
  journal = {arXiv preprint arXiv:2111.06889},
  year    = {2021}
}

@article{kothari2021drivergym,
  title   = {Drivergym: Democratising reinforcement learning for autonomous driving},
  author  = {Kothari, Parth and Perone, Christian and Bergamini, Luca and Alahi, Alexandre and Ondruska, Peter},
  journal = {arXiv preprint arXiv:2111.06889},
  year    = {2021}
}

@article{kuang_assessing_2019,
  title        = {Assessing the Socioeconomic Impacts of Intelligent Connected Vehicles in China: A Cost–Benefit Analysis},
  volume       = {11},
  doi          = {10.3390/su11123273},
  shorttitle   = {Assessing the Socioeconomic Impacts of Intelligent Connected Vehicles in China},
  abstract     = {The deployment of intelligent connected vehicles ({ICVs}) is regarded as a significant solution to improve road safety, transportation management, and energy efficiency. This study assessed the safety, traffic, environmental, and industrial economic benefits of {ICV} deployment in China under different scenarios. A bottom-up model was established to deal with these impacts within a unified framework, based on the existing theories and literature of {ICVs}’ cost–benefit analysis, as well as China’s most recent policies and statistics. The results indicate that the total benefits may reach 13.25 to 24.02 trillion renminbi ({RMB}) in 2050, while a cumulative benefit–cost ratio of 1.15 to 3.06 suggests high cost-effectiveness. However, if the government and industry only focus on their own interests, the break-even point may be delayed by several years. Hence, an effective business model is necessary to enhance public–private cooperation in {ICV} implementation. Meanwhile, the savings of travel time costs and fleet labor costs play an important part in all socioeconomic impacts. Therefore, the future design of {ICVs} should pay more attention to the utilization of in-vehicle time and the real substitution for human drivers.},
  pages        = {3273},
  journaltitle = {Sustainability},
  shortjournal = {Sustainability},
  author       = {Kuang, Xu and Zhao, Fuquan and Hao, Han and Liu, Zongwei},
  date         = {2019-06-13},
  file         = {Full Text:/home/olek/Zotero/storage/62MYE2IX/Kuang et al. - 2019 - Assessing the Socioeconomic Impacts of Intelligent.pdf:application/pdf}
}

@article{kuang_assessing_2019,
  title        = {Assessing the Socioeconomic Impacts of Intelligent Connected Vehicles in China: A Cost–Benefit Analysis},
  volume       = {11},
  doi          = {10.3390/su11123273},
  shorttitle   = {Assessing the Socioeconomic Impacts of Intelligent Connected Vehicles in China},
  abstract     = {The deployment of intelligent connected vehicles ({ICVs}) is regarded as a significant solution to improve road safety, transportation management, and energy efficiency. This study assessed the safety, traffic, environmental, and industrial economic benefits of {ICV} deployment in China under different scenarios. A bottom-up model was established to deal with these impacts within a unified framework, based on the existing theories and literature of {ICVs}’ cost–benefit analysis, as well as China’s most recent policies and statistics. The results indicate that the total benefits may reach 13.25 to 24.02 trillion renminbi ({RMB}) in 2050, while a cumulative benefit–cost ratio of 1.15 to 3.06 suggests high cost-effectiveness. However, if the government and industry only focus on their own interests, the break-even point may be delayed by several years. Hence, an effective business model is necessary to enhance public–private cooperation in {ICV} implementation. Meanwhile, the savings of travel time costs and fleet labor costs play an important part in all socioeconomic impacts. Therefore, the future design of {ICVs} should pay more attention to the utilization of in-vehicle time and the real substitution for human drivers.},
  pages        = {3273},
  journaltitle = {Sustainability},
  shortjournal = {Sustainability},
  author       = {Kuang, Xu and Zhao, Fuquan and Hao, Han and Liu, Zongwei},
  date         = {2019-06-13},
  file         = {Full Text:/home/olek/Zotero/storage/62MYE2IX/Kuang et al. - 2019 - Assessing the Socioeconomic Impacts of Intelligent.pdf:application/pdf}
}

@inproceedings{kuutti2020training,
  title        = {Training adversarial agents to exploit weaknesses in deep control policies},
  author       = {Kuutti, Sampo and Fallah, Saber and Bowden, Richard},
  booktitle    = {2020 IEEE International Conference on Robotics and Automation (ICRA)},
  pages        = {108--114},
  year         = {2020},
  organization = {IEEE}
}


@inproceedings{kuutti2020training,
  title        = {Training adversarial agents to exploit weaknesses in deep control policies},
  author       = {Kuutti, Sampo and Fallah, Saber and Bowden, Richard},
  booktitle    = {2020 IEEE International Conference on Robotics and Automation (ICRA)},
  pages        = {108--114},
  year         = {2020},
  organization = {IEEE}
}

@inproceedings{li_multi_fidelity_2020,
  title     = {Multi-Fidelity Bayesian Optimization via Deep Neural Networks},
  volume    = {33},
  year      = {2020},
  url       = {https://proceedings.neurips.cc/paper_files/paper/2020/hash/60e1deb043af37db5ea4ce9ae8d2c9ea-Abstract.html},
  abstract  = {Bayesian optimization ({BO}) is a popular framework for optimizing black-box functions. In many applications,  the objective function can be evaluated at multiple fidelities to enable a trade-off between the cost and accuracy. To reduce the optimization cost, many multi-fidelity {BO} methods have been proposed. Despite their success, these methods either ignore or over-simplify the strong, complex correlations across the fidelities. While the acquisition function is therefore easy and convenient to calculate,  these methods can be inefficient in estimating the objective function. To address this issue, we propose Deep Neural Network Multi-Fidelity Bayesian Optimization ({DNN}-{MFBO}) that can flexibly capture all kinds of complicated relationships between the fidelities to improve the objective function estimation and hence the optimization performance. We use sequential,  fidelity-wise Gauss-Hermite quadrature and moment-matching to compute a mutual information-based acquisition function in a tractable and highly efficient way. We show the advantages of our method in both synthetic benchmark datasets and real-world applications in engineering design.},
  pages     = {8521--8531},
  booktitle = {Advances in Neural Information Processing Systems},
  publisher = {Curran Associates, Inc.},
  author    = {Li, Shibo and Xing, Wei and Kirby, Robert and Zhe, Shandian},
  urldate   = {2024-10-02},
  date      = {2020},
  file      = {Full Text PDF:/home/olek/Zotero/storage/T2H3VUFN/Li et al. - 2020 - Multi-Fidelity Bayesian Optimization via Deep Neural Networks.pdf:application/pdf}
}


@article{li2022metadrive,
  title     = {Metadrive: Composing diverse driving scenarios for generalizable reinforcement learning},
  author    = {Li, Quanyi and Peng, Zhenghao and Feng, Lan and Zhang, Qihang and Xue, Zhenghai and Zhou, Bolei},
  journal   = {IEEE transactions on pattern analysis and machine intelligence},
  volume    = {45},
  number    = {3},
  pages     = {3461--3475},
  year      = {2022},
  publisher = {IEEE}
}

@article{li2024choose,
  title     = {Choose your simulator wisely: A review on open-source simulators for autonomous driving},
  author    = {Li, Yueyuan and Yuan, Wei and Zhang, Songan and Yan, Weihao and Shen, Qiyuan and Wang, Chunxiang and Yang, Ming},
  journal   = {IEEE Transactions on Intelligent Vehicles},
  year      = {2024},
  publisher = {IEEE}
}

@article{li2024multi,
  title   = {Multi-fidelity methods for optimization: a survey},
  author  = {Li, Ke and Li, Fan},
  journal = {arXiv preprint arXiv:2402.09638},
  year    = {2024}
}

@inproceedings{lin_commonroad-crime_2023,
  title      = {{CommonRoad}-{CriMe}: A Toolbox for Criticality Measures of Autonomous Vehicles},
  url        = {https://mediatum.ub.tum.de/1706306},
  shorttitle = {{CommonRoad}-{CriMe}},
  abstract   = {Criticality measures are essential for autonomous vehicles to capture the complexity of the surrounding environment, trigger emergency maneuvers, and verify safety. However, there is currently no publicly available toolbox that allows researchers to use or evaluate a large number of criticality measures on arbitrary traffic scenarios. To address this issue, we present {CommonRoad}-{CriMe}, an open-source toolbox for measuring the criticality of autonomous vehicles in a unified framework. Our toolbox covers a wide range of state-of-the-art criticality measures and provides visualized information to facilitate debugging and showcasing. Numerical experiments demonstrate how our toolbox facilitates the comparison of different criticality measures and the analysis of traffic conflicts.},
  eventtitle = {2023 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Lin, Yuanfei and Althoff, Matthias},
  urldate    = {2024-01-12},
  date       = {2023},
  file       = {Full Text PDF:/home/olek/Zotero/storage/42RLEALH/Lin and Althoff - 2023 - CommonRoad-CriMe A Toolbox for Criticality Measur.pdf:application/pdf}
}

@inproceedings{lin_commonroad-crime_2023,
  title      = {{CommonRoad}-{CriMe}: A Toolbox for Criticality Measures of Autonomous Vehicles},
  url        = {https://mediatum.ub.tum.de/1706306},
  shorttitle = {{CommonRoad}-{CriMe}},
  abstract   = {Criticality measures are essential for autonomous vehicles to capture the complexity of the surrounding environment, trigger emergency maneuvers, and verify safety. However, there is currently no publicly available toolbox that allows researchers to use or evaluate a large number of criticality measures on arbitrary traffic scenarios. To address this issue, we present {CommonRoad}-{CriMe}, an open-source toolbox for measuring the criticality of autonomous vehicles in a unified framework. Our toolbox covers a wide range of state-of-the-art criticality measures and provides visualized information to facilitate debugging and showcasing. Numerical experiments demonstrate how our toolbox facilitates the comparison of different criticality measures and the analysis of traffic conflicts.},
  eventtitle = {2023 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Lin, Yuanfei and Althoff, Matthias},
  urldate    = {2024-01-12},
  date       = {2023},
  file       = {Full Text PDF:/home/olek/Zotero/storage/42RLEALH/Lin and Althoff - 2023 - CommonRoad-CriMe A Toolbox for Criticality Measur.pdf:application/pdf}
}

@inproceedings{lin2023crime,
  editor       = {},
  author       = {Lin, Yuanfei and  Althoff, Matthias},
  title        = {CommonRoad-CriMe: A Toolbox for Criticality Measures of Autonomous Vehicles},
  booktitle    = {2023 IEEE Intelligent Vehicles Symposium (IV)},
  year         = {2023},
  month        = {},
  volume       = {},
  publisher    = {},
  organization = {},
  series       = {},
  number       = {},
  pages        = {1-8},
  isbn         = {},
  doi          = {},
  language     = {},
  abstract     = {Criticality measures are essential for autonomous vehicles to capture the complexity of the surrounding environment, trigger emergency maneuvers, and verify safety. However, there is currently no publicly available toolbox that allows researchers to use or evaluate a large number of criticality measures on arbitrary traffic scenarios. To address this issue, we present CommonRoad-CriMe, an open-source toolbox for measuring the criticality of autonomous vehicles in a unified framework. Our toolbox covers a wide range of state-of-the-art criticality measures and provides visualized information to facilitate debugging and showcasing. Numerical experiments demonstrate how our toolbox facilitates the comparison of different criticality measures and the analysis of traffic conflicts. },
  keywords     = {autonomous driving; trajectory repairing; criticality measures},
  note         = {},
  url          = {}
}

@inproceedings{lin2023crime,
  editor       = {},
  author       = {Lin, Yuanfei and  Althoff, Matthias},
  title        = {CommonRoad-CriMe: A Toolbox for Criticality Measures of Autonomous Vehicles},
  booktitle    = {2023 IEEE Intelligent Vehicles Symposium (IV)},
  year         = {2023},
  month        = {},
  volume       = {},
  publisher    = {},
  organization = {},
  series       = {},
  number       = {},
  pages        = {1-8},
  isbn         = {},
  doi          = {},
  language     = {},
  abstract     = {Criticality measures are essential for autonomous vehicles to capture the complexity of the surrounding environment, trigger emergency maneuvers, and verify safety. However, there is currently no publicly available toolbox that allows researchers to use or evaluate a large number of criticality measures on arbitrary traffic scenarios. To address this issue, we present CommonRoad-CriMe, an open-source toolbox for measuring the criticality of autonomous vehicles in a unified framework. Our toolbox covers a wide range of state-of-the-art criticality measures and provides visualized information to facilitate debugging and showcasing. Numerical experiments demonstrate how our toolbox facilitates the comparison of different criticality measures and the analysis of traffic conflicts. },
  keywords     = {autonomous driving; trajectory repairing; criticality measures},
  note         = {},
  url          = {}
}

@book{lukeEssentials2013,
  location   = {S.l.},
  edition    = {2. ed},
  title      = {Essentials of metaheuristics: a set of undergraduate lecture notes; Online Version 2.0},
  isbn       = {978-1-300-54962-8},
  shorttitle = {Essentials of metaheuristics},
  pagetotal  = {239},
  publisher  = {Lulu},
  author     = {Luke, Sean},
  year       = {2013},
  langid     = {english},
  file       = {Luke - 2013 - Essentials of metaheuristics a set of undergradua.pdf:/home/olek/Zotero/storage/CF4C9RNK/Luke - 2013 - Essentials of metaheuristics a set of undergradua.pdf:application/pdf}
}

@book{lukeEssentials2013,
  location   = {S.l.},
  edition    = {2. ed},
  title      = {Essentials of metaheuristics: a set of undergraduate lecture notes; Online Version 2.0},
  isbn       = {978-1-300-54962-8},
  shorttitle = {Essentials of metaheuristics},
  pagetotal  = {239},
  publisher  = {Lulu},
  author     = {Luke, Sean},
  year       = {2013},
  langid     = {english},
  file       = {Luke - 2013 - Essentials of metaheuristics a set of undergradua.pdf:/home/olek/Zotero/storage/CF4C9RNK/Luke - 2013 - Essentials of metaheuristics a set of undergradua.pdf:application/pdf}
}

@inproceedings{ma2018deepgauge,
  title     = {Deepgauge: Multi-granularity testing criteria for deep learning systems},
  author    = {Ma, Lei and Juefei-Xu, Felix and Zhang, Fuyuan and Sun, Jiyuan and Xue, Minhui and Li, Bo and Chen, Chunyang and Su, Ting and Li, Li and Liu, Yang and others},
  booktitle = {Proceedings of the 33rd ACM/IEEE international conference on automated software engineering},
  pages     = {120--131},
  year      = {2018}
}

@inproceedings{ma2018deepgauge,
  title     = {Deepgauge: Multi-granularity testing criteria for deep learning systems},
  author    = {Ma, Lei and Juefei-Xu, Felix and Zhang, Fuyuan and Sun, Jiyuan and Xue, Minhui and Li, Bo and Chen, Chunyang and Su, Ting and Li, Li and Liu, Yang and others},
  booktitle = {Proceedings of the 33rd ACM/IEEE international conference on automated software engineering},
  pages     = {120--131},
  year      = {2018}
}

@article{malan_survey_2021,
  title        = {A Survey of Advances in Landscape Analysis for Optimisation},
  volume       = {14},
  rights       = {http://creativecommons.org/licenses/by/3.0/},
  issn         = {1999-4893},
  url          = {https://www.mdpi.com/1999-4893/14/2/40},
  doi          = {10.3390/a14020040},
  abstract     = {Fitness landscapes were proposed in 1932 as an abstract notion for understanding biological evolution and were later used to explain evolutionary algorithm behaviour. The last ten years has seen the field of fitness landscape analysis develop from a largely theoretical idea in evolutionary computation to a practical tool applied in optimisation in general and more recently in machine learning. With this widened scope, new types of landscapes have emerged such as multiobjective landscapes, violation landscapes, dynamic and coupled landscapes and error landscapes. This survey is a follow-up from a 2013 survey on fitness landscapes and includes an additional 11 landscape analysis techniques. The paper also includes a survey on the applications of landscape analysis for understanding complex problems and explaining algorithm behaviour, as well as algorithm performance prediction and automated algorithm configuration and selection. The extensive use of landscape analysis in a broad range of areas highlights the wide applicability of the techniques and the paper discusses some opportunities for further research in this growing field.},
  pages        = {40},
  number       = {2},
  journaltitle = {Algorithms},
  author       = {Malan, Katherine Mary},
  urldate      = {2023-11-21},
  date         = {2021-02},
  langid       = {english},
  note         = {Number: 2
                  Publisher: Multidisciplinary Digital Publishing Institute},
  keywords     = {automated algorithm selection, error landscape, fitness landscape, landscape analysis, violation landscape},
  file         = {Full Text PDF:/home/olek/Zotero/storage/IYPABHXP/Malan - 2021 - A Survey of Advances in Landscape Analysis for Opt.pdf:application/pdf}
}

@article{malan_survey_2021,
  title        = {A Survey of Advances in Landscape Analysis for Optimisation},
  volume       = {14},
  rights       = {http://creativecommons.org/licenses/by/3.0/},
  issn         = {1999-4893},
  url          = {https://www.mdpi.com/1999-4893/14/2/40},
  doi          = {10.3390/a14020040},
  abstract     = {Fitness landscapes were proposed in 1932 as an abstract notion for understanding biological evolution and were later used to explain evolutionary algorithm behaviour. The last ten years has seen the field of fitness landscape analysis develop from a largely theoretical idea in evolutionary computation to a practical tool applied in optimisation in general and more recently in machine learning. With this widened scope, new types of landscapes have emerged such as multiobjective landscapes, violation landscapes, dynamic and coupled landscapes and error landscapes. This survey is a follow-up from a 2013 survey on fitness landscapes and includes an additional 11 landscape analysis techniques. The paper also includes a survey on the applications of landscape analysis for understanding complex problems and explaining algorithm behaviour, as well as algorithm performance prediction and automated algorithm configuration and selection. The extensive use of landscape analysis in a broad range of areas highlights the wide applicability of the techniques and the paper discusses some opportunities for further research in this growing field.},
  pages        = {40},
  number       = {2},
  journaltitle = {Algorithms},
  author       = {Malan, Katherine Mary},
  urldate      = {2023-11-21},
  date         = {2021-02},
  langid       = {english},
  note         = {Number: 2
                  Publisher: Multidisciplinary Digital Publishing Institute},
  keywords     = {automated algorithm selection, error landscape, fitness landscape, landscape analysis, violation landscape},
  file         = {Full Text PDF:/home/olek/Zotero/storage/IYPABHXP/Malan - 2021 - A Survey of Advances in Landscape Analysis for Opt.pdf:application/pdf}
}

@inproceedings{malayjerdi2023scenario,
  title        = {Scenario-based Validation for Autonomous Vehicles with Different Fidelity Levels},
  author       = {Malayjerdi, Mohsen and Kaljavesi, Gemb and Diermeyer, Frank and Sell, Raivo},
  booktitle    = {2023 IEEE 26th International Conference on Intelligent Transportation Systems (ITSC)},
  pages        = {3411--3416},
  year         = {2023},
  organization = {IEEE}
}

@article{martinez2017beyond,
  title   = {Beyond grand theft auto V for training, testing and enhancing deep learning in self driving cars},
  author  = {Martinez, Mark and Sitawarin, Chawin and Finch, Kevin and Meincke, Lennart and Yablonski, Alex and Kornhauser, Alain},
  journal = {arXiv preprint arXiv:1712.01397},
  year    = {2017}
}

@article{martinez2017beyond,
  title   = {Beyond grand theft auto V for training, testing and enhancing deep learning in self driving cars},
  author  = {Martinez, Mark and Sitawarin, Chawin and Finch, Kevin and Meincke, Lennart and Yablonski, Alex and Kornhauser, Alain},
  journal = {arXiv preprint arXiv:1712.01397},
  year    = {2017}
}

@article{mcdonald2025bayesian,
  title     = {Bayesian Optimization over Multiple Experimental Fidelities Accelerates Automated Discovery of Drug Molecules},
  author    = {McDonald, Matthew A and Koscher, Brent A and Canty, Richard B and Zhang, Jason and Ning, Angelina and Jensen, Klavs F},
  journal   = {ACS Central Science},
  year      = {2025},
  publisher = {ACS Publications}
}

@inproceedings{mcminn2011search,
  title        = {Search-based software testing: Past, present and future},
  author       = {McMinn, Phil},
  booktitle    = {2011 IEEE Fourth International Conference on Software Testing, Verification and Validation Workshops},
  pages        = {153--163},
  year         = {2011},
  organization = {IEEE}
}

@inproceedings{meijer_behind_2018,
  location  = {New York, {NY}, {USA}},
  title     = {Behind every great deep learning framework is an even greater programming languages concept (keynote)},
  isbn      = {978-1-4503-5573-5},
  url       = {https://dl.acm.org/doi/10.1145/3236024.3280855},
  doi       = {10.1145/3236024.3280855},
  series    = {{ESEC}/{FSE} 2018},
  abstract  = {In many areas, such as image recognition, natural language processing, search, recommendation, autonomous cars, systems software and infrastructure, and even Software Engineering tools themselves, Software 2.0 (= programming using learned models) is quickly swallowing Software 1.0 (= programming using handcrafted algorithms). Where the Software 1.0 Engineer formally specifies their problem, carefully designs algorithms, composes systems out of subsystems or decomposes complex systems into smaller components, the Software 2.0 Engineer amasses training data and simply feeds it into an {ML} algorithm that will synthesize an approximation of the function whose partial extensional definition is that training data. Instead of code as the artifact of interest, in Software 2.0 it is all about the data where compilation of source code is replaced by training models with data. This new style of programming has far-reaching consequences for traditional software engineering practices. Everything we have learned about life cycle models, project planning and estimation, requirements analysis, program design, construction, debugging, testing, maintenance and implementation, … runs the danger of becoming obsolete.  One way to try to prepare for the new realities of software engineering is not to zero in on the differences between Software 1.0 and Software 2.0 but instead focus on their similarities. If you carefully look at what a neural net actually represents, you realize that in essence it is a pure function, from multi-dimensional arrays of floating point numbers to multi-dimensional arrays of floating point numbers (tensors). What is special about these functions is that they are differentiable (yes, exactly as you remember from middle school calculus), which allows them to be trained using back propagation. The programming language community has also discovered that there is a deep connection between back propagation and continuations. Moreover, when you look closely at how Software 2.0 Engineers construct complex neural nets like {CNNs}, {RNNs}, {LSTMs}, … you recognize they are (implicitly) using high-order combinators like map, fold, zip, scan, recursion, conditionals, function composition, … to compose complex neural network architectures out of simple building blocks. Constructing neural networks using pure and higher-order differentiable functions and training them using reverse-mode automatic differentiation is unsurprisingly called Differentiable Programming. This talk will illustrate the deep programming language principles behind Differentiable Programming, which will hopefully inspire the working Software 1.0 engineer to pay serious attention to the threats and opportunities of Software 2.0.},
  pages     = {1},
  booktitle = {Proceedings of the 2018 26th {ACM} Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering},
  publisher = {Association for Computing Machinery},
  author    = {Meijer, Erik},
  urldate   = {2024-09-24},
  date      = {2018-10-26},
  file      = {Full Text PDF:/home/olek/Zotero/storage/R2GX4B9K/Meijer - 2018 - Behind every great deep learning framework is an even greater programming languages concept (keynote.pdf:application/pdf}
}

@inproceedings{meliani2019multi,
  title     = {Multi-fidelity efficient global optimization: Methodology and application to airfoil shape design},
  author    = {Meliani, Mostafa and Bartoli, Nathalie and Lefebvre, Thierry and Bouhlel, Mohamed-Amine and Martins, Joaquim RRA and Morlier, Joseph},
  booktitle = {AIAA aviation 2019 forum},
  pages     = {3236},
  year      = {2019}
}

@inproceedings{menzel_scenarios_2018,
  title      = {Scenarios for Development, Test and Validation of Automated Vehicles},
  url        = {https://ieeexplore.ieee.org/abstract/document/8500406},
  doi        = {10.1109/IVS.2018.8500406},
  abstract   = {The latest version of the {ISO} 26262 standard from 2016 represents the state of the art for a safety-guided development of safety-critical electric/electronic vehicle systems. These vehicle systems include advanced driver assistance systems and vehicle guidance systems. The development process proposed in the {ISO} 26262 standard is based upon multiple V-models, and defines activities and work products for each process step. In many of these process steps, scenario based approaches can be applied to achieve the defined work products for the development of automated driving functions. To accomplish the work products of different process steps, scenarios have to focus on various aspects like a human understandable notation or a description via state variables. This leads to contradictory requirements regarding the level of detail and way of notation for the representation of scenarios. In this paper, the authors discuss requirements for the representation of scenarios in different process steps defined by the {ISO} 26262 standard, propose a consistent terminology based on prior publications for the identified levels of abstraction, and demonstrate how scenarios can be systematically evolved along the phases of the development process outlined in the {ISO} 26262 standard.},
  eventtitle = {2018 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {1821--1827},
  booktitle  = {2018 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Menzel, Till and Bagschik, Gerrit and Maurer, Markus},
  urldate    = {2024-10-24},
  date       = {2018-06},
  note       = {{ISSN}: 1931-0587},
  keywords   = {Hardware, Hazards, {ISO} Standards, Product development, Software},
  file       = {Full Text PDF:/home/olek/Zotero/storage/7FE6CB9Q/Menzel et al. - 2018 - Scenarios for Development, Test and Validation of Automated Vehicles.pdf:application/pdf;IEEE Xplore Abstract Record:/home/olek/Zotero/storage/67FFUXFW/8500406.html:text/html}
}

@inproceedings{menzel2018scenarios,
  title        = {Scenarios for development, test and validation of automated vehicles},
  author       = {Menzel, Till and Bagschik, Gerrit and Maurer, Markus},
  booktitle    = {2018 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {1821--1827},
  year         = {2018},
  organization = {IEEE}
}


@inproceedings{menzel2018scenarios,
  title        = {Scenarios for development, test and validation of automated vehicles},
  author       = {Menzel, Till and Bagschik, Gerrit and Maurer, Markus},
  booktitle    = {2018 IEEE Intelligent Vehicles Symposium (IV)},
  pages        = {1821--1827},
  year         = {2018},
  organization = {IEEE}
}

@article{menzies2017delayed,
  title     = {Are delayed issues harder to resolve? Revisiting cost-to-fix of defects throughout the lifecycle},
  author    = {Menzies, Tim and Nichols, William and Shull, Forrest and Layman, Lucas},
  journal   = {Empirical Software Engineering},
  volume    = {22},
  pages     = {1903--1935},
  year      = {2017},
  publisher = {Springer}
}


@article{menzies2017delayed,
  title     = {Are delayed issues harder to resolve? Revisiting cost-to-fix of defects throughout the lifecycle},
  author    = {Menzies, Tim and Nichols, William and Shull, Forrest and Layman, Lucas},
  journal   = {Empirical Software Engineering},
  volume    = {22},
  pages     = {1903--1935},
  year      = {2017},
  publisher = {Springer}
}

@inproceedings{mersmann_benchmarking_2010,
  location   = {Berlin, Heidelberg},
  title      = {Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis},
  isbn       = {978-3-642-15844-5},
  doi        = {10.1007/978-3-642-15844-5_8},
  series     = {Lecture Notes in Computer Science},
  shorttitle = {Benchmarking Evolutionary Algorithms},
  abstract   = {We present methods to answer two basic questions that arise when benchmarking optimization algorithms. The first one is: which algorithm is the ‘best’ one? and the second one: which algorithm should I use for my real world problem? Both are connected and neither is easy to answer. We present methods which can be used to analyse the raw data of a benchmark experiment and derive some insight regarding the answers to these questions. We employ the presented methods to analyse the {BBOB}’09 benchmark results and present some initial findings.},
  pages      = {73--82},
  booktitle  = {Parallel Problem Solving from Nature, {PPSN} {XI}},
  publisher  = {Springer},
  author     = {Mersmann, Olaf and Preuss, Mike and Trautmann, Heike},
  editor     = {Schaefer, Robert and Cotta, Carlos and Kołodziej, Joanna and Rudolph, Günter},
  date       = {2010},
  langid     = {english},
  keywords   = {{BBOB} test set, benchmarking, consensus ranking, evolutionary optimization, multidimensional scaling},
  file       = {Full Text PDF:/home/olek/Zotero/storage/XABA72CB/Mersmann et al. - 2010 - Benchmarking Evolutionary Algorithms Towards Expl.pdf:application/pdf}
}

@inproceedings{mersmann_benchmarking_2010,
  location   = {Berlin, Heidelberg},
  title      = {Benchmarking Evolutionary Algorithms: Towards Exploratory Landscape Analysis},
  isbn       = {978-3-642-15844-5},
  doi        = {10.1007/978-3-642-15844-5_8},
  series     = {Lecture Notes in Computer Science},
  shorttitle = {Benchmarking Evolutionary Algorithms},
  abstract   = {We present methods to answer two basic questions that arise when benchmarking optimization algorithms. The first one is: which algorithm is the ‘best’ one? and the second one: which algorithm should I use for my real world problem? Both are connected and neither is easy to answer. We present methods which can be used to analyse the raw data of a benchmark experiment and derive some insight regarding the answers to these questions. We employ the presented methods to analyse the {BBOB}’09 benchmark results and present some initial findings.},
  pages      = {73--82},
  booktitle  = {Parallel Problem Solving from Nature, {PPSN} {XI}},
  publisher  = {Springer},
  author     = {Mersmann, Olaf and Preuss, Mike and Trautmann, Heike},
  editor     = {Schaefer, Robert and Cotta, Carlos and Kołodziej, Joanna and Rudolph, Günter},
  date       = {2010},
  langid     = {english},
  keywords   = {{BBOB} test set, benchmarking, consensus ranking, evolutionary optimization, multidimensional scaling},
  file       = {Full Text PDF:/home/olek/Zotero/storage/XABA72CB/Mersmann et al. - 2010 - Benchmarking Evolutionary Algorithms Towards Expl.pdf:application/pdf}
}

@inproceedings{mersmann_exploratory_2011,
  location  = {New York, {NY}, {USA}},
  title     = {Exploratory landscape analysis},
  isbn      = {978-1-4503-0557-0},
  url       = {https://dl.acm.org/doi/10.1145/2001576.2001690},
  doi       = {10.1145/2001576.2001690},
  series    = {{GECCO} '11},
  abstract  = {Exploratory Landscape Analysis subsumes a number of techniques employed to obtain knowledge about the properties of an unknown optimization problem, especially insofar as these properties are important for the performance of optimization algorithms. Where in a first attempt, one could rely on high-level features designed by experts, we approach the problem from a different angle here, namely by using relatively cheap low-level computer generated features. Interestingly, very few features are needed to separate the {BBOB} problem groups and also for relating a problem to high-level, expert designed features, paving the way for automatic algorithm selection.},
  pages     = {829--836},
  booktitle = {Proceedings of the 13th annual conference on Genetic and evolutionary computation},
  publisher = {Association for Computing Machinery},
  author    = {Mersmann, Olaf and Bischl, Bernd and Trautmann, Heike and Preuss, Mike and Weihs, Claus and Rudolph, Günter},
  urldate   = {2023-11-21},
  date      = {2011-07-12},
  keywords  = {{BBOB} test set, benchmarking, evolutionary optimization, fitness landscape, exploratory landscape analysis},
  file      = {Full Text PDF:/home/olek/Zotero/storage/8E2HPPZB/Mersmann et al. - 2011 - Exploratory landscape analysis.pdf:application/pdf}
}

@misc{metadrive_docs,
  author  = {{MetaDrive Contributors}},
  title   = {{MetaDrive} Documentation — {MetaDrive} 0.1.1 documentation},
  url     = {https://metadrive-simulator.readthedocs.io/en/latest/},
  urldate = {2024-10-29},
  note    = {Online; accessed 27 October 2024},
  year    = 2024
}

@online{mfboreplicationpackage-ase,
  title     = {Replication Package},
  year      = {2025},
  publisher = {Figshare},
  url       = {https://figshare.com/s/07a31cb250288ced500b}
}

@online{mfboreplicationpackage-iv,
  author = {Anonymous},
  title  = {Replication Package},
  year   = {2025},
  url    = {https://figshare.com/s/ba8790c8ad504514be41}
}

@article{moon2021current,
  title     = {Current advances of artificial pancreas systems: a comprehensive review of the clinical evidence},
  author    = {Moon, Sun Joon and Jung, Inha and Park, Cheol-Young},
  journal   = {Diabetes \& Metabolism Journal},
  volume    = {45},
  number    = {6},
  pages     = {813},
  year      = {2021},
  publisher = {Korean Diabetes Association}
}


@article{moon2021current,
  title     = {Current advances of artificial pancreas systems: a comprehensive review of the clinical evidence},
  author    = {Moon, Sun Joon and Jung, Inha and Park, Cheol-Young},
  journal   = {Diabetes \& Metabolism Journal},
  volume    = {45},
  number    = {6},
  pages     = {813},
  year      = {2021},
  publisher = {Korean Diabetes Association}
}

@inproceedings{mostadi_virtual_2022,
  title      = {Virtual Test Scenarios for {ADAS}: Distance to Real Scenarios Matters!},
  url        = {https://ieeexplore.ieee.org/abstract/document/9827170},
  doi        = {10.1109/IV51971.2022.9827170},
  shorttitle = {Virtual Test Scenarios for {ADAS}},
  abstract   = {Testing in virtual road environments is a widespread approach to validate advanced driver assistance systems ({ADAS}). A number of automated strategies have been proposed to explore dangerous scenarios, like search-based strategies guided by fitness functions. However, such strategies are likely to produce many uninteresting scenarios, representing so extreme driving situations that fatal accidents are unavoidable irrespective of the action of the {ADAS}. We propose leveraging datasets from real drives to better align the virtual scenarios to reasonable ones. The alignment is based on a simple distance metric that relates the virtual scenario parameters to the real data. We demonstrate the use of this metric for testing an autonomous emergency braking ({AEB}) system, taking the {highD} dataset as a reference for normal situations. We show how search-based testing quickly converges toward very distant scenarios that do not bring much insight into the {AEB} performance. We then provide an example of a distance-aware strategy that searches for less extreme scenarios that the {AEB} cannot overcome.},
  eventtitle = {2022 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {836--841},
  booktitle  = {2022 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Mostadi, Mohamed El and Waeselynck, Hélène and Gabriel, Jean-Marc},
  urldate    = {2024-01-12},
  date       = {2022-06},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/JZAHEBXN/Mostadi et al. - 2022 - Virtual Test Scenarios for ADAS Distance to Real .pdf:application/pdf}
}

@inproceedings{mostadi_virtual_2022,
  title      = {Virtual Test Scenarios for {ADAS}: Distance to Real Scenarios Matters!},
  url        = {https://ieeexplore.ieee.org/abstract/document/9827170},
  doi        = {10.1109/IV51971.2022.9827170},
  shorttitle = {Virtual Test Scenarios for {ADAS}},
  abstract   = {Testing in virtual road environments is a widespread approach to validate advanced driver assistance systems ({ADAS}). A number of automated strategies have been proposed to explore dangerous scenarios, like search-based strategies guided by fitness functions. However, such strategies are likely to produce many uninteresting scenarios, representing so extreme driving situations that fatal accidents are unavoidable irrespective of the action of the {ADAS}. We propose leveraging datasets from real drives to better align the virtual scenarios to reasonable ones. The alignment is based on a simple distance metric that relates the virtual scenario parameters to the real data. We demonstrate the use of this metric for testing an autonomous emergency braking ({AEB}) system, taking the {highD} dataset as a reference for normal situations. We show how search-based testing quickly converges toward very distant scenarios that do not bring much insight into the {AEB} performance. We then provide an example of a distance-aware strategy that searches for less extreme scenarios that the {AEB} cannot overcome.},
  eventtitle = {2022 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {836--841},
  booktitle  = {2022 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Mostadi, Mohamed El and Waeselynck, Hélène and Gabriel, Jean-Marc},
  urldate    = {2024-01-12},
  date       = {2022-06},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/JZAHEBXN/Mostadi et al. - 2022 - Virtual Test Scenarios for ADAS Distance to Real .pdf:application/pdf}
}

@misc{mousavian_3d_2017,
  title      = {3D Bounding Box Estimation Using Deep Learning and Geometry},
  url        = {http://arxiv.org/abs/1612.00496},
  doi        = {10.48550/arXiv.1612.00496},
  abstract   = {We present a method for 3D object detection and pose estimation from a single image. In contrast to current techniques that only regress the 3D orientation of an object, our method first regresses relatively stable 3D object properties using a deep convolutional neural network and then combines these estimates with geometric constraints provided by a 2D object bounding box to produce a complete 3D bounding box. The first network output estimates the 3D object orientation using a novel hybrid discrete-continuous loss, which significantly outperforms the L2 loss. The second output regresses the 3D object dimensions, which have relatively little variance compared to alternatives and can often be predicted for many object types. These estimates, combined with the geometric constraints on translation imposed by the 2D bounding box, enable us to recover a stable and accurate 3D object pose. We evaluate our method on the challenging {KITTI} object detection benchmark both on the official metric of 3D orientation estimation and also on the accuracy of the obtained 3D bounding boxes. Although conceptually simple, our method outperforms more complex and computationally expensive approaches that leverage semantic segmentation, instance level segmentation and flat ground priors and sub-category detection. Our discrete-continuous loss also produces state of the art results for 3D viewpoint estimation on the Pascal 3D+ dataset.},
  number     = {{arXiv}:1612.00496},
  publisher  = {{arXiv}},
  author     = {Mousavian, Arsalan and Anguelov, Dragomir and Flynn, John and Kosecka, Jana},
  urldate    = {2024-02-13},
  date       = {2017-04-10},
  eprinttype = {arxiv},
  eprint     = {1612.00496 [cs]},
  keywords   = {Computer Science - Computer Vision and Pattern Recognition},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/KX6FD5ZN/Mousavian et al. - 2017 - 3D Bounding Box Estimation Using Deep Learning and.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/YZ4GQWZL/1612.html:text/html}
}

@misc{mousavian_3d_2017,
  title      = {3D Bounding Box Estimation Using Deep Learning and Geometry},
  url        = {http://arxiv.org/abs/1612.00496},
  doi        = {10.48550/arXiv.1612.00496},
  abstract   = {We present a method for 3D object detection and pose estimation from a single image. In contrast to current techniques that only regress the 3D orientation of an object, our method first regresses relatively stable 3D object properties using a deep convolutional neural network and then combines these estimates with geometric constraints provided by a 2D object bounding box to produce a complete 3D bounding box. The first network output estimates the 3D object orientation using a novel hybrid discrete-continuous loss, which significantly outperforms the L2 loss. The second output regresses the 3D object dimensions, which have relatively little variance compared to alternatives and can often be predicted for many object types. These estimates, combined with the geometric constraints on translation imposed by the 2D bounding box, enable us to recover a stable and accurate 3D object pose. We evaluate our method on the challenging {KITTI} object detection benchmark both on the official metric of 3D orientation estimation and also on the accuracy of the obtained 3D bounding boxes. Although conceptually simple, our method outperforms more complex and computationally expensive approaches that leverage semantic segmentation, instance level segmentation and flat ground priors and sub-category detection. Our discrete-continuous loss also produces state of the art results for 3D viewpoint estimation on the Pascal 3D+ dataset.},
  number     = {{arXiv}:1612.00496},
  publisher  = {{arXiv}},
  author     = {Mousavian, Arsalan and Anguelov, Dragomir and Flynn, John and Kosecka, Jana},
  urldate    = {2024-02-13},
  date       = {2017-04-10},
  eprinttype = {arxiv},
  eprint     = {1612.00496 [cs]},
  keywords   = {Computer Science - Computer Vision and Pattern Recognition},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/KX6FD5ZN/Mousavian et al. - 2017 - 3D Bounding Box Estimation Using Deep Learning and.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/YZ4GQWZL/1612.html:text/html}
}

@article{muller2018sim4cv,
  title     = {Sim4cv: A photo-realistic simulator for computer vision applications},
  author    = {M{\"u}ller, Matthias and Casser, Vincent and Lahoud, Jean and Smith, Neil and Ghanem, Bernard},
  journal   = {International Journal of Computer Vision},
  volume    = {126},
  pages     = {902--919},
  year      = {2018},
  publisher = {Springer}
}

@article{muller2018sim4cv,
  title     = {Sim4cv: A photo-realistic simulator for computer vision applications},
  author    = {M{\"u}ller, Matthias and Casser, Vincent and Lahoud, Jean and Smith, Neil and Ghanem, Bernard},
  journal   = {International Journal of Computer Vision},
  volume    = {126},
  pages     = {902--919},
  year      = {2018},
  publisher = {Springer}
}

@inproceedings{multifidelity,
  author    = {Li, Shibo and Xing, Wei and Kirby, Robert and Zhe, Shandian},
  booktitle = {Advances in Neural Information Processing Systems},
  editor    = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
  pages     = {8521--8531},
  publisher = {Curran Associates, Inc.},
  title     = {Multi-Fidelity Bayesian Optimization via Deep Neural Networks},
  url       = {https://proceedings.neurips.cc/paper_files/paper/2020/file/60e1deb043af37db5ea4ce9ae8d2c9ea-Paper.pdf},
  volume    = {33},
  year      = {2020}
}

@techreport{najm2007pre,
  title       = {Pre-crash scenario typology for crash avoidance research},
  author      = {Najm, Wassim G and Smith, John D and Yanagisawa, Mikio and others},
  year        = {2007},
  institution = {United States. Department of Transportation. National Highway Traffic Safety~…}
}

@techreport{najm2007pre,
  title       = {Pre-crash scenario typology for crash avoidance research},
  author      = {Najm, Wassim G and Smith, John D and Yanagisawa, Mikio and others},
  year        = {2007},
  institution = {United States. Department of Transportation. National Highway Traffic Safety~…}
}
@article{naziri_misbehavior_nodate,
  title    = {Misbehavior Forecasting for Focused Autonomous Driving Systems Testing},
  volume   = {1},
  abstract = {Simulation-based testing is the standard practice for assessing the reliability of self-driving car software before deployment on public roads. Many automated testing techniques generate challenging simulation scenarios to maximize failure detection. However, these techniques require executing a large number of test cases to expose realistic failures. Yet, failure-free test cases may still contain segments where the autonomous driving system was close to a failure. These near misses are often overlooked, reducing testing effectiveness. This paper focuses on enhancing the efficiency and effectiveness of automated testing for autonomous driving systems by focusing on near misses observed during simulation-based testing. Our approach, implemented in a tool called Foresee, identifies potential near misses using a misbehavior forecaster that computes possible future states of the ego-vehicle under test. Foresee performs local fuzzing in the neighborhood of each candidate near miss to surface previously unknown failures. In our empirical study, we evaluate the effectiveness of different configurations of Foresee using several scenarios provided in the {CARLA} simulator. Our results show that Foresee is both more effective and more efficient than the baselines. Foresee exposes 124.14\% and 63.87\% more failures than a random approach and a state-of-the-art failure predictor while being 2.62x and 1.44x faster, respectively.},
  number   = {1},
  author   = {Naziri, M M Abid and Lambertenghi, Stefano Carlo and Stocco, Andrea and D’Amorim, Marcelo},
  langid   = {english},
  file     = {Naziri et al. - Misbehavior Forecasting for Focused Autonomous Dri.pdf:/home/olek/Zotero/storage/CKY9H9NF/Naziri et al. - Misbehavior Forecasting for Focused Autonomous Dri.pdf:application/pdf}
}
@article{naziri_misbehavior_nodate,
  title    = {Misbehavior Forecasting for Focused Autonomous Driving Systems Testing},
  volume   = {1},
  abstract = {Simulation-based testing is the standard practice for assessing the reliability of self-driving car software before deployment on public roads. Many automated testing techniques generate challenging simulation scenarios to maximize failure detection. However, these techniques require executing a large number of test cases to expose realistic failures. Yet, failure-free test cases may still contain segments where the autonomous driving system was close to a failure. These near misses are often overlooked, reducing testing effectiveness. This paper focuses on enhancing the efficiency and effectiveness of automated testing for autonomous driving systems by focusing on near misses observed during simulation-based testing. Our approach, implemented in a tool called Foresee, identifies potential near misses using a misbehavior forecaster that computes possible future states of the ego-vehicle under test. Foresee performs local fuzzing in the neighborhood of each candidate near miss to surface previously unknown failures. In our empirical study, we evaluate the effectiveness of different configurations of Foresee using several scenarios provided in the {CARLA} simulator. Our results show that Foresee is both more effective and more efficient than the baselines. Foresee exposes 124.14\% and 63.87\% more failures than a random approach and a state-of-the-art failure predictor while being 2.62x and 1.44x faster, respectively.},
  number   = {1},
  author   = {Naziri, M M Abid and Lambertenghi, Stefano Carlo and Stocco, Andrea and D’Amorim, Marcelo},
  langid   = {english},
  file     = {Naziri et al. - Misbehavior Forecasting for Focused Autonomous Dri.pdf:/home/olek/Zotero/storage/CKY9H9NF/Naziri et al. - Misbehavior Forecasting for Focused Autonomous Dri.pdf:application/pdf}
}

@article{nejati2023reflections,
  title   = {Reflections on surrogate-assisted search-based testing: A taxonomy and two replication studies based on industrial adas and simulink models},
  author  = {Nejati, Shiva and Sorokin, Lev and Safin, Damir and Formica, Federico and Mahboob, Mohammad Mahdi and Menghi, Claudio},
  journal = {ISE},
  year    = {2023}
}

@book{Newkirk2004TestDD,
  title     = {Test driven development in Microsoft.Net},
  publisher = {Microsoft Press},
  author    = {James Newkirk and Alexei A. Vorontsov},
  year      = {2004},
  url       = {https://api.semanticscholar.org/CorpusID:61100400}
}
@book{Newkirk2004TestDD,
  title     = {Test driven development in Microsoft.Net},
  publisher = {Microsoft Press},
  author    = {James Newkirk and Alexei A. Vorontsov},
  year      = {2004},
  url       = {https://api.semanticscholar.org/CorpusID:61100400}
}

@online{noauthor_san_2024,
  title      = {San Francisco residents fed up with Waymo cars that won’t stop honking at each other},
  url        = {https://www.independent.co.uk/news/world/americas/san-francisco-waymo-taxi-honking-b2596498.html},
  abstract   = {Waymo says it’s working on a fix for the problem},
  titleaddon = {The Independent},
  urldate    = {2024-09-24},
  date       = {2024-08-14},
  langid     = {english},
  note       = {Section: News},
  file       = {Snapshot:/home/olek/Zotero/storage/L9MIXXUX/san-francisco-waymo-taxi-honking-b2596498.html:text/html}
}

@online{noauthor_virtual_nodate,
  title   = {Virtual Test Scenarios for {ADAS}: Distance to Real Scenarios Matters! {\textbar} {IEEE} Conference Publication {\textbar} {IEEE} Xplore},
  url     = {https://ieeexplore.ieee.org/abstract/document/9827170},
  urldate = {2024-01-12}
}

@online{noauthor_virtual_nodate,
  title   = {Virtual Test Scenarios for {ADAS}: Distance to Real Scenarios Matters! {\textbar} {IEEE} Conference Publication {\textbar} {IEEE} Xplore},
  url     = {https://ieeexplore.ieee.org/abstract/document/9827170},
  urldate = {2024-01-12}
}

@inproceedings{ochoa_deconstructing_2016,
  location  = {Cham},
  title     = {Deconstructing the Big Valley Search Space Hypothesis},
  isbn      = {978-3-319-30698-8},
  doi       = {10.1007/978-3-319-30698-8_5},
  series    = {Lecture Notes in Computer Science},
  abstract  = {The big valley hypothesis suggests that, in combinatorial optimisation, local optima of good quality are clustered and surround the global optimum. We show here that the idea of a single valley does not always hold. Instead the big valley seems to de-construct into several valleys, also called ‘funnels’ in theoretical chemistry. We use the local optima networks model and propose an effective procedure for extracting the network data. We conduct a detailed study on four selected {TSP} instances of moderate size and observe that the big valley decomposes into a number of sub-valleys of different sizes and fitness distributions. Sometimes the global optimum is located in the largest valley, which suggests an easy to search landscape, but this is not generally the case. The global optimum might be located in a small valley, which offers a clear and visual explanation of the increased search difficulty in these cases. Our study opens up new possibilities for analysing and visualising combinatorial landscapes as complex networks.},
  pages     = {58--73},
  booktitle = {Evolutionary Computation in Combinatorial Optimization},
  publisher = {Springer International Publishing},
  author    = {Ochoa, Gabriela and Veerapen, Nadarajen},
  editor    = {Chicano, Francisco and Hu, Bin and García-Sánchez, Pablo},
  date      = {2016},
  langid    = {english},
  keywords  = {Fitness landscapes, Funnels, Local optima networks, Traveling salesman problem},
  file      = {Full Text PDF:/home/olek/Zotero/storage/QSYVZM6L/Ochoa and Veerapen - 2016 - Deconstructing the Big Valley Search Space Hypothe.pdf:application/pdf;Full Text PDF:/home/olek/Zotero/storage/9LZKHUCY/Ochoa and Veerapen - 2016 - Deconstructing the Big Valley Search Space Hypothe.pdf:application/pdf}
}

@inproceedings{ochoa_deconstructing_2016,
  location  = {Cham},
  title     = {Deconstructing the Big Valley Search Space Hypothesis},
  isbn      = {978-3-319-30698-8},
  doi       = {10.1007/978-3-319-30698-8_5},
  series    = {Lecture Notes in Computer Science},
  abstract  = {The big valley hypothesis suggests that, in combinatorial optimisation, local optima of good quality are clustered and surround the global optimum. We show here that the idea of a single valley does not always hold. Instead the big valley seems to de-construct into several valleys, also called ‘funnels’ in theoretical chemistry. We use the local optima networks model and propose an effective procedure for extracting the network data. We conduct a detailed study on four selected {TSP} instances of moderate size and observe that the big valley decomposes into a number of sub-valleys of different sizes and fitness distributions. Sometimes the global optimum is located in the largest valley, which suggests an easy to search landscape, but this is not generally the case. The global optimum might be located in a small valley, which offers a clear and visual explanation of the increased search difficulty in these cases. Our study opens up new possibilities for analysing and visualising combinatorial landscapes as complex networks.},
  pages     = {58--73},
  booktitle = {Evolutionary Computation in Combinatorial Optimization},
  publisher = {Springer International Publishing},
  author    = {Ochoa, Gabriela and Veerapen, Nadarajen},
  editor    = {Chicano, Francisco and Hu, Bin and García-Sánchez, Pablo},
  date      = {2016},
  langid    = {english},
  keywords  = {Fitness landscapes, Funnels, Local optima networks, Traveling salesman problem},
  file      = {Full Text PDF:/home/olek/Zotero/storage/QSYVZM6L/Ochoa and Veerapen - 2016 - Deconstructing the Big Valley Search Space Hypothe.pdf:application/pdf;Full Text PDF:/home/olek/Zotero/storage/9LZKHUCY/Ochoa and Veerapen - 2016 - Deconstructing the Big Valley Search Space Hypothe.pdf:application/pdf}
}

@online{opendrive,
  title   = {Detail},
  url     = {https://www.asam.net/standards/detail/opendrive/},
  urldate = {2024-04-03},
  langid  = {english}
}

@online{opendrive,
  title   = {Detail},
  url     = {https://www.asam.net/standards/detail/opendrive/},
  urldate = {2024-04-03},
  langid  = {english}
}

@online{openscenario,
  title   = {Detail},
  url     = {https://www.asam.net/standards/detail/openscenario/},
  urldate = {2024-04-03},
  langid  = {english}
}



@online{openscenario,
  title   = {Detail},
  url     = {https://www.asam.net/standards/detail/openscenario/},
  urldate = {2024-04-03},
  langid  = {english}
}

@inproceedings{osikowicz2024empirically,
  title        = {Empirically Evaluating Flaky Tests for Autonomous Driving Systems in Simulated Environments},
  author       = {Osikowicz, Olek and McMinn, Phil and Shin, Donghwan},
  booktitle    = {2025 IEEE/ACM International Flaky Tests Workshop (FTW) Proceedings},
  year         = {2024},
  organization = {Institute of Electrical and Electronics Engineers (IEEE)}
}

@article{owidinternet,
  author  = {Hannah Ritchie and Edouard Mathieu and Max Roser and Esteban Ortiz-Ospina},
  title   = {Internet},
  journal = {Our World in Data},
  year    = {2023},
  note    = {https://ourworldindata.org/internet}
}

@article{owidinternet,
  author  = {Hannah Ritchie and Edouard Mathieu and Max Roser and Esteban Ortiz-Ospina},
  title   = {Internet},
  journal = {Our World in Data},
  year    = {2023},
  note    = {https://ourworldindata.org/internet}
}

@inproceedings{palanisamy2020multi,
  title        = {Multi-agent connected autonomous driving using deep reinforcement learning},
  author       = {Palanisamy, Praveen},
  booktitle    = {2020 International Joint Conference on Neural Networks (IJCNN)},
  pages        = {1--7},
  year         = {2020},
  organization = {IEEE}
}

@inproceedings{palanisamy2020multi,
  title        = {Multi-agent connected autonomous driving using deep reinforcement learning},
  author       = {Palanisamy, Praveen},
  booktitle    = {2020 International Joint Conference on Neural Networks (IJCNN)},
  pages        = {1--7},
  year         = {2020},
  organization = {IEEE}
}

@inproceedings{paranjape2020modular,
  title        = {A modular architecture for procedural generation of towns, intersections and scenarios for testing autonomous vehicles},
  author       = {Paranjape, Ishaan and Jawad, Abdul and Xu, Yanwen and Song, Asiiah and Whitehead, Jim},
  booktitle    = {2020 ieee intelligent vehicles symposium (iv)},
  pages        = {162--168},
  year         = {2020},
  organization = {IEEE}
}

@inproceedings{paranjape2020modular,
  title        = {A modular architecture for procedural generation of towns, intersections and scenarios for testing autonomous vehicles},
  author       = {Paranjape, Ishaan and Jawad, Abdul and Xu, Yanwen and Song, Asiiah and Whitehead, Jim},
  booktitle    = {2020 ieee intelligent vehicles symposium (iv)},
  pages        = {162--168},
  year         = {2020},
  organization = {IEEE}
}

@article{parry2021survey,
  title     = {A survey of flaky tests},
  author    = {Parry, Owain and Kapfhammer, Gregory M and Hilton, Michael and McMinn, Phil},
  journal   = {ACM Transactions on Software Engineering and Methodology (TOSEM)},
  volume    = {31},
  number    = {1},
  pages     = {1--74},
  year      = {2021},
  publisher = {ACM New York, NY}
}

@inproceedings{pei2017deepxplore,
  title     = {Deepxplore: Automated whitebox testing of deep learning systems},
  author    = {Pei, Kexin and Cao, Yinzhi and Yang, Junfeng and Jana, Suman},
  booktitle = {proceedings of the 26th Symposium on Operating Systems Principles},
  pages     = {1--18},
  year      = {2017}
}

@inproceedings{pei2017deepxplore,
  title     = {Deepxplore: Automated whitebox testing of deep learning systems},
  author    = {Pei, Kexin and Cao, Yinzhi and Yang, Junfeng and Jana, Suman},
  booktitle = {proceedings of the 26th Symposium on Operating Systems Principles},
  pages     = {1--18},
  year      = {2017}
}

@article{peters_updated_2020,
  title        = {Updated methodological guidance for the conduct of scoping reviews},
  volume       = {18},
  issn         = {2689-8381},
  url          = {https://journals.lww.com/10.11124/JBIES-20-00167},
  doi          = {10.11124/JBIES-20-00167},
  pages        = {2119--2126},
  number       = {10},
  journaltitle = {{JBI} Evidence Synthesis},
  author       = {Peters, Micah D.J. and Marnie, Casey and Tricco, Andrea C. and Pollock, Danielle and Munn, Zachary and Alexander, Lyndsay and {McInerney}, Patricia and Godfrey, Christina M. and Khalil, Hanan},
  urldate      = {2023-12-10},
  date         = {2020-10},
  langid       = {english},
  file         = {Peters et al. - 2020 - Updated methodological guidance for the conduct of.pdf:/home/olek/Zotero/storage/DGZVB3LF/Peters et al. - 2020 - Updated methodological guidance for the conduct of.pdf:application/pdf}
}


@article{peters_updated_2020,
  title        = {Updated methodological guidance for the conduct of scoping reviews},
  volume       = {18},
  issn         = {2689-8381},
  url          = {https://journals.lww.com/10.11124/JBIES-20-00167},
  doi          = {10.11124/JBIES-20-00167},
  pages        = {2119--2126},
  number       = {10},
  journaltitle = {{JBI} Evidence Synthesis},
  author       = {Peters, Micah D.J. and Marnie, Casey and Tricco, Andrea C. and Pollock, Danielle and Munn, Zachary and Alexander, Lyndsay and {McInerney}, Patricia and Godfrey, Christina M. and Khalil, Hanan},
  urldate      = {2023-12-10},
  date         = {2020-10},
  langid       = {english},
  file         = {Peters et al. - 2020 - Updated methodological guidance for the conduct of.pdf:/home/olek/Zotero/storage/DGZVB3LF/Peters et al. - 2020 - Updated methodological guidance for the conduct of.pdf:application/pdf}
}


@inproceedings{petersen_systematic_2008,
  title      = {Systematic Mapping Studies in Software Engineering},
  url        = {https://scienceopen.com/hosted-document?doi=10.14236/ewic/EASE2008.8},
  doi        = {10.14236/ewic/EASE2008.8},
  eventtitle = {12th International Conference on Evaluation and Assessment in Software Engineering ({EASE})},
  author     = {Petersen, Kai and Feldt, Robert and Mujtaba, Shahid and Mattsson, Michael},
  urldate    = {2024-01-22},
  date       = {2008-06},
  file       = {Full Text:/home/olek/Zotero/storage/73N3EUET/Petersen et al. - 2008 - Systematic Mapping Studies in Software Engineering.pdf:application/pdf}
}

@inproceedings{petersen_systematic_2008,
  title      = {Systematic Mapping Studies in Software Engineering},
  url        = {https://scienceopen.com/hosted-document?doi=10.14236/ewic/EASE2008.8},
  doi        = {10.14236/ewic/EASE2008.8},
  eventtitle = {12th International Conference on Evaluation and Assessment in Software Engineering ({EASE})},
  author     = {Petersen, Kai and Feldt, Robert and Mujtaba, Shahid and Mattsson, Michael},
  urldate    = {2024-01-22},
  date       = {2008-06},
  file       = {Full Text:/home/olek/Zotero/storage/73N3EUET/Petersen et al. - 2008 - Systematic Mapping Studies in Software Engineering.pdf:application/pdf}
}
  
  @misc{philion_trajeglish_2023,
  title      = {Trajeglish: Learning the Language of Driving Scenarios},
  url        = {http://arxiv.org/abs/2312.04535},
  doi        = {10.48550/arXiv.2312.04535},
  shorttitle = {Trajeglish},
  abstract   = {A longstanding challenge for self-driving development is simulating dynamic driving scenarios seeded from recorded driving logs. In pursuit of this functionality, we apply tools from discrete sequence modeling to model how vehicles, pedestrians and cyclists interact in driving scenarios. Using a simple data-driven tokenization scheme, we discretize trajectories to centimeter-level resolution using a small vocabulary. We then model the multi-agent sequence of motion tokens with a {GPT}-like encoder-decoder that is autoregressive in time and takes into account intra-timestep interaction between agents. Scenarios sampled from our model exhibit state-of-the-art realism; our model tops the Waymo Sim Agents Benchmark, surpassing prior work along the realism meta metric by 3.3\% and along the interaction metric by 9.9\%. We ablate our modeling choices in full autonomy and partial autonomy settings, and show that the representations learned by our model can quickly be adapted to improve performance on {nuScenes}. We additionally evaluate the scalability of our model with respect to parameter count and dataset size, and use density estimates from our model to quantify the saliency of context length and intra-timestep interaction for the traffic modeling task.},
  number     = {{arXiv}:2312.04535},
  publisher  = {{arXiv}},
  author     = {Philion, Jonah and Peng, Xue Bin and Fidler, Sanja},
  urldate    = {2024-02-26},
  date       = {2023-12-07},
  eprinttype = {arxiv},
  eprint     = {2312.04535 [cs]},
  note       = {version: 1},
  keywords   = {Computer Science - Machine Learning, Computer Science - Robotics},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/XBUJZ6SL/Philion et al. - 2023 - Trajeglish Learning the Language of Driving Scena.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/ZE9XTS33/2312.html:text/html}
}

@misc{philion_trajeglish_2023,
  title      = {Trajeglish: Learning the Language of Driving Scenarios},
  url        = {http://arxiv.org/abs/2312.04535},
  doi        = {10.48550/arXiv.2312.04535},
  shorttitle = {Trajeglish},
  abstract   = {A longstanding challenge for self-driving development is simulating dynamic driving scenarios seeded from recorded driving logs. In pursuit of this functionality, we apply tools from discrete sequence modeling to model how vehicles, pedestrians and cyclists interact in driving scenarios. Using a simple data-driven tokenization scheme, we discretize trajectories to centimeter-level resolution using a small vocabulary. We then model the multi-agent sequence of motion tokens with a {GPT}-like encoder-decoder that is autoregressive in time and takes into account intra-timestep interaction between agents. Scenarios sampled from our model exhibit state-of-the-art realism; our model tops the Waymo Sim Agents Benchmark, surpassing prior work along the realism meta metric by 3.3\% and along the interaction metric by 9.9\%. We ablate our modeling choices in full autonomy and partial autonomy settings, and show that the representations learned by our model can quickly be adapted to improve performance on {nuScenes}. We additionally evaluate the scalability of our model with respect to parameter count and dataset size, and use density estimates from our model to quantify the saliency of context length and intra-timestep interaction for the traffic modeling task.},
  number     = {{arXiv}:2312.04535},
  publisher  = {{arXiv}},
  author     = {Philion, Jonah and Peng, Xue Bin and Fidler, Sanja},
  urldate    = {2024-02-26},
  date       = {2023-12-07},
  eprinttype = {arxiv},
  eprint     = {2312.04535 [cs]},
  note       = {version: 1},
  keywords   = {Computer Science - Machine Learning, Computer Science - Robotics},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/XBUJZ6SL/Philion et al. - 2023 - Trajeglish Learning the Language of Driving Scena.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/ZE9XTS33/2312.html:text/html}
}

@misc{philion_trajeglish_2023-1,
  title      = {Trajeglish: Learning the Language of Driving Scenarios},
  url        = {http://arxiv.org/abs/2312.04535},
  doi        = {10.48550/arXiv.2312.04535},
  shorttitle = {Trajeglish},
  abstract   = {A longstanding challenge for self-driving development is simulating dynamic driving scenarios seeded from recorded driving logs. In pursuit of this functionality, we apply tools from discrete sequence modeling to model how vehicles, pedestrians and cyclists interact in driving scenarios. Using a simple data-driven tokenization scheme, we discretize trajectories to centimeter-level resolution using a small vocabulary. We then model the multi-agent sequence of motion tokens with a {GPT}-like encoder-decoder that is autoregressive in time and takes into account intra-timestep interaction between agents. Scenarios sampled from our model exhibit state-of-the-art realism; our model tops the Waymo Sim Agents Benchmark, surpassing prior work along the realism meta metric by 3.3\% and along the interaction metric by 9.9\%. We ablate our modeling choices in full autonomy and partial autonomy settings, and show that the representations learned by our model can quickly be adapted to improve performance on {nuScenes}. We additionally evaluate the scalability of our model with respect to parameter count and dataset size, and use density estimates from our model to quantify the saliency of context length and intra-timestep interaction for the traffic modeling task.},
  number     = {{arXiv}:2312.04535},
  publisher  = {{arXiv}},
  author     = {Philion, Jonah and Peng, Xue Bin and Fidler, Sanja},
  urldate    = {2024-03-12},
  date       = {2023-12-07},
  eprinttype = {arxiv},
  eprint     = {2312.04535 [cs]},
  note       = {version: 1},
  keywords   = {Computer Science - Machine Learning, Computer Science - Robotics},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/BDSXXFUU/Philion et al. - 2023 - Trajeglish Learning the Language of Driving Scena.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/EUV4933V/2312.html:text/html}
}

@misc{philion_trajeglish_2023-1,
  title      = {Trajeglish: Learning the Language of Driving Scenarios},
  url        = {http://arxiv.org/abs/2312.04535},
  doi        = {10.48550/arXiv.2312.04535},
  shorttitle = {Trajeglish},
  abstract   = {A longstanding challenge for self-driving development is simulating dynamic driving scenarios seeded from recorded driving logs. In pursuit of this functionality, we apply tools from discrete sequence modeling to model how vehicles, pedestrians and cyclists interact in driving scenarios. Using a simple data-driven tokenization scheme, we discretize trajectories to centimeter-level resolution using a small vocabulary. We then model the multi-agent sequence of motion tokens with a {GPT}-like encoder-decoder that is autoregressive in time and takes into account intra-timestep interaction between agents. Scenarios sampled from our model exhibit state-of-the-art realism; our model tops the Waymo Sim Agents Benchmark, surpassing prior work along the realism meta metric by 3.3\% and along the interaction metric by 9.9\%. We ablate our modeling choices in full autonomy and partial autonomy settings, and show that the representations learned by our model can quickly be adapted to improve performance on {nuScenes}. We additionally evaluate the scalability of our model with respect to parameter count and dataset size, and use density estimates from our model to quantify the saliency of context length and intra-timestep interaction for the traffic modeling task.},
  number     = {{arXiv}:2312.04535},
  publisher  = {{arXiv}},
  author     = {Philion, Jonah and Peng, Xue Bin and Fidler, Sanja},
  urldate    = {2024-03-12},
  date       = {2023-12-07},
  eprinttype = {arxiv},
  eprint     = {2312.04535 [cs]},
  note       = {version: 1},
  keywords   = {Computer Science - Machine Learning, Computer Science - Robotics},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/BDSXXFUU/Philion et al. - 2023 - Trajeglish Learning the Language of Driving Scena.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/EUV4933V/2312.html:text/html}
}

@article{poloczek2017multi,
  title   = {Multi-information source optimization},
  author  = {Poloczek, Matthias and Wang, Jialei and Frazier, Peter},
  journal = {Advances in neural information processing systems},
  volume  = {30},
  year    = {2017}
}

@misc{rajabli_software_2021,
  title        = {Software Verification and Validation of Safe Autonomous Cars: A Systematic Literature Review},
  volume       = {9},
  issn         = {2169-3536},
  url          = {https://ieeexplore.ieee.org/abstract/document/9310181},
  doi          = {10.1109/ACCESS.2020.3048047},
  shorttitle   = {Software Verification and Validation of Safe Autonomous Cars},
  abstract     = {Autonomous, or self-driving, cars are emerging as the solution to several problems primarily caused by humans on roads, such as accidents and traffic congestion. However, those benefits come with great challenges in the verification and validation (V\&V) for safety assessment. In fact, due to the possibly unpredictable nature of Artificial Intelligence ({AI}), its use in autonomous cars creates concerns that need to be addressed using appropriate V\&V processes that can address trustworthy {AI} and safe autonomy. In this study, the relevant research literature in recent years has been systematically reviewed and classified in order to investigate the state-of-the-art in the software V\&V of autonomous cars. By appropriate criteria, a subset of primary studies has been selected for more in-depth analysis. The first part of the review addresses certification issues against reference standards, challenges in assessing machine learning, as well as general V\&V methodologies. The second part investigates more specific approaches, including simulation environments and mutation testing, corner cases and adversarial examples, fault injection, software safety cages, techniques for cyber-physical systems, and formal methods. Relevant approaches and related tools have been discussed and compared in order to highlight open issues and opportunities.},
  pages        = {4797--4819},
  journaltitle = {{IEEE} Access},
  author       = {Rajabli, Nijat and Flammini, Francesco and Nardone, Roberto and Vittorini, Valeria},
  urldate      = {2023-10-26},
  date         = {2021},
  note         = {Conference Name: {IEEE} Access},
  file         = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/HSLGU2B3/9310181.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/4RQBRY69/Rajabli et al. - 2021 - Software Verification and Validation of Safe Auton.pdf:application/pdf}
}

@misc{rajabli_software_2021,
  title        = {Software Verification and Validation of Safe Autonomous Cars: A Systematic Literature Review},
  volume       = {9},
  issn         = {2169-3536},
  url          = {https://ieeexplore.ieee.org/abstract/document/9310181},
  doi          = {10.1109/ACCESS.2020.3048047},
  shorttitle   = {Software Verification and Validation of Safe Autonomous Cars},
  abstract     = {Autonomous, or self-driving, cars are emerging as the solution to several problems primarily caused by humans on roads, such as accidents and traffic congestion. However, those benefits come with great challenges in the verification and validation (V\&V) for safety assessment. In fact, due to the possibly unpredictable nature of Artificial Intelligence ({AI}), its use in autonomous cars creates concerns that need to be addressed using appropriate V\&V processes that can address trustworthy {AI} and safe autonomy. In this study, the relevant research literature in recent years has been systematically reviewed and classified in order to investigate the state-of-the-art in the software V\&V of autonomous cars. By appropriate criteria, a subset of primary studies has been selected for more in-depth analysis. The first part of the review addresses certification issues against reference standards, challenges in assessing machine learning, as well as general V\&V methodologies. The second part investigates more specific approaches, including simulation environments and mutation testing, corner cases and adversarial examples, fault injection, software safety cages, techniques for cyber-physical systems, and formal methods. Relevant approaches and related tools have been discussed and compared in order to highlight open issues and opportunities.},
  pages        = {4797--4819},
  journaltitle = {{IEEE} Access},
  author       = {Rajabli, Nijat and Flammini, Francesco and Nardone, Roberto and Vittorini, Valeria},
  urldate      = {2023-10-26},
  date         = {2021},
  note         = {Conference Name: {IEEE} Access},
  file         = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/HSLGU2B3/9310181.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/4RQBRY69/Rajabli et al. - 2021 - Software Verification and Validation of Safe Auton.pdf:application/pdf}
}

@online{rankin_ghost_2023,
  title        = {Ghost Gym: A Neural Simulator for Autonomous Driving},
  url          = {https://wayve.ai/thinking/ghost-gym-neural-simulator/},
  shorttitle   = {Ghost Gym},
  abstract     = {Discover Wayve Ghost Gym, a neural simulator that creates photorealistic 4D worlds and thousands of simulated scenarios for training, testing, and debugging our end-to-end {AI} driving models for autonomous vehicles.},
  titleaddon   = {Wayve},
  author       = {Rankin, Rudi},
  urldate      = {2024-05-10},
  year         = {2023},
  organization = {Wayve},
  date         = {2023-12-21},
  langid       = {british}
}
@online{rankin_ghost_2023,
  title        = {Ghost Gym: A Neural Simulator for Autonomous Driving},
  url          = {https://wayve.ai/thinking/ghost-gym-neural-simulator/},
  shorttitle   = {Ghost Gym},
  abstract     = {Discover Wayve Ghost Gym, a neural simulator that creates photorealistic 4D worlds and thousands of simulated scenarios for training, testing, and debugging our end-to-end {AI} driving models for autonomous vehicles.},
  titleaddon   = {Wayve},
  author       = {Rankin, Rudi},
  urldate      = {2024-05-10},
  year         = {2023},
  organization = {Wayve},
  date         = {2023-12-21},
  langid       = {british}
}

@inproceedings{ren_survey_2022,
  title      = {A Survey of Autonomous Driving Scenarios and Scenario Databases},
  doi        = {10.1109/DSA56465.2022.00107},
  abstract   = {With the development of autonomous driving technology, traditional road testing methods can no longer meet the needs of autonomous driving testing. These methods lack sufficiency, comprehensiveness and efficiency. Using the autonomous driving scenario databases for testing can greatly shorten the test time and cost, and can improve the safety and reliability of the test. By systematically sorting out a large number of related publications, this paper summarizes the research progress of autonomous driving scenarios and scenario databases. The article firstly compares and analyzes the different definitions of autonomous driving scenarios, clarifies the connotation of the scenarios, summarizes the types of elements of the scenarios, and introduces the scenario layered model; secondly, we outline the description standards of scenario. We mainly summarize the two scenario data formats, {OpenDRIVE} and {OpenSCENARIO}, which are commonly used in the world. Thirdly, the scenario data collection and research work carried out at home and abroad is reviewed from the perspective of scenario data sources, and different datasets are compared; In addition, the definition of the scenario database, the construction process of the scenario database and the typical scenario databases are summarized; Finally, the problems and future development trends of autonomous driving scenarios and scenario databases are discussed and prospected.},
  eventtitle = {2022 9th International Conference on Dependable Systems and Their Applications ({DSA})},
  pages      = {754--762},
  year       = {2022},
  booktitle  = {2022 9th International Conference on Dependable Systems and Their Applications ({DSA})},
  author     = {Ren, Hongping and Gao, Hui and Chen, He and Liu, Guangzhen},
  date       = {2022-08},
  note       = {{ISSN}: 2767-6684},
  keywords   = {autonomous driving scenarios, Autonomous vehicles, Databases, Market research, Reliability, Roads, Safety, scenario database, scenario dataset, scenario description, Soft sensors},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/LVH9W4Y7/9914464.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/Z3S2QGIV/Ren et al. - 2022 - A Survey of Autonomous Driving Scenarios and Scena.pdf:application/pdf}
}

@inproceedings{ren_survey_2022,
  title      = {A Survey of Autonomous Driving Scenarios and Scenario Databases},
  doi        = {10.1109/DSA56465.2022.00107},
  abstract   = {With the development of autonomous driving technology, traditional road testing methods can no longer meet the needs of autonomous driving testing. These methods lack sufficiency, comprehensiveness and efficiency. Using the autonomous driving scenario databases for testing can greatly shorten the test time and cost, and can improve the safety and reliability of the test. By systematically sorting out a large number of related publications, this paper summarizes the research progress of autonomous driving scenarios and scenario databases. The article firstly compares and analyzes the different definitions of autonomous driving scenarios, clarifies the connotation of the scenarios, summarizes the types of elements of the scenarios, and introduces the scenario layered model; secondly, we outline the description standards of scenario. We mainly summarize the two scenario data formats, {OpenDRIVE} and {OpenSCENARIO}, which are commonly used in the world. Thirdly, the scenario data collection and research work carried out at home and abroad is reviewed from the perspective of scenario data sources, and different datasets are compared; In addition, the definition of the scenario database, the construction process of the scenario database and the typical scenario databases are summarized; Finally, the problems and future development trends of autonomous driving scenarios and scenario databases are discussed and prospected.},
  eventtitle = {2022 9th International Conference on Dependable Systems and Their Applications ({DSA})},
  pages      = {754--762},
  year       = {2022},
  booktitle  = {2022 9th International Conference on Dependable Systems and Their Applications ({DSA})},
  author     = {Ren, Hongping and Gao, Hui and Chen, He and Liu, Guangzhen},
  date       = {2022-08},
  note       = {{ISSN}: 2767-6684},
  keywords   = {autonomous driving scenarios, Autonomous vehicles, Databases, Market research, Reliability, Roads, Safety, scenario database, scenario dataset, scenario description, Soft sensors},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/LVH9W4Y7/9914464.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/Z3S2QGIV/Ren et al. - 2022 - A Survey of Autonomous Driving Scenarios and Scena.pdf:application/pdf}
}

@article{riedmaier_survey_2020,
  author   = {Riedmaier, Stefan and Ponn, Thomas and Ludwig, Dieter and Schick, Bernhard and Diermeyer, Frank},
  journal  = {IEEE Access},
  title    = {Survey on Scenario-Based Safety Assessment of Automated Vehicles},
  year     = {2020},
  volume   = {8},
  number   = {},
  pages    = {87456-87477},
  keywords = {Safety;Testing;Bibliographies;Vehicles;Microscopy;Taxonomy;Automation;Automated vehicles;autonomous vehicles;data analysis;formal verification;intelligent vehicles;key performance indicators;simulation;vehicle safety},
  doi      = {10.1109/ACCESS.2020.2993730}
}


@article{riedmaier_survey_2020,
  author   = {Riedmaier, Stefan and Ponn, Thomas and Ludwig, Dieter and Schick, Bernhard and Diermeyer, Frank},
  journal  = {IEEE Access},
  title    = {Survey on Scenario-Based Safety Assessment of Automated Vehicles},
  year     = {2020},
  volume   = {8},
  number   = {},
  pages    = {87456-87477},
  keywords = {Safety;Testing;Bibliographies;Vehicles;Microscopy;Taxonomy;Automation;Automated vehicles;autonomous vehicles;data analysis;formal verification;intelligent vehicles;key performance indicators;simulation;vehicle safety},
  doi      = {10.1109/ACCESS.2020.2993730}
}

@inproceedings{rong2020lgsvl,
  title        = {{LGSVL} simulator: A high fidelity simulator for autonomous driving},
  author       = {Rong, Guodong and Shin, Byung Hyun and Tabatabaee, Hadi and Lu, Qiang and Lemke, Steve and Mo{\v{z}}eiko, M{\=a}rti{\c{n}}{\v{s}} and Boise, Eric and Uhm, Geehoon and Gerow, Mark and Mehta, Shalin and others},
  booktitle    = {IEEE ITSC},
  pages        = {1--6},
  year         = {2020},
  organization = {IEEE}
}

@article{sadri2022survey,
  title     = {Survey on test case generation, selection and prioritization for cyber-physical systems},
  author    = {Sadri-Moshkenani, Zahra and Bradley, Justin and Rothermel, Gregg},
  journal   = {Software Testing, Verification and Reliability},
  volume    = {32},
  number    = {1},
  pages     = {e1794},
  year      = {2022},
  publisher = {Wiley Online Library}
}

@online{sae-levels,
  title      = {J3016\_202104: Taxonomy and Definitions for Terms Related to Driving Automation Systems for On-Road Motor Vehicles - {SAE} International},
  url        = {https://www.sae.org/standards/content/j3016_202104/},
  shorttitle = {J3016\_202104},
  abstract   = {This document describes [motor] vehicle driving automation systems that perform part or all of the dynamic driving task ({DDT}) on a sustained basis. It provides a taxonomy with detailed definitions for six levels of driving automation, ranging from no driving automation (Level 0) to full driving auto},
  urldate    = {2024-04-13},
  file       = {Snapshot:/home/olek/Zotero/storage/ID2UR2L4/j3016_202104.html:text/html}
}
@online{sae-levels,
  title      = {J3016\_202104: Taxonomy and Definitions for Terms Related to Driving Automation Systems for On-Road Motor Vehicles - {SAE} International},
  url        = {https://www.sae.org/standards/content/j3016_202104/},
  shorttitle = {J3016\_202104},
  abstract   = {This document describes [motor] vehicle driving automation systems that perform part or all of the dynamic driving task ({DDT}) on a sustained basis. It provides a taxonomy with detailed definitions for six levels of driving automation, ranging from no driving automation (Level 0) to full driving auto},
  urldate    = {2024-04-13},
  file       = {Snapshot:/home/olek/Zotero/storage/ID2UR2L4/j3016_202104.html:text/html}
}

@article{santara2021madras,
  title   = {Madras: Multi agent driving simulator},
  author  = {Santara, Anirban and Rudra, Sohan and Buridi, Sree Aditya and Kaushik, Meha and Naik, Abhishek and Kaul, Bharat and Ravindran, Balaraman},
  journal = {Journal of Artificial Intelligence Research},
  volume  = {70},
  pages   = {1517--1555},
  year    = {2021}
}


@article{santara2021madras,
  title   = {Madras: Multi agent driving simulator},
  author  = {Santara, Anirban and Rudra, Sohan and Buridi, Sree Aditya and Kaushik, Meha and Naik, Abhishek and Kaul, Bharat and Ravindran, Balaraman},
  journal = {Journal of Artificial Intelligence Research},
  volume  = {70},
  pages   = {1517--1555},
  year    = {2021}
}


@inproceedings{SBST,
  author    = {McMinn, Phil},
  booktitle = {2011 IEEE Fourth International Conference on Software Testing, Verification and Validation Workshops},
  title     = {Search-Based Software Testing: Past, Present and Future},
  year      = {2011},
  volume    = {},
  number    = {},
  pages     = {153-163},
  keywords  = {Software testing;Optimization;Search problems;Software;Genetic algorithms;Databases;Search-Based Software Testing;Search-Based Software Engineering},
  doi       = {10.1109/ICSTW.2011.100}
}

@inproceedings{scheuer_stretch_2023,
  title      = {{STRETCH}: Generating Challenging Scenarios for Testing Collision Avoidance Systems},
  url        = {https://ieeexplore.ieee.org/abstract/document/10186634},
  doi        = {10.1109/IV55152.2023.10186634},
  shorttitle = {{STRETCH}},
  abstract   = {Collision avoidance systems are fundamental for autonomous driving and need to be tested thoroughly to check whether they safely handle critical scenarios. Testing collision avoidance systems is generally done by means of scenario-based testing using simulators and comes with the main challenge of generating situations that are realistic but avoidable. In other words, driving scenarios must stress the collision avoidance functionalities while being representative. Existing crash databases and accident reports describe observed accidents and enable to (re)create realistic collisions in simulations; however, as those data sources focus on the impact, their data do not generally lead to avoidable collision scenarios. To address this issue, we propose {STRETCH}, which generates realistic, critical, and avoidable collision scenarios by extending focused collision descriptions using a multi-objective optimization algorithm. Thanks to {STRETCH}, developers and testers can automatically generate challenging test cases based on realistic crash scenarios.},
  eventtitle = {2023 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {1--6},
  booktitle  = {2023 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Scheuer, Franz and Gambi, Alessio and Arcaini, Paolo},
  urldate    = {2023-12-11},
  date       = {2023-06},
  note       = {{ISSN}: 2642-7214},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/THKW4I9B/Scheuer et al. - 2023 - STRETCH Generating Challenging Scenarios for Test.pdf:application/pdf}
}

@inproceedings{scheuer_stretch_2023,
  title      = {{STRETCH}: Generating Challenging Scenarios for Testing Collision Avoidance Systems},
  url        = {https://ieeexplore.ieee.org/abstract/document/10186634},
  doi        = {10.1109/IV55152.2023.10186634},
  shorttitle = {{STRETCH}},
  abstract   = {Collision avoidance systems are fundamental for autonomous driving and need to be tested thoroughly to check whether they safely handle critical scenarios. Testing collision avoidance systems is generally done by means of scenario-based testing using simulators and comes with the main challenge of generating situations that are realistic but avoidable. In other words, driving scenarios must stress the collision avoidance functionalities while being representative. Existing crash databases and accident reports describe observed accidents and enable to (re)create realistic collisions in simulations; however, as those data sources focus on the impact, their data do not generally lead to avoidable collision scenarios. To address this issue, we propose {STRETCH}, which generates realistic, critical, and avoidable collision scenarios by extending focused collision descriptions using a multi-objective optimization algorithm. Thanks to {STRETCH}, developers and testers can automatically generate challenging test cases based on realistic crash scenarios.},
  eventtitle = {2023 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {1--6},
  booktitle  = {2023 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Scheuer, Franz and Gambi, Alessio and Arcaini, Paolo},
  urldate    = {2023-12-11},
  date       = {2023-06},
  note       = {{ISSN}: 2642-7214},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/THKW4I9B/Scheuer et al. - 2023 - STRETCH Generating Challenging Scenarios for Test.pdf:application/pdf}
}

@article{scholtes20216,
  title     = {6-layer model for a structured description and categorization of urban traffic and environment},
  author    = {Scholtes, Maike and Westhofen, Lukas and Turner, Lara Ruth and Lotto, Katrin and Schuldes, Michael and Weber, Hendrik and Wagener, Nicolas and Neurohr, Christian and Bollmann, Martin Herbert and K{\"o}rtke, Franziska and others},
  journal   = {IEEE Access},
  volume    = {9},
  pages     = {59131--59147},
  year      = {2021},
  publisher = {IEEE}
}

@article{scholtes20216,
  title     = {6-layer model for a structured description and categorization of urban traffic and environment},
  author    = {Scholtes, Maike and Westhofen, Lukas and Turner, Lara Ruth and Lotto, Katrin and Schuldes, Michael and Weber, Hendrik and Wagener, Nicolas and Neurohr, Christian and Bollmann, Martin Herbert and K{\"o}rtke, Franziska and others},
  journal   = {IEEE Access},
  volume    = {9},
  pages     = {59131--59147},
  year      = {2021},
  publisher = {IEEE}
}


@article{schulman2017proximal,
  title   = {Proximal policy optimization algorithms},
  author  = {Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
  journal = {arXiv preprint arXiv:1707.06347},
  year    = {2017}
}


@misc{schutt10012023,
  title      = {1001 Ways of Scenario Generation for Testing of Self-driving Cars: A Survey},
  url        = {http://arxiv.org/abs/2304.10850},
  doi        = {10.48550/arXiv.2304.10850},
  shorttitle = {1001 Ways of Scenario Generation for Testing of Self-driving Cars},
  abstract   = {Scenario generation is one of the essential steps in scenario-based testing and, therefore, a significant part of the verification and validation of driver assistance functions and autonomous driving systems. However, the term scenario generation is used for many different methods, e.g., extraction of scenarios from naturalistic driving data or variation of scenario parameters. This survey aims to give a systematic overview of different approaches, establish different categories of scenario acquisition and generation, and show that each group of methods has typical input and output types. It shows that although the term is often used throughout literature, the evaluated methods use different inputs and the resulting scenarios differ in abstraction level and from a systematical point of view. Additionally, recent research and literature examples are given to underline this categorization.},
  number     = {{arXiv}:2304.10850},
  publisher  = {{arXiv}},
  author     = {Schütt, Barbara and Ransiek, Joshua and Braun, Thilo and Sax, Eric},
  urldate    = {2024-01-12},
  date       = {2023-04-21},
  year       = {2023},
  eprinttype = {arxiv},
  eprint     = {2304.10850 [cs]},
  keywords   = {Computer Science - Robotics, Computer Science - Software Engineering},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/EZEVJJJG/Schütt et al. - 2023 - 1001 Ways of Scenario Generation for Testing of Se.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/JJKWUX9I/2304.html:text/html}
}


@misc{schutt10012023,
  title      = {1001 Ways of Scenario Generation for Testing of Self-driving Cars: A Survey},
  url        = {http://arxiv.org/abs/2304.10850},
  doi        = {10.48550/arXiv.2304.10850},
  shorttitle = {1001 Ways of Scenario Generation for Testing of Self-driving Cars},
  abstract   = {Scenario generation is one of the essential steps in scenario-based testing and, therefore, a significant part of the verification and validation of driver assistance functions and autonomous driving systems. However, the term scenario generation is used for many different methods, e.g., extraction of scenarios from naturalistic driving data or variation of scenario parameters. This survey aims to give a systematic overview of different approaches, establish different categories of scenario acquisition and generation, and show that each group of methods has typical input and output types. It shows that although the term is often used throughout literature, the evaluated methods use different inputs and the resulting scenarios differ in abstraction level and from a systematical point of view. Additionally, recent research and literature examples are given to underline this categorization.},
  number     = {{arXiv}:2304.10850},
  publisher  = {{arXiv}},
  author     = {Schütt, Barbara and Ransiek, Joshua and Braun, Thilo and Sax, Eric},
  urldate    = {2024-01-12},
  date       = {2023-04-21},
  year       = {2023},
  eprinttype = {arxiv},
  eprint     = {2304.10850 [cs]},
  keywords   = {Computer Science - Robotics, Computer Science - Software Engineering},
  file       = {arXiv Fulltext PDF:/home/olek/Zotero/storage/EZEVJJJG/Schütt et al. - 2023 - 1001 Ways of Scenario Generation for Testing of Se.pdf:application/pdf;arXiv.org Snapshot:/home/olek/Zotero/storage/JJKWUX9I/2304.html:text/html}
}

@article{scikit-learn,
  title   = {Scikit-learn: Machine Learning in {P}ython},
  author  = {Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
             and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
             and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
             Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal = {Journal of Machine Learning Research},
  volume  = {12},
  pages   = {2825--2830},
  year    = {2011}
}


@inproceedings{shao2022interfuser,
  title        = {Safety-enhanced autonomous driving using interpretable sensor fusion transformer},
  author       = {Shao, Hao and Wang, Letian and Chen, Ruobing and Li, Hongsheng and Liu, Yu},
  booktitle    = {Conference on Robot Learning},
  pages        = {726--737},
  year         = {2023},
  organization = {PMLR}
}

@inproceedings{shao2023reasonnet,
  title     = {Reasonnet: End-to-end driving with temporal and global reasoning},
  author    = {Shao, Hao and Wang, Letian and Chen, Ruobing and Waslander, Steven L and Li, Hongsheng and Liu, Yu},
  booktitle = {Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages     = {13723--13733},
  year      = {2023}
}

@inproceedings{shin2017illusion,
  title        = {Illusion and dazzle: Adversarial optical channel exploits against lidars for automotive applications},
  author       = {Shin, Hocheol and Kim, Dohyun and Kwon, Yujin and Kim, Yongdae},
  booktitle    = {Cryptographic Hardware and Embedded Systems--CHES 2017: 19th International Conference, Taipei, Taiwan, September 25-28, 2017, Proceedings},
  pages        = {445--467},
  year         = {2017},
  organization = {Springer}
}

@inproceedings{shin2017illusion,
  title        = {Illusion and dazzle: Adversarial optical channel exploits against lidars for automotive applications},
  author       = {Shin, Hocheol and Kim, Dohyun and Kwon, Yujin and Kim, Yongdae},
  booktitle    = {Cryptographic Hardware and Embedded Systems--CHES 2017: 19th International Conference, Taipei, Taiwan, September 25-28, 2017, Proceedings},
  pages        = {445--467},
  year         = {2017},
  organization = {Springer}
}
@misc{sim-atav,
  title         = {Simulation-based Adversarial Test Generation for Autonomous Vehicles with Machine Learning Components},
  author        = {Cumhur Erkan Tuncali and Georgios Fainekos and Hisahiro Ito and James Kapinski},
  year          = {2019},
  eprint        = {1804.06760},
  archiveprefix = {arXiv},
  primaryclass  = {cs.SY}
}

@misc{sim-atav,
  title         = {Simulation-based Adversarial Test Generation for Autonomous Vehicles with Machine Learning Components},
  author        = {Cumhur Erkan Tuncali and Georgios Fainekos and Hisahiro Ito and James Kapinski},
  year          = {2019},
  eprint        = {1804.06760},
  archiveprefix = {arXiv},
  primaryclass  = {cs.SY}
}

@online{simphera_nodate,
  title    = {{SIMPHERA}},
  url      = {https://www.dspace.com/en/ltd/home/products/sw/simulation_software/simphera.cfm},
  abstract = {The web-based and scalable cloud solution by {dSPACE} for simplicity in simulation and validation},
  urldate  = {2024-04-15},
  langid   = {english},
  file     = {Snapshot:/home/olek/Zotero/storage/ERMCN9RX/simphera.html:text/html}
}

@online{simphera_nodate,
  title    = {{SIMPHERA}},
  url      = {https://www.dspace.com/en/ltd/home/products/sw/simulation_software/simphera.cfm},
  abstract = {The web-based and scalable cloud solution by {dSPACE} for simplicity in simulation and validation},
  urldate  = {2024-04-15},
  langid   = {english},
  file     = {Snapshot:/home/olek/Zotero/storage/ERMCN9RX/simphera.html:text/html}
}

@online{simulation_city,
  title      = {Simulation City: Introducing Waymo's most advanced simulation system yet for autonomous driving},
  url        = {https://waymo.com/blog/2021/07/simulation-city},
  shorttitle = {Simulation City},
  abstract   = {At Waymo, we use simulation to advance the Waymo Driver's abilities in several ways. We can amplify our number of real-world miles to accelerate our Driver’s learning, prepare for rare events, train new models, validate new software, and even evaluate how the Waymo Driver would have performed in actual fatal crashes that have already occurred. For simulation to be valuable as a learning tool, it has to closely match our target domain – performing rider-only trips for Waymo One and goods delivery trips for Waymo Via in the real world– in which case, there is only one question that matters. Did the Waymo Driver safely and efficiently complete the trip?},
  titleaddon = {Waymo},
  urldate    = {2024-05-10},
  year       = {2021},
  langid     = {english}
}

@online{simulation_city,
  title      = {Simulation City: Introducing Waymo's most advanced simulation system yet for autonomous driving},
  url        = {https://waymo.com/blog/2021/07/simulation-city},
  shorttitle = {Simulation City},
  abstract   = {At Waymo, we use simulation to advance the Waymo Driver's abilities in several ways. We can amplify our number of real-world miles to accelerate our Driver’s learning, prepare for rare events, train new models, validate new software, and even evaluate how the Waymo Driver would have performed in actual fatal crashes that have already occurred. For simulation to be valuable as a learning tool, it has to closely match our target domain – performing rider-only trips for Waymo One and goods delivery trips for Waymo Via in the real world– in which case, there is only one question that matters. Did the Waymo Driver safely and efficiently complete the trip?},
  titleaddon = {Waymo},
  urldate    = {2024-05-10},
  year       = {2021},
  langid     = {english}
}

@article{snoek2012practical,
  title   = {Practical bayesian optimization of machine learning algorithms},
  author  = {Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P},
  journal = {Advances in neural information processing systems},
  volume  = {25},
  year    = {2012}
}

@article{song_critical_2023,
  title        = {Critical scenario identification for realistic testing of autonomous driving systems},
  volume       = {31},
  issn         = {1573-1367},
  url          = {https://doi.org/10.1007/s11219-022-09604-2},
  doi          = {10.1007/s11219-022-09604-2},
  abstract     = {Autonomous driving has become an important research area for road traffic, whereas testing of autonomous driving systems to ensure a safe and reliable operation remains an open challenge. Substantial real-world testing or massive driving data collection does not scale since the potential test scenarios in real-world traffic are infinite, and covering large shares of them in the test is impractical. Thus, critical ones have to be prioritized. We have developed an approach for critical test scenario identification and in this study, we implement the approach and validate it on two real autonomous driving systems from industry by integrating it into their tool-chain. Our main contribution in this work is the demonstration and validation of our approach for critical scenario identification for testing real autonomous driving systems.},
  pages        = {441--469},
  number       = {2},
  journaltitle = {Software Quality Journal},
  shortjournal = {Software Qual J},
  author       = {Song, Qunying and Tan, Kaige and Runeson, Per and Persson, Stefan},
  urldate      = {2024-01-12},
  date         = {2023-06-01},
  langid       = {english},
  keywords     = {Software testing, Autonomous driving, Critical scenario identification, Test scenario generation},
  file         = {Full Text PDF:/home/olek/Zotero/storage/5WJ29Y55/Song et al. - 2023 - Critical scenario identification for realistic tes.pdf:application/pdf}
}

@article{song_critical_2023,
  title        = {Critical scenario identification for realistic testing of autonomous driving systems},
  volume       = {31},
  issn         = {1573-1367},
  url          = {https://doi.org/10.1007/s11219-022-09604-2},
  doi          = {10.1007/s11219-022-09604-2},
  abstract     = {Autonomous driving has become an important research area for road traffic, whereas testing of autonomous driving systems to ensure a safe and reliable operation remains an open challenge. Substantial real-world testing or massive driving data collection does not scale since the potential test scenarios in real-world traffic are infinite, and covering large shares of them in the test is impractical. Thus, critical ones have to be prioritized. We have developed an approach for critical test scenario identification and in this study, we implement the approach and validate it on two real autonomous driving systems from industry by integrating it into their tool-chain. Our main contribution in this work is the demonstration and validation of our approach for critical scenario identification for testing real autonomous driving systems.},
  pages        = {441--469},
  number       = {2},
  journaltitle = {Software Quality Journal},
  shortjournal = {Software Qual J},
  author       = {Song, Qunying and Tan, Kaige and Runeson, Per and Persson, Stefan},
  urldate      = {2024-01-12},
  date         = {2023-06-01},
  langid       = {english},
  keywords     = {Software testing, Autonomous driving, Critical scenario identification, Test scenario generation},
  file         = {Full Text PDF:/home/olek/Zotero/storage/5WJ29Y55/Song et al. - 2023 - Critical scenario identification for realistic tes.pdf:application/pdf}
}

@inproceedings{stahlbauer2019testing,
  title     = {Testing scratch programs automatically},
  author    = {Stahlbauer, Andreas and Kreis, Marvin and Fraser, Gordon},
  booktitle = {Proceedings of the 2019 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering},
  pages     = {165--175},
  year      = {2019}
}

@article{sun_scenario-based_2022,
  author   = {Sun, Jian and Zhang, He and Zhou, Huajun and Yu, Rongjie and Tian, Ye},
  journal  = {IEEE Transactions on Intelligent Transportation Systems},
  title    = {Scenario-Based Test Automation for Highly Automated Vehicles: A Review and Paving the Way for Systematic Safety Assurance},
  year     = {2022},
  volume   = {23},
  number   = {9},
  pages    = {14088-14103},
  keywords = {Testing;Automation;Safety;Systematics;Roads;Accidents;Software;Highly automated vehicles;test automation;safety assurance;traffic scenarios},
  doi      = {10.1109/TITS.2021.3136353}
}

@article{sun_scenario-based_2022,
  author   = {Sun, Jian and Zhang, He and Zhou, Huajun and Yu, Rongjie and Tian, Ye},
  journal  = {IEEE Transactions on Intelligent Transportation Systems},
  title    = {Scenario-Based Test Automation for Highly Automated Vehicles: A Review and Paving the Way for Systematic Safety Assurance},
  year     = {2022},
  volume   = {23},
  number   = {9},
  pages    = {14088-14103},
  keywords = {Testing;Automation;Safety;Systematics;Roads;Accidents;Software;Highly automated vehicles;test automation;safety assurance;traffic scenarios},
  doi      = {10.1109/TITS.2021.3136353}
}

@article{sun2021scenario,
  title     = {Scenario-based test automation for highly automated vehicles: A review and paving the way for systematic safety assurance},
  author    = {Sun, Jian and Zhang, He and Zhou, Huajun and Yu, Rongjie and Tian, Ye},
  journal   = {IEEE transactions on intelligent transportation systems},
  volume    = {23},
  number    = {9},
  pages     = {14088--14103},
  year      = {2021},
  publisher = {IEEE}
}

@article{sun2021scenario,
  title     = {Scenario-based test automation for highly automated vehicles: A review and paving the way for systematic safety assurance},
  author    = {Sun, Jian and Zhang, He and Zhou, Huajun and Yu, Rongjie and Tian, Ye},
  journal   = {IEEE transactions on intelligent transportation systems},
  volume    = {23},
  number    = {9},
  pages     = {14088--14103},
  year      = {2021},
  publisher = {IEEE}
}

@article{sun2022intersim,
  title   = {InterSim: Interactive Traffic Simulation via Explicit Relation Modeling},
  author  = {Sun, Qiao and Huang, Xin and Williams, Brian C and Zhao, Hang},
  journal = {arXiv preprint arXiv:2210.14413},
  year    = {2022}
}

@article{sun2022intersim,
  title   = {InterSim: Interactive Traffic Simulation via Explicit Relation Modeling},
  author  = {Sun, Qiao and Huang, Xin and Williams, Brian C and Zhao, Hang},
  journal = {arXiv preprint arXiv:2210.14413},
  year    = {2022}
}

@misc{tang_survey_2023,
  title        = {A Survey on Automated Driving System Testing: Landscapes and Trends},
  issn         = {1049-331X, 1557-7392},
  url          = {https://dl.acm.org/doi/10.1145/3579642},
  doi          = {10.1145/3579642},
  shorttitle   = {A Survey on Automated Driving System Testing},
  abstract     = {Automated Driving Systems
                  (
                  {ADS}
                  ) have made great achievements in recent years thanks to the efforts from both academia and industry. A typical {ADS} is composed of multiple modules, including sensing, perception, planning, and control, which brings together the latest advances in different domains. Despite these achievements, safety assurance of {ADS} is of great significance, since unsafe behavior of {ADS} can bring catastrophic consequences. Testing has been recognized as an important system validation approach that aims to expose unsafe system behavior; however, in the context of {ADS}, it is extremely challenging to devise effective testing techniques, due to the high complexity and multidisciplinarity of the systems. There has been great much literature that focuses on the testing of {ADS}, and a number of surveys have also emerged to summarize the technical advances. Most of the surveys focus on the system-level testing performed within software simulators, and they thereby ignore the distinct features of different modules. In this paper, we provide a comprehensive survey on the existing {ADS} testing literature, which takes into account both module-level and system-level testing. Specifically, we make the following contributions: (1) we survey the module-level testing techniques for {ADS} and highlight the technical differences affected by the features of different modules; (2) we also survey the system-level testing techniques, with focuses on the empirical studies that summarize the issues occurring in system development or deployment, the problems due to the collaborations between different modules, and the gap between {ADS} testing in simulators and the real world; (3) we identify the challenges and opportunities in {ADS} testing, which pave the path to the future research in this field.},
  pages        = {3579642},
  journaltitle = {{ACM} Transactions on Software Engineering and Methodology},
  shortjournal = {{ACM} Trans. Softw. Eng. Methodol.},
  author       = {Tang, Shuncheng and Zhang, Zhenya and Zhang, Yi and Zhou, Jixiang and Guo, Yan and Liu, Shuang and Guo, Shengjian and Li, Yan-Fu and Ma, Lei and Xue, Yinxing and Liu, Yang},
  urldate      = {2023-06-16},
  date         = {2023-02-10},
  langid       = {english},
  keywords     = {{ADS} testing, module-level testing, system security, system-level testing},
  file         = {Full Text PDF:/home/olek/Zotero/storage/U4FAUTWD/Tang et al. - 2023 - A Survey on Automated Driving System Testing Land.pdf:application/pdf;Full Text PDF:/home/olek/Zotero/storage/DAAFYN5R/Tang et al. - 2023 - A Survey on Automated Driving System Testing Land.pdf:application/pdf}
}

@misc{tang_survey_2023,
  title        = {A Survey on Automated Driving System Testing: Landscapes and Trends},
  issn         = {1049-331X, 1557-7392},
  url          = {https://dl.acm.org/doi/10.1145/3579642},
  doi          = {10.1145/3579642},
  shorttitle   = {A Survey on Automated Driving System Testing},
  abstract     = {Automated Driving Systems
                  (
                  {ADS}
                  ) have made great achievements in recent years thanks to the efforts from both academia and industry. A typical {ADS} is composed of multiple modules, including sensing, perception, planning, and control, which brings together the latest advances in different domains. Despite these achievements, safety assurance of {ADS} is of great significance, since unsafe behavior of {ADS} can bring catastrophic consequences. Testing has been recognized as an important system validation approach that aims to expose unsafe system behavior; however, in the context of {ADS}, it is extremely challenging to devise effective testing techniques, due to the high complexity and multidisciplinarity of the systems. There has been great much literature that focuses on the testing of {ADS}, and a number of surveys have also emerged to summarize the technical advances. Most of the surveys focus on the system-level testing performed within software simulators, and they thereby ignore the distinct features of different modules. In this paper, we provide a comprehensive survey on the existing {ADS} testing literature, which takes into account both module-level and system-level testing. Specifically, we make the following contributions: (1) we survey the module-level testing techniques for {ADS} and highlight the technical differences affected by the features of different modules; (2) we also survey the system-level testing techniques, with focuses on the empirical studies that summarize the issues occurring in system development or deployment, the problems due to the collaborations between different modules, and the gap between {ADS} testing in simulators and the real world; (3) we identify the challenges and opportunities in {ADS} testing, which pave the path to the future research in this field.},
  pages        = {3579642},
  journaltitle = {{ACM} Transactions on Software Engineering and Methodology},
  shortjournal = {{ACM} Trans. Softw. Eng. Methodol.},
  author       = {Tang, Shuncheng and Zhang, Zhenya and Zhang, Yi and Zhou, Jixiang and Guo, Yan and Liu, Shuang and Guo, Shengjian and Li, Yan-Fu and Ma, Lei and Xue, Yinxing and Liu, Yang},
  urldate      = {2023-06-16},
  date         = {2023-02-10},
  langid       = {english},
  keywords     = {{ADS} testing, module-level testing, system security, system-level testing},
  file         = {Full Text PDF:/home/olek/Zotero/storage/U4FAUTWD/Tang et al. - 2023 - A Survey on Automated Driving System Testing Land.pdf:application/pdf;Full Text PDF:/home/olek/Zotero/storage/DAAFYN5R/Tang et al. - 2023 - A Survey on Automated Driving System Testing Land.pdf:application/pdf}
}

@inproceedings{tenbrock_conscend_2021,
  title      = {The {ConScenD} Dataset: Concrete Scenarios from the {highD} Dataset According to {ALKS} Regulation {UNECE} R157 in {OpenX}},
  url        = {https://ieeexplore.ieee.org/abstract/document/9669219?casa_token=y0D8WD10x2kAAAAA:JqLMQs5VpeVeBksjYKaevr8NHQR4Ev2Pe4BRdOdrqzPWf3_IQKDbUtbx8K2HzsydAE5ivuqKf6s},
  doi        = {10.1109/IVWorkshops54471.2021.9669219},
  shorttitle = {The {ConScenD} Dataset},
  abstract   = {With Regulation {UNECE} R157 on Automated Lane-Keeping Systems, the first framework for the introduction of passenger cars with Level 3 systems has become available in 2020. In accordance with recent research projects including academia and the automotive industry, the Regulation utilizes scenario based testing for the safety assessment. The complexity of safety validation of automated driving systems necessitates system-level simulations. The Regulation, however, is missing the required parameterization necessary for test case generation. To overcome this problem, we incorporate the exposure and consider the heterogeneous behavior of the traffic participants by extracting concrete scenarios according to Regulation's scenario definition from the established naturalistic highway dataset {highD}. We present a methodology to find the scenarios in real-world data, extract the parameters for modeling the scenarios and transfer them to simulation. In this process, more than 340 scenarios were extracted. {OpenSCENARIO} files were generated to enable an exemplary transfer of the scenarios to {CARLA} and esmini. We compare the trajectories to examine the similarity of the scenarios in the simulation to the recorded scenarios. In order to foster research, we publish the resulting dataset called {ConScenD} together with instructions for usage with both simulation tools. The dataset is available online at https://www.{levelXdata}.com/scenarios.},
  eventtitle = {2021 {IEEE} Intelligent Vehicles Symposium Workshops ({IV} Workshops)},
  pages      = {174--181},
  booktitle  = {2021 {IEEE} Intelligent Vehicles Symposium Workshops ({IV} Workshops)},
  author     = {Tenbrock, Alexander and König, Alexander and Keutgens, Thomas and Weber, Hendrik},
  urldate    = {2024-03-06},
  date       = {2021-07},
  keywords   = {Safety, Conferences, Data models, Intelligent vehicles, Regulation, Road transportation, Trajectory},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/AMTCRTUA/9669219.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/RMRMP6XY/Tenbrock et al. - 2021 - The ConScenD Dataset Concrete Scenarios from the .pdf:application/pdf}
}

@inproceedings{tenbrock_conscend_2021,
  title      = {The {ConScenD} Dataset: Concrete Scenarios from the {highD} Dataset According to {ALKS} Regulation {UNECE} R157 in {OpenX}},
  url        = {https://ieeexplore.ieee.org/abstract/document/9669219?casa_token=y0D8WD10x2kAAAAA:JqLMQs5VpeVeBksjYKaevr8NHQR4Ev2Pe4BRdOdrqzPWf3_IQKDbUtbx8K2HzsydAE5ivuqKf6s},
  doi        = {10.1109/IVWorkshops54471.2021.9669219},
  shorttitle = {The {ConScenD} Dataset},
  abstract   = {With Regulation {UNECE} R157 on Automated Lane-Keeping Systems, the first framework for the introduction of passenger cars with Level 3 systems has become available in 2020. In accordance with recent research projects including academia and the automotive industry, the Regulation utilizes scenario based testing for the safety assessment. The complexity of safety validation of automated driving systems necessitates system-level simulations. The Regulation, however, is missing the required parameterization necessary for test case generation. To overcome this problem, we incorporate the exposure and consider the heterogeneous behavior of the traffic participants by extracting concrete scenarios according to Regulation's scenario definition from the established naturalistic highway dataset {highD}. We present a methodology to find the scenarios in real-world data, extract the parameters for modeling the scenarios and transfer them to simulation. In this process, more than 340 scenarios were extracted. {OpenSCENARIO} files were generated to enable an exemplary transfer of the scenarios to {CARLA} and esmini. We compare the trajectories to examine the similarity of the scenarios in the simulation to the recorded scenarios. In order to foster research, we publish the resulting dataset called {ConScenD} together with instructions for usage with both simulation tools. The dataset is available online at https://www.{levelXdata}.com/scenarios.},
  eventtitle = {2021 {IEEE} Intelligent Vehicles Symposium Workshops ({IV} Workshops)},
  pages      = {174--181},
  booktitle  = {2021 {IEEE} Intelligent Vehicles Symposium Workshops ({IV} Workshops)},
  author     = {Tenbrock, Alexander and König, Alexander and Keutgens, Thomas and Weber, Hendrik},
  urldate    = {2024-03-06},
  date       = {2021-07},
  keywords   = {Safety, Conferences, Data models, Intelligent vehicles, Regulation, Road transportation, Trajectory},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/AMTCRTUA/9669219.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/RMRMP6XY/Tenbrock et al. - 2021 - The ConScenD Dataset Concrete Scenarios from the .pdf:application/pdf}
}

@online{thadani_how_2023,
  title        = {How a robotaxi crash got Cruise’s self-driving cars pulled from Californian roads},
  issn         = {0190-8286},
  url          = {https://www.washingtonpost.com/technology/2023/10/28/robotaxi-cruise-crash-driverless-car-san-francisco/},
  abstract     = {A prominent self-driving car company went from approval to ban in just two months, highlighting the fragmented oversight governing the fledgling industry.},
  journaltitle = {Washington Post},
  author       = {Thadani, Trisha},
  urldate      = {2024-09-24},
  date         = {2023-10-28},
  langid       = {american}
}

@inproceedings{transfuser,
  title     = {Multi-modal fusion transformer for end-to-end autonomous driving},
  author    = {Prakash, Aditya and Chitta, Kashyap and Geiger, Andreas},
  booktitle = {Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages     = {7077--7087},
  year      = {2021}
}
@inproceedings{ulbrich2015defining,
  title        = {Defining and substantiating the terms scene, situation, and scenario for automated driving},
  author       = {Ulbrich, Simon and Menzel, Till and Reschka, Andreas and Schuldt, Fabian and Maurer, Markus},
  booktitle    = {2015 IEEE 18th international conference on intelligent transportation systems},
  pages        = {982--988},
  year         = {2015},
  organization = {IEEE}
}

@inproceedings{ulbrich2015defining,
  title        = {Defining and substantiating the terms scene, situation, and scenario for automated driving},
  author       = {Ulbrich, Simon and Menzel, Till and Reschka, Andreas and Schuldt, Fabian and Maurer, Markus},
  booktitle    = {2015 IEEE 18th international conference on intelligent transportation systems},
  pages        = {982--988},
  year         = {2015},
  organization = {IEEE}
}


@article{vaswani2017attention,
  title   = {Attention is all you need},
  author  = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal = {Advances in neural information processing systems},
  volume  = {30},
  year    = {2017}
}
@article{vinitsky2022nocturne,
  title   = {Nocturne: a scalable driving benchmark for bringing multi-agent learning one step closer to the real world},
  author  = {Vinitsky, Eugene and Lichtl{\'e}, Nathan and Yang, Xiaomeng and Amos, Brandon and Foerster, Jakob},
  journal = {arXiv preprint arXiv:2206.09889},
  year    = {2022}
}

@article{vinitsky2022nocturne,
  title   = {Nocturne: a scalable driving benchmark for bringing multi-agent learning one step closer to the real world},
  author  = {Vinitsky, Eugene and Lichtl{\'e}, Nathan and Yang, Xiaomeng and Amos, Brandon and Foerster, Jakob},
  journal = {arXiv preprint arXiv:2206.09889},
  year    = {2022}
}

@online{wad_at_cvpr_cvpr23_2023,
  title    = {[{CVPR}'23 {WAD}] Keynote - Andreas Geiger, Universität Tübingen},
  url      = {https://www.youtube.com/watch?v=x_42Fji1Z2M},
  abstract = {Talk given at the {CVPR} Workshop on Autonomous Driving 2023: https://cvpr2023.wad.vision/.
              
              00:00 Introduction
              03:12 {nuPlan}
              04:53 Open-Loop and Closed-Loop
              05:57 Lesson 1: A route centerline is all you need for ego-forecasting
              08:25 Lesson 2: Rule-based beat learned planners in closed-loop
              10:06 Lesson 3: Planning \& ego-forecasting tasks are misaligned
              16:01 Lesson 4: Learned/long-term forecasting doesn't improve closed-loop
              17:49 {CARLA}
              22:00 {TransFuser}
              22:58 Lesson 5: Target point conditioned methods learn shortcut when out-of-distribution
              25:26 Lesson 6: Average pooling remove spatial info and increases bias
              27:11 Lesson 7: Waypoint representations work well as they learn to interpolate
              28:48 Summary},
  author   = { Andreas Geiger},
  urldate  = {2024-03-24},
  date     = {2023-06-29}
}

@online{wad_at_cvpr_cvpr23_2023,
  title    = {[{CVPR}'23 {WAD}] Keynote - Andreas Geiger, Universität Tübingen},
  url      = {https://www.youtube.com/watch?v=x_42Fji1Z2M},
  abstract = {Talk given at the {CVPR} Workshop on Autonomous Driving 2023: https://cvpr2023.wad.vision/.
              
              00:00 Introduction
              03:12 {nuPlan}
              04:53 Open-Loop and Closed-Loop
              05:57 Lesson 1: A route centerline is all you need for ego-forecasting
              08:25 Lesson 2: Rule-based beat learned planners in closed-loop
              10:06 Lesson 3: Planning \& ego-forecasting tasks are misaligned
              16:01 Lesson 4: Learned/long-term forecasting doesn't improve closed-loop
              17:49 {CARLA}
              22:00 {TransFuser}
              22:58 Lesson 5: Target point conditioned methods learn shortcut when out-of-distribution
              25:26 Lesson 6: Average pooling remove spatial info and increases bias
              27:11 Lesson 7: Waypoint representations work well as they learn to interpolate
              28:48 Summary},
  author   = { Andreas Geiger},
  urldate  = {2024-03-24},
  date     = {2023-06-29}
}

@article{wang_autonomous_2022,
  title        = {Autonomous driving testing scenario generation based on in-depth vehicle-to-powered two-wheeler crash data in China},
  volume       = {176},
  issn         = {0001-4575},
  url          = {https://www.sciencedirect.com/science/article/pii/S0001457522002470},
  doi          = {10.1016/j.aap.2022.106812},
  abstract     = {A reliable critical-scenario-based safety assessment of autonomous vehicles in China requires a thorough understanding of complex crash scenarios in Chinese background traffic. Based on actual crashes between a vehicle and a powered two-wheeler ({PTW}) in China, this study generated the autonomous driving testing scenarios from functional, logical and concrete levels. First, 239 video-recorded crash cases were selected from the China In-depth mobility Safety Study - Traffic Accident ({CIMSS}-{TA}) database. Using the k-medoids clustering method, six functional scenarios were generalized according to seven crash characteristics (time of day, road type, road surface, obstruction, motion of vehicle, motion of {PTW}, relative moving direction and position of {PTW} with respect to vehicle), which contained two straight road scenarios, two T-junction scenarios and two intersection scenarios. Then, using a trajectory analysis program written by Python, the dangerous time instant of each crash was extracted based on the relative trajectory. According to five dynamic parameters of dangerous time instant, namely vehicle velocity (Vehicle\_V), {PTW} X'-coordinate velocity ({PTW}\_VX'), {PTW} Y'-coordinate velocity ({PTW}\_VY'), {PTW} X'-coordinate relative position ({PTW}\_LocX') and {PTW} Y'-coordinate relative position ({PTW}\_LocY'), a crash trigger scheme was built to remain a case challenging when the involved vehicle is replaced by an autonomous vehicle with completely different maneuvers. Using the kernel density estimation ({KDE}), the logical scenarios were evolved by calculating the distribution of these dynamic parameters in each cluster. The results showed that there were differences in the distribution of dynamic parameters between six functional scenarios. For instance, the Vehicle\_V in the scenario where a vehicle turning right impacts with a right/right rear {PTW} traveling straight ahead was higher than that in the scenario where a vehicle changing to the left lane impacts with a left/left rear {PTW} traveling straight ahead, with ranges of (10 km/h, 30 km/h) and (5 km/h, 15 km/h), respectively. Finally, considering the correlation of dynamic parameters, a virtual crash generation approach based on the independent component analysis ({ICA}) representing the original crashes with independent parameters was proposed to obtain sufficient concrete testing scenarios. The results showed that the statistical characteristics of virtual crashes were consistent with those of original crashes. Therefore, the virtual crash generation approach was effective. And a concrete crossing testing scenario with the crash trigger conditions of Vehicle\_V = 26.272 km/h, {PTW}\_VX' = 15.567 km/h, {PTW}\_VY' = −1.670 km/h, {PTW}\_LocX' = −27.265 m and {PTW}\_LocY' = 52. 149 m was especially demonstrated. This study provides a theoretical basis for generating autonomous driving testing scenarios and data support for establishing relevant testing schemes tailored to the traffic environment in China.},
  pages        = {106812},
  journal      = {Accident Analysis \& Prevention},
  journaltitle = {Accident Analysis \& Prevention},
  shortjournal = {Accident Analysis \& Prevention},
  author       = {Wang, Xinghua and Peng, Yong and Xu, Tuo and Xu, Qian and Wu, Xianhui and Xiang, Guoliang and Yi, Shengen and Wang, Honggang},
  urldate      = {2024-01-12},
  date         = {2022-10-01},
  year         = {2022},
  keywords     = {Clustering analysis, Crash trigger scheme, Testing scenario, Vehicle-to-powered two-wheeler crash, Virtual crash generation approach},
  file         = {ScienceDirect Snapshot:/home/olek/Zotero/storage/RISMZHVM/S0001457522002470.html:text/html;Wang et al. - 2022 - Autonomous driving testing scenario generation bas.pdf:/home/olek/Zotero/storage/7NHRC3B2/Wang et al. - 2022 - Autonomous driving testing scenario generation bas.pdf:application/pdf}
}

@article{wang_autonomous_2022,
  title        = {Autonomous driving testing scenario generation based on in-depth vehicle-to-powered two-wheeler crash data in China},
  volume       = {176},
  issn         = {0001-4575},
  url          = {https://www.sciencedirect.com/science/article/pii/S0001457522002470},
  doi          = {10.1016/j.aap.2022.106812},
  abstract     = {A reliable critical-scenario-based safety assessment of autonomous vehicles in China requires a thorough understanding of complex crash scenarios in Chinese background traffic. Based on actual crashes between a vehicle and a powered two-wheeler ({PTW}) in China, this study generated the autonomous driving testing scenarios from functional, logical and concrete levels. First, 239 video-recorded crash cases were selected from the China In-depth mobility Safety Study - Traffic Accident ({CIMSS}-{TA}) database. Using the k-medoids clustering method, six functional scenarios were generalized according to seven crash characteristics (time of day, road type, road surface, obstruction, motion of vehicle, motion of {PTW}, relative moving direction and position of {PTW} with respect to vehicle), which contained two straight road scenarios, two T-junction scenarios and two intersection scenarios. Then, using a trajectory analysis program written by Python, the dangerous time instant of each crash was extracted based on the relative trajectory. According to five dynamic parameters of dangerous time instant, namely vehicle velocity (Vehicle\_V), {PTW} X'-coordinate velocity ({PTW}\_VX'), {PTW} Y'-coordinate velocity ({PTW}\_VY'), {PTW} X'-coordinate relative position ({PTW}\_LocX') and {PTW} Y'-coordinate relative position ({PTW}\_LocY'), a crash trigger scheme was built to remain a case challenging when the involved vehicle is replaced by an autonomous vehicle with completely different maneuvers. Using the kernel density estimation ({KDE}), the logical scenarios were evolved by calculating the distribution of these dynamic parameters in each cluster. The results showed that there were differences in the distribution of dynamic parameters between six functional scenarios. For instance, the Vehicle\_V in the scenario where a vehicle turning right impacts with a right/right rear {PTW} traveling straight ahead was higher than that in the scenario where a vehicle changing to the left lane impacts with a left/left rear {PTW} traveling straight ahead, with ranges of (10 km/h, 30 km/h) and (5 km/h, 15 km/h), respectively. Finally, considering the correlation of dynamic parameters, a virtual crash generation approach based on the independent component analysis ({ICA}) representing the original crashes with independent parameters was proposed to obtain sufficient concrete testing scenarios. The results showed that the statistical characteristics of virtual crashes were consistent with those of original crashes. Therefore, the virtual crash generation approach was effective. And a concrete crossing testing scenario with the crash trigger conditions of Vehicle\_V = 26.272 km/h, {PTW}\_VX' = 15.567 km/h, {PTW}\_VY' = −1.670 km/h, {PTW}\_LocX' = −27.265 m and {PTW}\_LocY' = 52. 149 m was especially demonstrated. This study provides a theoretical basis for generating autonomous driving testing scenarios and data support for establishing relevant testing schemes tailored to the traffic environment in China.},
  pages        = {106812},
  journal      = {Accident Analysis \& Prevention},
  journaltitle = {Accident Analysis \& Prevention},
  shortjournal = {Accident Analysis \& Prevention},
  author       = {Wang, Xinghua and Peng, Yong and Xu, Tuo and Xu, Qian and Wu, Xianhui and Xiang, Guoliang and Yi, Shengen and Wang, Honggang},
  urldate      = {2024-01-12},
  date         = {2022-10-01},
  year         = {2022},
  keywords     = {Clustering analysis, Crash trigger scheme, Testing scenario, Vehicle-to-powered two-wheeler crash, Virtual crash generation approach},
  file         = {ScienceDirect Snapshot:/home/olek/Zotero/storage/RISMZHVM/S0001457522002470.html:text/html;Wang et al. - 2022 - Autonomous driving testing scenario generation bas.pdf:/home/olek/Zotero/storage/7NHRC3B2/Wang et al. - 2022 - Autonomous driving testing scenario generation bas.pdf:application/pdf}
}

@inproceedings{wang_gar_2022,
  title      = {{GAR}: Generalized Autoregression for Multi-Fidelity Fusion},
  url        = {https://openreview.net/forum?id=aLNWp0pn1Ij},
  shorttitle = {{GAR}},
  abstract   = {In many scientific research and engineering applications, where repeated simulations of complex systems are conducted, a surrogate is commonly adopted to quickly estimate the whole system. To reduce the expensive cost of generating training examples, it has become a promising approach to combine the results of low-fidelity (fast but inaccurate) and high-fidelity (slow but accurate) simulations. Despite the fast developments of multi-fidelity fusion techniques, most existing methods require particular data structures and do not scale well to high-dimensional output. To resolve these issues, we generalize the classic autoregression ({AR}), which is wildly used due to its simplicity, robustness, accuracy, and tractability, and propose generalized autoregression ({GAR}) using tensor formulation and latent features. {GAR} can deal with arbitrary dimensional outputs and arbitrary multifidelity data structure to satisfy the demand of multi-fidelity fusion for complex problems; it admits a fully tractable likelihood and posterior requiring no approximate inference and scales well to high-dimensional problems. Furthermore, we prove the autokrigeability theorem based on {GAR} in the multi-fidelity case and develop {CIGAR}, a simplified {GAR} with the same predictive mean accuracy but requires significantly less computation. In experiments of canonical {PDEs} and scientific computational examples, the proposed method consistently outperforms the {SOTA} methods with a large margin (up to 6x improvement in {RMSE}) with only a few high-fidelity training samples.},
  eventtitle = {Advances in Neural Information Processing Systems},
  author     = {Wang, Yuxin and Xing, Zheng and Xing, Wei W.},
  urldate    = {2024-02-05},
  date       = {2022-05-16},
  langid     = {english},
  file       = {Full Text PDF:/home/olek/Zotero/storage/R2LAK2SW/Wang et al. - 2022 - GAR Generalized Autoregression for Multi-Fidelity.pdf:application/pdf}
}

@inproceedings{wang_gar_2022,
  title      = {{GAR}: Generalized Autoregression for Multi-Fidelity Fusion},
  url        = {https://openreview.net/forum?id=aLNWp0pn1Ij},
  shorttitle = {{GAR}},
  abstract   = {In many scientific research and engineering applications, where repeated simulations of complex systems are conducted, a surrogate is commonly adopted to quickly estimate the whole system. To reduce the expensive cost of generating training examples, it has become a promising approach to combine the results of low-fidelity (fast but inaccurate) and high-fidelity (slow but accurate) simulations. Despite the fast developments of multi-fidelity fusion techniques, most existing methods require particular data structures and do not scale well to high-dimensional output. To resolve these issues, we generalize the classic autoregression ({AR}), which is wildly used due to its simplicity, robustness, accuracy, and tractability, and propose generalized autoregression ({GAR}) using tensor formulation and latent features. {GAR} can deal with arbitrary dimensional outputs and arbitrary multifidelity data structure to satisfy the demand of multi-fidelity fusion for complex problems; it admits a fully tractable likelihood and posterior requiring no approximate inference and scales well to high-dimensional problems. Furthermore, we prove the autokrigeability theorem based on {GAR} in the multi-fidelity case and develop {CIGAR}, a simplified {GAR} with the same predictive mean accuracy but requires significantly less computation. In experiments of canonical {PDEs} and scientific computational examples, the proposed method consistently outperforms the {SOTA} methods with a large margin (up to 6x improvement in {RMSE}) with only a few high-fidelity training samples.},
  eventtitle = {Advances in Neural Information Processing Systems},
  author     = {Wang, Yuxin and Xing, Zheng and Xing, Wei W.},
  urldate    = {2024-02-05},
  date       = {2022-05-16},
  langid     = {english},
  file       = {Full Text PDF:/home/olek/Zotero/storage/R2LAK2SW/Wang et al. - 2022 - GAR Generalized Autoregression for Multi-Fidelity.pdf:application/pdf}
}

@inproceedings{wang2022gar,
  title     = {{GAR}: Generalized Autoregression for Multi-Fidelity Fusion},
  author    = {Yuxin Wang and Zheng Xing and WEI W. XING},
  booktitle = {Advances in Neural Information Processing Systems},
  editor    = {Alice H. Oh and Alekh Agarwal and Danielle Belgrave and Kyunghyun Cho},
  year      = {2022},
  url       = {https://openreview.net/forum?id=aLNWp0pn1Ij}
}

@online{waymo-one,
  title      = {Ride-Hailing App - Make the Most of Your Drive - Waymo One},
  url        = {https://waymo.com/waymo-one/},
  abstract   = {Waymo One, our autonomous ride-hailing app, offers stress-free rides so you can make the most of your time. Sit back and relax—We'll handle the drive.},
  titleaddon = {Waymo},
  urldate    = {2024-04-11},
  langid     = {english}
}

@online{waymo-one,
  title      = {Ride-Hailing App - Make the Most of Your Drive - Waymo One},
  url        = {https://waymo.com/waymo-one/},
  abstract   = {Waymo One, our autonomous ride-hailing app, offers stress-free rides so you can make the most of your time. Sit back and relax—We'll handle the drive.},
  titleaddon = {Waymo},
  urldate    = {2024-04-11},
  langid     = {english}
}

@inproceedings{waymoOpenMotionDataset,
  author    = {Ettinger, Scott and Cheng, Shuyang and Caine, Benjamin and Liu, Chenxi and Zhao, Hang and Pradhan, Sabeek and Chai, Yuning and Sapp, Ben and Qi, Charles R. and Zhou, Yin and Yang, Zoey and Chouard, Aur'elien and Sun, Pei and Ngiam, Jiquan and Vasudevan, Vijay and McCauley, Alexander and Shlens, Jonathon and Anguelov, Dragomir},
  title     = {Large Scale Interactive Motion Forecasting for Autonomous Driving: The Waymo Open Motion Dataset},
  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
  month     = {October},
  year      = {2021},
  pages     = {9710-9719}
}

@inproceedings{waymoOpenMotionDataset,
  author    = {Ettinger, Scott and Cheng, Shuyang and Caine, Benjamin and Liu, Chenxi and Zhao, Hang and Pradhan, Sabeek and Chai, Yuning and Sapp, Ben and Qi, Charles R. and Zhou, Yin and Yang, Zoey and Chouard, Aur'elien and Sun, Pei and Ngiam, Jiquan and Vasudevan, Vijay and McCauley, Alexander and Shlens, Jonathon and Anguelov, Dragomir},
  title     = {Large Scale Interactive Motion Forecasting for Autonomous Driving: The Waymo Open Motion Dataset},
  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
  month     = {October},
  year      = {2021},
  pages     = {9710-9719}
}

@online{waymoOpenOnline,
  title      = {Download - Waymo Open Dataset},
  url        = {https://waymo.com/open/download/},
  abstract   = {Download Waymo Open Dataset},
  titleaddon = {Waymo},
  urldate    = {2024-04-04},
  langid     = {english},
  file       = {Snapshot:/home/olek/Zotero/storage/NXF3HL4M/download.html:text/html}
}

@online{waymoOpenOnline,
  title      = {Download - Waymo Open Dataset},
  url        = {https://waymo.com/open/download/},
  abstract   = {Download Waymo Open Dataset},
  titleaddon = {Waymo},
  urldate    = {2024-04-04},
  langid     = {english},
  file       = {Snapshot:/home/olek/Zotero/storage/NXF3HL4M/download.html:text/html}
}

@inproceedings{werth_surrogate-assisted_2020,
  location  = {Cham},
  title     = {Surrogate-Assisted Fitness Landscape Analysis for Computationally Expensive Optimization},
  isbn      = {978-3-030-45093-9},
  doi       = {10.1007/978-3-030-45093-9_30},
  series    = {Lecture Notes in Computer Science},
  abstract  = {Exploratory fitness landscape analysis ({FLA}) is a category of techniques that try to capture knowledge about a black-box optimization problem. This is achieved by assigning features to a certain problem instance utilizing only information obtained by evaluating the black-box. This knowledge can be used to obtain new domain knowledge but more often the intended use is to automatically find an appropriate heuristic optimization algorithm [9]. {FLA}-based algorithm selection and parametrization hinges on the idea, that, while no optimization algorithm can be the optimal choice for all black-box problems, algorithms are expected to work similarly well on problems with similar statistical characteristics [8, 15].},
  pages     = {247--254},
  booktitle = {Computer Aided Systems Theory – {EUROCAST} 2019},
  publisher = {Springer International Publishing},
  author    = {Werth, Bernhard and Pitzer, Erik and Affenzeller, Michael},
  editor    = {Moreno-Díaz, Roberto and Pichler, Franz and Quesada-Arencibia, Alexis},
  date      = {2020},
  langid    = {english},
  file      = {Full Text PDF:/home/olek/Zotero/storage/TEWLA6VH/Werth et al. - 2020 - Surrogate-Assisted Fitness Landscape Analysis for .pdf:application/pdf}
}

@inproceedings{werth_surrogate-assisted_2020,
  location  = {Cham},
  title     = {Surrogate-Assisted Fitness Landscape Analysis for Computationally Expensive Optimization},
  isbn      = {978-3-030-45093-9},
  doi       = {10.1007/978-3-030-45093-9_30},
  series    = {Lecture Notes in Computer Science},
  abstract  = {Exploratory fitness landscape analysis ({FLA}) is a category of techniques that try to capture knowledge about a black-box optimization problem. This is achieved by assigning features to a certain problem instance utilizing only information obtained by evaluating the black-box. This knowledge can be used to obtain new domain knowledge but more often the intended use is to automatically find an appropriate heuristic optimization algorithm [9]. {FLA}-based algorithm selection and parametrization hinges on the idea, that, while no optimization algorithm can be the optimal choice for all black-box problems, algorithms are expected to work similarly well on problems with similar statistical characteristics [8, 15].},
  pages     = {247--254},
  booktitle = {Computer Aided Systems Theory – {EUROCAST} 2019},
  publisher = {Springer International Publishing},
  author    = {Werth, Bernhard and Pitzer, Erik and Affenzeller, Michael},
  editor    = {Moreno-Díaz, Roberto and Pichler, Franz and Quesada-Arencibia, Alexis},
  date      = {2020},
  langid    = {english},
  file      = {Full Text PDF:/home/olek/Zotero/storage/TEWLA6VH/Werth et al. - 2020 - Surrogate-Assisted Fitness Landscape Analysis for .pdf:application/pdf}
}

@article{westhofen2023criticality,
  title     = {Criticality metrics for automated driving: A review and suitability analysis of the state of the art},
  author    = {Westhofen, Lukas and Neurohr, Christian and Koopmann, Tjark and Butz, Martin and Sch{\"u}tt, Barbara and Utesch, Fabian and Neurohr, Birte and Gutenkunst, Christian and B{\"o}de, Eckard},
  journal   = {Archives of Computational Methods in Engineering},
  volume    = {30},
  number    = {1},
  pages     = {1--35},
  year      = {2023},
  publisher = {Springer}
}

@article{westhofen2023criticality,
  title     = {Criticality metrics for automated driving: A review and suitability analysis of the state of the art},
  author    = {Westhofen, Lukas and Neurohr, Christian and Koopmann, Tjark and Butz, Martin and Sch{\"u}tt, Barbara and Utesch, Fabian and Neurohr, Birte and Gutenkunst, Christian and B{\"o}de, Eckard},
  journal   = {Archives of Computational Methods in Engineering},
  volume    = {30},
  number    = {1},
  pages     = {1--35},
  year      = {2023},
  publisher = {Springer}
}

@inproceedings{wu_practical_2020,
  year       = {2020},
  title      = {Practical Multi-fidelity Bayesian Optimization for Hyperparameter Tuning},
  url        = {https://proceedings.mlr.press/v115/wu20a.html},
  abstract   = {Bayesian optimization is popular for optimizing time-consuming black-box objectives.  Nonetheless, for hyperparameter tuning in deep neural networks, the time required to evaluate the validation error for even a few hyperparameter settings remains a bottleneck.  Multi-fidelity optimization promises relief using cheaper proxies to such objectives —  for example, validation error for a network trained using a subset of the training points or fewer iterations than required for convergence.  We propose a highly flexible and practical approach to multi-fidelity Bayesian optimization, focused on efficiently optimizing hyperparameters for iteratively trained supervised learning models.  We introduce a new acquisition function, the trace-aware knowledge-gradient, which efficiently leverages both multiple continuous fidelity controls and trace observations — values of the objective at a sequence of fidelities, available when varying fidelity using training iterations.  We provide a provably convergent method for optimizing our acquisition function and show it outperforms state-of-the-art alternatives for  hyperparameter tuning of deep neural networks and large-scale kernel learning.},
  eventtitle = {Uncertainty in Artificial Intelligence},
  pages      = {788--798},
  booktitle  = {Proceedings of The 35th Uncertainty in Artificial Intelligence Conference},
  publisher  = {{PMLR}},
  author     = {Wu, Jian and Toscano-Palmerin, Saul and Frazier, Peter I. and Wilson, Andrew Gordon},
  urldate    = {2024-10-02},
  date       = {2020-08-06},
  langid     = {english},
  note       = {{ISSN}: 2640-3498},
  file       = {Full Text PDF:/home/olek/Zotero/storage/JZ4GGV8H/Wu et al. - 2020 - Practical Multi-fidelity Bayesian Optimization for Hyperparameter Tuning.pdf:application/pdf;Supplementary PDF:/home/olek/Zotero/storage/PTEJQ696/Wu et al. - 2020 - Practical Multi-fidelity Bayesian Optimization for Hyperparameter Tuning.pdf:application/pdf}
}

@article{wu2018continuous,
  title  = {Continuous-fidelity Bayesian optimization with knowledge gradient},
  author = {Wu, Jian and Frazier, Peter I},
  year   = {2018}
}

@article{wymann2000torcs,
  title   = {Torcs, the open racing car simulator},
  author  = {Wymann, Bernhard and Espi{\'e}, Eric and Guionneau, Christophe and Dimitrakakis, Christos and Coulom, R{\'e}mi and Sumner, Andrew},
  journal = {Software available at http://torcs. sourceforge. net},
  volume  = {4},
  number  = {6},
  pages   = {2},
  year    = {2000}
}

@article{wymann2000torcs,
  title   = {Torcs, the open racing car simulator},
  author  = {Wymann, Bernhard and Espi{\'e}, Eric and Guionneau, Christophe and Dimitrakakis, Christos and Coulom, R{\'e}mi and Sumner, Andrew},
  journal = {Software available at http://torcs. sourceforge. net},
  volume  = {4},
  number  = {6},
  pages   = {2},
  year    = {2000}
}

@inproceedings{xinxin_csg_2020,
  title      = {{CSG}: Critical Scenario Generation from Real Traffic Accidents},
  url        = {https://ieeexplore.ieee.org/abstract/document/9304609},
  doi        = {10.1109/IV47402.2020.9304609},
  shorttitle = {{CSG}},
  abstract   = {Autonomous driving ({AD}) is getting closer to our life, but the severe traffic accidents of autonomous vehicle ({AV}) happened in the past several years warn us that the safety of {AVs} is still a big challenge for the {AD} industry. Before volume production, the automotive industry and regulators must ensure the {AV} can deal with dangerous scenarios. Although road test is the most common method to test the performance and safety of an {AV}, it has some manifest disadvantages, e.g., highly risky and unrepeatable, low efficiency and lack of useful critical scenarios. Critical-scenario-based simulation can effectively address these problems and become an important complement to road test. In this paper, we present a novel approach to extract critical scenarios from real traffic accident videos and re-generate them in a simulator. We also introduce our integrated toolkit for scenario extraction and scenario test. With the toolkit, we can build a critical scenario library quickly and use it as a benchmark for {AV} safety assessment, among other purposes. On top of this, we further introduce our safety assessment criteria and scoring method.},
  eventtitle = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {1330--1336},
  booktitle  = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Xinxin, Zhang and Fei, Li and Xiangbin, Wu},
  urldate    = {2023-12-07},
  date       = {2020-10},
  year       = {2020},
  note       = {{ISSN}: 2642-7214},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/Y9NAHQWD/Xinxin et al. - 2020 - CSG Critical Scenario Generation from Real Traffi.pdf:application/pdf}
}

@inproceedings{xinxin_csg_2020,
  title      = {{CSG}: Critical Scenario Generation from Real Traffic Accidents},
  url        = {https://ieeexplore.ieee.org/abstract/document/9304609},
  doi        = {10.1109/IV47402.2020.9304609},
  shorttitle = {{CSG}},
  abstract   = {Autonomous driving ({AD}) is getting closer to our life, but the severe traffic accidents of autonomous vehicle ({AV}) happened in the past several years warn us that the safety of {AVs} is still a big challenge for the {AD} industry. Before volume production, the automotive industry and regulators must ensure the {AV} can deal with dangerous scenarios. Although road test is the most common method to test the performance and safety of an {AV}, it has some manifest disadvantages, e.g., highly risky and unrepeatable, low efficiency and lack of useful critical scenarios. Critical-scenario-based simulation can effectively address these problems and become an important complement to road test. In this paper, we present a novel approach to extract critical scenarios from real traffic accident videos and re-generate them in a simulator. We also introduce our integrated toolkit for scenario extraction and scenario test. With the toolkit, we can build a critical scenario library quickly and use it as a benchmark for {AV} safety assessment, among other purposes. On top of this, we further introduce our safety assessment criteria and scoring method.},
  eventtitle = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  pages      = {1330--1336},
  booktitle  = {2020 {IEEE} Intelligent Vehicles Symposium ({IV})},
  author     = {Xinxin, Zhang and Fei, Li and Xiangbin, Wu},
  urldate    = {2023-12-07},
  date       = {2020-10},
  year       = {2020},
  note       = {{ISSN}: 2642-7214},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/Y9NAHQWD/Xinxin et al. - 2020 - CSG Critical Scenario Generation from Real Traffi.pdf:application/pdf}
}

@inproceedings{xu2018attngan,
  title     = {Attngan: Fine-grained text to image generation with attentional generative adversarial networks},
  author    = {Xu, Tao and Zhang, Pengchuan and Huang, Qiuyuan and Zhang, Han and Gan, Zhe and Huang, Xiaolei and He, Xiaodong},
  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages     = {1316--1324},
  year      = {2018}
}

@inproceedings{xu2018attngan,
  title     = {Attngan: Fine-grained text to image generation with attentional generative adversarial networks},
  author    = {Xu, Tao and Zhang, Pengchuan and Huang, Qiuyuan and Zhang, Han and Gan, Zhe and Huang, Xiaolei and He, Xiaodong},
  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages     = {1316--1324},
  year      = {2018}
}

@article{xu2022bits,
  title   = {Bits: Bi-level imitation for traffic simulation},
  author  = {Xu, Danfei and Chen, Yuxiao and Ivanovic, Boris and Pavone, Marco},
  journal = {arXiv preprint arXiv:2208.12403},
  year    = {2022}
}

@article{xu2022bits,
  title   = {Bits: Bi-level imitation for traffic simulation},
  author  = {Xu, Danfei and Chen, Yuxiao and Ivanovic, Boris and Pavone, Marco},
  journal = {arXiv preprint arXiv:2208.12403},
  year    = {2022}
}

@inproceedings{zhang_building_2023,
  location   = {Seattle {WA} {USA}},
  title      = {Building Critical Testing Scenarios for Autonomous Driving from Real Accidents},
  isbn       = {9798400702211},
  url        = {https://dl.acm.org/doi/10.1145/3597926.3598070},
  doi        = {10.1145/3597926.3598070},
  eventtitle = {{ISSTA} '23: 32nd {ACM} {SIGSOFT} International Symposium on Software Testing and Analysis},
  pages      = {462--474},
  booktitle  = {Proceedings of the 32nd {ACM} {SIGSOFT} International Symposium on Software Testing and Analysis},
  publisher  = {{ACM}},
  author     = {Zhang, Xudong and Cai, Yan},
  urldate    = {2023-11-16},
  date       = {2023-07-12},
  year       = {2023},
  langid     = {english},
  keywords   = {Autonomous Driving Systems, Panoptic Segmentation, Parameter Mutation, Scene Recovery, Testing},
  file       = {Full Text PDF:/home/olek/Zotero/storage/89NUPS3E/Zhang and Cai - 2023 - Building Critical Testing Scenarios for Autonomous.pdf:application/pdf}
}

@inproceedings{zhang_building_2023,
  location   = {Seattle {WA} {USA}},
  title      = {Building Critical Testing Scenarios for Autonomous Driving from Real Accidents},
  isbn       = {9798400702211},
  url        = {https://dl.acm.org/doi/10.1145/3597926.3598070},
  doi        = {10.1145/3597926.3598070},
  eventtitle = {{ISSTA} '23: 32nd {ACM} {SIGSOFT} International Symposium on Software Testing and Analysis},
  pages      = {462--474},
  booktitle  = {Proceedings of the 32nd {ACM} {SIGSOFT} International Symposium on Software Testing and Analysis},
  publisher  = {{ACM}},
  author     = {Zhang, Xudong and Cai, Yan},
  urldate    = {2023-11-16},
  date       = {2023-07-12},
  year       = {2023},
  langid     = {english},
  keywords   = {Autonomous Driving Systems, Panoptic Segmentation, Parameter Mutation, Scene Recovery, Testing},
  file       = {Full Text PDF:/home/olek/Zotero/storage/89NUPS3E/Zhang and Cai - 2023 - Building Critical Testing Scenarios for Autonomous.pdf:application/pdf}
}

@article{zhang2023Finding,
  author   = {Zhang, Xinhai and Tao, Jianbo and Tan, Kaige and Törngren, Martin and Sánchez, José Manuel Gaspar and Ramli, Muhammad Rusyadi and Tao, Xin and Gyllenhammar, Magnus and Wotawa, Franz and Mohan, Naveen and Nica, Mihai and Felbinger, Hermann},
  journal  = {IEEE Transactions on Software Engineering},
  title    = {Finding Critical Scenarios for Automated Driving Systems: A Systematic Mapping Study},
  year     = {2023},
  volume   = {49},
  number   = {3},
  pages    = {991-1026},
  keywords = {Taxonomy;Systematics;Bibliographies;Complexity theory;Roads;Uncertainty;Terminology;Critical scenario;automated driving;systematic mapping study},
  doi      = {10.1109/TSE.2022.3170122}
}

@inproceedings{zheng_rapid_2020,
  title      = {Rapid Generation of Challenging Simulation Scenarios for Autonomous Vehicles Based on Adversarial Test},
  url        = {https://ieeexplore.ieee.org/document/9233535},
  doi        = {10.1109/ICMA49215.2020.9233535},
  abstract   = {As the development of the autonomous vehicles, the test and evaluation of the autonomy of autonomous vehicles has receiving considerable attentions from both academic and industrial research fields recently. How to rapid identify the performance boundaries of autonomous vehicles and then to accelerate the testing has been considered as one of the most important problems. Simulation test and challenging scenarios are considered as two effective ways to solve above problems. In this research, a framework of rapid generation of challenging simulation scenarios based on adversarial test was proposed. An optimization model based on newly developed indicators, i.e., safety, smoothness, sharpness and smartness, was proposed to assess the autonomy of autonomous vehicles. Due to the high dimensionality of the configuration space and computation complexity of the optimization problem, a quantum genetic algorithm was employed to solve the optimization problem owing to its high search ability and lower population size required. A simulation system based on Prescan, Carsim and Simulink was built to validate the effectiveness of the proposed method. Comparative experiments also were conducted to illustrate the efficiency of the proposed optimization algorithm.},
  eventtitle = {2020 {IEEE} International Conference on Mechatronics and Automation ({ICMA})},
  pages      = {1166--1172},
  booktitle  = {2020 {IEEE} International Conference on Mechatronics and Automation ({ICMA})},
  author     = {Zheng, Xiaokun and Liang, Huawei and Yu, Biao and Li, Bichun and Wang, Shaobo and Chen, Zhilei},
  urldate    = {2024-01-10},
  date       = {2020-10},
  note       = {{ISSN}: 2152-744X},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/XDQR2W28/9233535.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/EDBC4G95/Zheng et al. - 2020 - Rapid Generation of Challenging Simulation Scenari.pdf:application/pdf}
}

@inproceedings{zheng_rapid_2020,
  title      = {Rapid Generation of Challenging Simulation Scenarios for Autonomous Vehicles Based on Adversarial Test},
  url        = {https://ieeexplore.ieee.org/document/9233535},
  doi        = {10.1109/ICMA49215.2020.9233535},
  abstract   = {As the development of the autonomous vehicles, the test and evaluation of the autonomy of autonomous vehicles has receiving considerable attentions from both academic and industrial research fields recently. How to rapid identify the performance boundaries of autonomous vehicles and then to accelerate the testing has been considered as one of the most important problems. Simulation test and challenging scenarios are considered as two effective ways to solve above problems. In this research, a framework of rapid generation of challenging simulation scenarios based on adversarial test was proposed. An optimization model based on newly developed indicators, i.e., safety, smoothness, sharpness and smartness, was proposed to assess the autonomy of autonomous vehicles. Due to the high dimensionality of the configuration space and computation complexity of the optimization problem, a quantum genetic algorithm was employed to solve the optimization problem owing to its high search ability and lower population size required. A simulation system based on Prescan, Carsim and Simulink was built to validate the effectiveness of the proposed method. Comparative experiments also were conducted to illustrate the efficiency of the proposed optimization algorithm.},
  eventtitle = {2020 {IEEE} International Conference on Mechatronics and Automation ({ICMA})},
  pages      = {1166--1172},
  booktitle  = {2020 {IEEE} International Conference on Mechatronics and Automation ({ICMA})},
  author     = {Zheng, Xiaokun and Liang, Huawei and Yu, Biao and Li, Bichun and Wang, Shaobo and Chen, Zhilei},
  urldate    = {2024-01-10},
  date       = {2020-10},
  note       = {{ISSN}: 2152-744X},
  file       = {IEEE Xplore Abstract Record:/home/olek/Zotero/storage/XDQR2W28/9233535.html:text/html;IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/EDBC4G95/Zheng et al. - 2020 - Rapid Generation of Challenging Simulation Scenari.pdf:application/pdf}
}

@misc{zhong_survey_2021,
  title  = {A Survey on Scenario-Based Testing for Automated Driving Systems in High-Fidelity Simulation},
  url    = {http://arxiv.org/abs/2112.00964},
  author = {Zhong, Ziyuan and Tang, Yun and Zhou, Yuan and Neves, Vania de Oliveira and Liu, Yang and Ray, Baishakhi},
  year   = {2021}
}

@article{zhong2021survey,
  title   = {A survey on scenario-based testing for automated driving systems in high-fidelity simulation},
  author  = {Zhong, Ziyuan and Tang, Yun and Zhou, Yuan and Neves, Vania de Oliveira and Liu, Yang and Ray, Baishakhi},
  journal = {arXiv preprint arXiv:2112.00964},
  year    = {2021}
}

@article{zhou2020smarts,
  title   = {Smarts: Scalable multi-agent reinforcement learning training school for autonomous driving},
  author  = {Zhou, Ming and Luo, Jun and Villella, Julian and Yang, Yaodong and Rusu, David and Miao, Jiayu and Zhang, Weinan and Alban, Montgomery and Fadakar, Iman and Chen, Zheng and others},
  journal = {arXiv preprint arXiv:2010.09776},
  year    = {2020}
}

@article{zhou2020smarts,
  title   = {Smarts: Scalable multi-agent reinforcement learning training school for autonomous driving},
  author  = {Zhou, Ming and Luo, Jun and Villella, Julian and Yang, Yaodong and Rusu, David and Miao, Jiayu and Zhang, Weinan and Alban, Montgomery and Fadakar, Iman and Chen, Zheng and others},
  journal = {arXiv preprint arXiv:2010.09776},
  year    = {2020}
}

@inproceedings{zhou2022strategic,
  title        = {Strategic safety-critical attacks against an advanced driver assistance system},
  author       = {Zhou, Xugui and Schmedding, Anna and Ren, Haotian and Yang, Lishan and Schowitz, Philip and Smirni, Evgenia and Alemzadeh, Homa},
  booktitle    = {2022 52nd Annual IEEE/IFIP International Conference on Dependable Systems and Networks (DSN)},
  pages        = {79--87},
  year         = {2022},
  organization = {IEEE}
}

@inproceedings{zhou2022strategic,
  title        = {Strategic safety-critical attacks against an advanced driver assistance system},
  author       = {Zhou, Xugui and Schmedding, Anna and Ren, Haotian and Yang, Lishan and Schowitz, Philip and Smirni, Evgenia and Alemzadeh, Homa},
  booktitle    = {2022 52nd Annual IEEE/IFIP International Conference on Dependable Systems and Networks (DSN)},
  pages        = {79--87},
  year         = {2022},
  organization = {IEEE}
}

@article{Zimmerlin2024ArXiv,
  title   = {Hidden Biases of End-to-End Driving Datasets},
  author  = {Julian Zimmerlin and Jens Beißwenger and Bernhard Jaeger and Andreas Geiger and Kashyap Chitta},
  journal = {arXiv.org},
  volume  = {2412.09602},
  year    = {2024}
}

@mastersthesis{Zimmerlin2024thesis,
  title        = {Tackling {CARLA} Leaderboard 2.0 with End-to-End Imitation Learning},
  author       = {Julian Zimmerlin},
  school       = {University of Tübingen},
  howpublished = {\textsc{url:}~\url{https://kashyap7x.github.io/assets/pdf/students/Zimmerlin2024.pdf}},
  year         = {2024}
}

@inproceedings{zofka_traffic_2018,
  title      = {Traffic Participants in the Loop: A Mixed Reality-Based Interaction Testbed for the Verification and Validation of Autonomous Vehicles},
  url        = {https://ieeexplore.ieee.org/abstract/document/8569226},
  doi        = {10.1109/ITSC.2018.8569226},
  shorttitle = {Traffic Participants in the Loop},
  abstract   = {In order to verify and validate autonomous vehicles, testbeds integrating the whole system from perception to actuation are necessary. Above all, this applies to the assessment of the autonomous vehicle's performance in the presence of vulnerable road users' behavior: While experiments for the interaction between autonomous vehicles and pedestrians in critical traffic scenarios are hardly conductable in reality, virtual experiments often suffer from plausibility. In order to solve this issue, we present a mixed reality testbed for the verification and validation of autonomous vehicles faced with realistic road user behavior in critical, worst case traffic scenarios. We achieve this by registering an immersed pedestrian and the automated driving function within a common environment model, providing challenging traffic scenarios. The testbed is applicable within different integration levels of the automated driving function and enables a high level of behavioral realism. The testbed is evaluated qualitatively and discussed within a concrete use case.},
  eventtitle = {2018 21st International Conference on Intelligent Transportation Systems ({ITSC})},
  pages      = {3583--3590},
  booktitle  = {2018 21st International Conference on Intelligent Transportation Systems ({ITSC})},
  author     = {Zofka, Marc René and Ulbrich, Stefan and Karl, Daniel and Fleck, Tobias and Kohlhaas, Ralf and Rönnau, Arne and Dillmann, Rüdiger and Zöllner, J. Marius},
  urldate    = {2023-11-29},
  date       = {2018-11},
  note       = {{ISSN}: 2153-0017},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/LUQNANSU/Zofka et al. - 2018 - Traffic Participants in the Loop A Mixed Reality-.pdf:application/pdf}
}

@inproceedings{zofka_traffic_2018,
  title      = {Traffic Participants in the Loop: A Mixed Reality-Based Interaction Testbed for the Verification and Validation of Autonomous Vehicles},
  url        = {https://ieeexplore.ieee.org/abstract/document/8569226},
  doi        = {10.1109/ITSC.2018.8569226},
  shorttitle = {Traffic Participants in the Loop},
  abstract   = {In order to verify and validate autonomous vehicles, testbeds integrating the whole system from perception to actuation are necessary. Above all, this applies to the assessment of the autonomous vehicle's performance in the presence of vulnerable road users' behavior: While experiments for the interaction between autonomous vehicles and pedestrians in critical traffic scenarios are hardly conductable in reality, virtual experiments often suffer from plausibility. In order to solve this issue, we present a mixed reality testbed for the verification and validation of autonomous vehicles faced with realistic road user behavior in critical, worst case traffic scenarios. We achieve this by registering an immersed pedestrian and the automated driving function within a common environment model, providing challenging traffic scenarios. The testbed is applicable within different integration levels of the automated driving function and enables a high level of behavioral realism. The testbed is evaluated qualitatively and discussed within a concrete use case.},
  eventtitle = {2018 21st International Conference on Intelligent Transportation Systems ({ITSC})},
  pages      = {3583--3590},
  booktitle  = {2018 21st International Conference on Intelligent Transportation Systems ({ITSC})},
  author     = {Zofka, Marc René and Ulbrich, Stefan and Karl, Daniel and Fleck, Tobias and Kohlhaas, Ralf and Rönnau, Arne and Dillmann, Rüdiger and Zöllner, J. Marius},
  urldate    = {2023-11-29},
  date       = {2018-11},
  note       = {{ISSN}: 2153-0017},
  file       = {IEEE Xplore Full Text PDF:/home/olek/Zotero/storage/LUQNANSU/Zofka et al. - 2018 - Traffic Participants in the Loop A Mixed Reality-.pdf:application/pdf}
}

@article{zohdinasab_efficient_2023,
  title        = {Efficient and Effective Feature Space Exploration for Testing Deep Learning Systems},
  volume       = {32},
  issn         = {1049-331X},
  url          = {https://dl.acm.org/doi/10.1145/3544792},
  doi          = {10.1145/3544792},
  pages        = {49:1--49:38},
  number       = {2},
  journaltitle = {{ACM} Transactions on Software Engineering and Methodology},
  shortjournal = {{ACM} Trans. Softw. Eng. Methodol.},
  author       = {Zohdinasab, Tahereh and Riccio, Vincenzo and Gambi, Alessio and Tonella, Paolo},
  urldate      = {2023-12-05},
  date         = {2023-03-29},
  keywords     = {self-driving cars, Deep Learning, search based software engineering, Software testing},
  file         = {Full Text PDF:/home/olek/Zotero/storage/FWGI2Z68/Zohdinasab et al. - 2023 - Efficient and Effective Feature Space Exploration .pdf:application/pdf}
}

@article{zohdinasab_efficient_2023,
  title        = {Efficient and Effective Feature Space Exploration for Testing Deep Learning Systems},
  volume       = {32},
  issn         = {1049-331X},
  url          = {https://dl.acm.org/doi/10.1145/3544792},
  doi          = {10.1145/3544792},
  abstract     = {Assessing the quality of Deep Learning ({DL}) systems is crucial, as they are increasingly adopted in safety-critical domains. Researchers have proposed several input generation techniques for {DL} systems. While such techniques can expose failures, they do not explain which features of the test inputs influenced the system’s (mis-) behaviour. {DeepHyperion} was the first test generator to overcome this limitation by exploring the {DL} systems’ feature space at large. In this article, we propose {DeepHyperion}-{CS}, a test generator for {DL} systems that enhances {DeepHyperion} by promoting the inputs that contributed more to feature space exploration during the previous search iterations. We performed an empirical study involving two different test subjects (i.e., a digit classifier and a lane-keeping system for self-driving cars). Our results proved that the contribution-based guidance implemented within {DeepHyperion}-{CS} outperforms state-of-the-art tools and significantly improves the efficiency and the effectiveness of {DeepHyperion}. {DeepHyperion}-{CS} exposed significantly more misbehaviours for five out of six feature combinations and was up to 65\% more efficient than {DeepHyperion} in finding misbehaviour-inducing inputs and exploring the feature space. {DeepHyperion}-{CS} was useful for expanding the datasets used to train the {DL} systems, populating up to 200\% more feature map cells than the original training set.},
  pages        = {49:1--49:38},
  number       = {2},
  journaltitle = {{ACM} Transactions on Software Engineering and Methodology},
  shortjournal = {{ACM} Trans. Softw. Eng. Methodol.},
  author       = {Zohdinasab, Tahereh and Riccio, Vincenzo and Gambi, Alessio and Tonella, Paolo},
  urldate      = {2023-12-05},
  date         = {2023-03-29},
  keywords     = {self-driving cars, Deep Learning, search based software engineering, Software testing},
  file         = {Full Text PDF:/home/olek/Zotero/storage/FWGI2Z68/Zohdinasab et al. - 2023 - Efficient and Effective Feature Space Exploration .pdf:application/pdf}
}