EvolvableRNN/performance.py at master · RubenBroekx/EvolvableRNN · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
import argparse
import multiprocessing as mp
import os
import shutil
from timeit import timeit

from six import iteritems

from config import Config
from environment.cy.env_multi_cy import MultiEnvironmentCy
from environment.env_multi import MultiEnvironment
from main import get_game_ids
from population.population import Population


def simulation(cython: bool = False, parallel: bool = False):
    """Execute a simulation of one training iteration."""
    # Create the population
    cfg = Config()
    cfg.game.duration = 200
    cfg.update()
    pop = Population(
            name="delete_me",
            config=cfg,
            folder_name="test_performance",
            use_backup=True,
            overwrite=True,  # Every iteration, create a new population from scratch
    )

    # Perform the simulations of experiment3
    train_games, _ = get_game_ids(experiment_id=3)
    if cython:
        multi_env = MultiEnvironmentCy(game_config=cfg, pop_config=pop.config)
    else:
        multi_env = MultiEnvironment(game_config=cfg, pop_config=pop.config)
    multi_env.set_games(train_games, noise=True)

    # Prepare the generation's reporters for the generation
    pop.reporters.start_generation(gen=pop.generation, logger=pop.log)

    # Fetch the dictionary of genomes
    genomes = list(iteritems(pop.population))

    if parallel:
        # Initialize the evaluation-pool
        pool = mp.Pool(mp.cpu_count())
        manager = mp.Manager()
        return_dict = manager.dict()

        for genome in genomes:
            pool.apply_async(func=multi_env.eval_genome, args=(genome, return_dict))
        pool.close()  # Close the pool
        pool.join()  # Postpone continuation until everything is finished
    else:
        return_dict = dict()
        for genome in genomes:
            multi_env.eval_genome(genome, return_dict)

    path = f"population_backup/storage/test_performance/"
    if os.path.exists(path):
        shutil.rmtree(path)


def test_drive():
    print("--> Running the drive test <--")

    # Do measures
    N = 100
    time_cy = timeit('drive_test_cy.main()',
                     setup='from tests.cy import drive_test_cy',
                     number=N) / N
    time_py = timeit('drive_test.main()',
                     setup='from tests import drive_test',
                     number=N) / N

    # Print results
    print("Time Python:", time_py)
    print("Time Cython:", time_cy)
    print("Cython is {} times faster on the drive test".format(round(time_py / time_cy, 2)))


def test_simulation():
    print("--> Running the simulation test (may take a while..) <--")

    # Do measures
    N = 20
    time_py = timeit('simulation(False, False)',
                     setup='from performance import simulation',
                     number=N) / N
    time_cy = timeit('simulation(True, False)',
                     setup='from performance import simulation',
                     number=N) / N
    time_py_p = timeit('simulation(False, True)',
                       setup='from performance import simulation',
                       number=N) / N
    time_cy_p = timeit('simulation(True, True)',
                       setup='from performance import simulation',
                       number=N) / N

    # Print results
    print("Time Python:", time_py)
    print("Time Cython:", time_cy)
    print("Time Python - parallel:", time_py_p)
    print("Time Cython - parallel:", time_cy_p)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--eval', type=str, default='all')
    args = parser.parse_args()

    x = args.eval
    # if x == 'all' or x == 'drive':
    #     test_drive()
    if x == 'all' or x == 'simulation':
        test_simulation()