From 5a49ec87eabc3a791a4cb2c6b3ed933b5186e983 Mon Sep 17 00:00:00 2001 From: gabrieledcjr Date: Tue, 11 Apr 2017 17:33:59 -0700 Subject: [PATCH] updated for tensorflow 1.0, support for Mnih 2015 net arch 1) Updated to parallel master branch updated code, 2) Uses tensorflow 1.0, 3) Add support Mnih 2015 network architecture, 4) Create gym wrapper to modify frame skipping and override step method --- a3c.py | 26 +++--- a3c_display.py | 30 +----- a3c_training_thread.py | 69 ++++++-------- a3c_visualize.py | 14 +-- accum_trainer.py | 62 ------------- accum_trainer_test.py | 89 ------------------ constants.py | 11 ++- custom_lstm.py | 108 --------------------- game_ac_network.py | 201 ++++++++++++++++++++++------------------ game_state.py | 70 ++++++++------ game_state_test.py | 6 +- rmsprop_applier.py | 4 +- rmsprop_applier_test.py | 10 +- 13 files changed, 220 insertions(+), 480 deletions(-) delete mode 100644 accum_trainer.py delete mode 100644 accum_trainer_test.py delete mode 100644 custom_lstm.py diff --git a/a3c.py b/a3c.py index 841ee80..42ad5d0 100644 --- a/a3c.py +++ b/a3c.py @@ -49,7 +49,7 @@ def log_uniform(lo, hi, rate): if USE_LSTM: global_network = GameACLSTMNetwork(ACTION_SIZE, -1, device) else: - global_network = GameACFFNetwork(ACTION_SIZE, device) + global_network = GameACFFNetwork(ACTION_SIZE, -1, device) training_threads = [] @@ -74,15 +74,15 @@ def log_uniform(lo, hi, rate): sess = tf.Session(config=tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)) -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() sess.run(init) # summary for tensorboard score_input = tf.placeholder(tf.int32) -tf.scalar_summary("score", score_input) +tf.summary.scalar("score", score_input) -summary_op = tf.merge_all_summaries() -summary_writer = tf.train.SummaryWriter(LOG_FILE, sess.graph_def) +summary_op = tf.summary.merge_all() +summary_writer = tf.summary.FileWriter(LOG_FILE, sess.graph) # init or load checkpoint with saver saver = tf.train.Saver() @@ -106,7 +106,7 @@ def log_uniform(lo, hi, rate): def train_function(parallel_index): global global_t - + training_thread = training_threads[parallel_index] # set start_time start_time = time.time() - wall_t @@ -121,17 +121,17 @@ def train_function(parallel_index): diff_global_t = training_thread.process(sess, global_t, summary_writer, summary_op, score_input) global_t += diff_global_t - - + + def signal_handler(signal, frame): global stop_requested print('You pressed Ctrl+C!') stop_requested = True - + train_threads = [] for i in range(PARALLEL_SIZE): train_threads.append(threading.Thread(target=train_function, args=(i,))) - + signal.signal(signal.SIGINT, signal_handler) # set start time @@ -144,12 +144,12 @@ def signal_handler(signal, frame): signal.pause() print('Now saving data. Please wait') - + for t in train_threads: t.join() if not os.path.exists(CHECKPOINT_DIR): - os.mkdir(CHECKPOINT_DIR) + os.mkdir(CHECKPOINT_DIR) # write wall time wall_t = time.time() - start_time @@ -158,5 +158,3 @@ def signal_handler(signal, frame): f.write(str(wall_t)) saver.save(sess, CHECKPOINT_DIR + '/' + 'checkpoint', global_step = global_t) - - diff --git a/a3c_display.py b/a3c_display.py index a70856a..1461850 100644 --- a/a3c_display.py +++ b/a3c_display.py @@ -18,19 +18,7 @@ from constants import USE_LSTM def choose_action(pi_values): - values = [] - sum = 0.0 - for rate in pi_values: - sum = sum + rate - value = sum - values.append(value) - - r = random.random() * sum - for i in range(len(values)): - if values[i] >= r: - return i; - #fail safe - return len(values)-1 + return np.random.choice(range(len(pi_values)), p=pi_values) # use CPU for display tool device = "/cpu:0" @@ -38,7 +26,7 @@ def choose_action(pi_values): if USE_LSTM: global_network = GameACLSTMNetwork(ACTION_SIZE, -1, device) else: - global_network = GameACFFNetwork(ACTION_SIZE, device) + global_network = GameACFFNetwork(ACTION_SIZE, -1, device) learning_rate_input = tf.placeholder("float") @@ -49,17 +37,8 @@ def choose_action(pi_values): clip_norm = GRAD_NORM_CLIP, device = device) -# training_threads = [] -# for i in range(PARALLEL_SIZE): -# training_thread = A3CTrainingThread(i, global_network, 1.0, -# learning_rate_input, -# grad_applier, -# 8000000, -# device = device) -# training_threads.append(training_thread) - sess = tf.Session() -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver() @@ -70,7 +49,7 @@ def choose_action(pi_values): else: print("Could not find old checkpoint") -game_state = GameState(display=True, no_op_max=0) +game_state = GameState(0, display=True, no_op_max=0) while True: pi_values = global_network.run_policy(sess, game_state.s_t) @@ -82,4 +61,3 @@ def choose_action(pi_values): game_state.reset() else: game_state.update() - diff --git a/a3c_training_thread.py b/a3c_training_thread.py index 2383432..c7d2780 100644 --- a/a3c_training_thread.py +++ b/a3c_training_thread.py @@ -5,7 +5,6 @@ import time import sys -from accum_trainer import AccumTrainer from game_state import GameState from game_state import ACTION_SIZE from game_ac_network import GameACFFNetwork, GameACLSTMNetwork @@ -35,26 +34,26 @@ def __init__(self, if USE_LSTM: self.local_network = GameACLSTMNetwork(ACTION_SIZE, thread_index, device) else: - self.local_network = GameACFFNetwork(ACTION_SIZE, device) + self.local_network = GameACFFNetwork(ACTION_SIZE, thread_index, device) self.local_network.prepare_loss(ENTROPY_BETA) - # TODO: don't need accum trainer anymore with batch - self.trainer = AccumTrainer(device) - self.trainer.prepare_minimize( self.local_network.total_loss, - self.local_network.get_vars() ) - - self.accum_gradients = self.trainer.accumulate_gradients() - self.reset_gradients = self.trainer.reset_gradients() - + with tf.device(device): + var_refs = [v._ref() for v in self.local_network.get_vars()] + self.gradients = tf.gradients( + self.local_network.total_loss, var_refs, + gate_gradients=False, + aggregation_method=None, + colocate_gradients_with_ops=False) + self.apply_gradients = grad_applier.apply_gradients( global_network.get_vars(), - self.trainer.get_accum_grad_list() ) + self.gradients ) self.sync = self.local_network.sync_from(global_network) - + self.game_state = GameState() - + self.local_t = 0 self.initial_learning_rate = initial_learning_rate @@ -71,26 +70,15 @@ def _anneal_learning_rate(self, global_time_step): return learning_rate def choose_action(self, pi_values): - values = [] - sum = 0.0 - for rate in pi_values: - sum = sum + rate - value = sum - values.append(value) - - r = random.random() * sum - for i in range(len(values)): - if values[i] >= r: - return i; - #fail safe - return len(values)-1 + return np.random.choice(range(len(pi_values)), p=pi_values) def _record_score(self, sess, summary_writer, summary_op, score_input, score, global_t): summary_str = sess.run(summary_op, feed_dict={ score_input: score }) summary_writer.add_summary(summary_str, global_t) - + summary_writer.flush() + def set_start_time(self, start_time): self.start_time = start_time @@ -102,9 +90,6 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): terminal_end = False - # reset accumulated gradients - sess.run( self.reset_gradients ) - # copy weights from shared to local sess.run( self.sync ) @@ -112,7 +97,7 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): if USE_LSTM: start_lstm_state = self.local_network.lstm_state_out - + # t_max times loop for i in range(LOCAL_T_MAX): pi_, value_ = self.local_network.run_policy_and_value(sess, self.game_state.s_t) @@ -142,14 +127,14 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): # s_t1 -> s_t self.game_state.update() - + if terminal: terminal_end = True print("score={}".format(self.episode_reward)) self._record_score(sess, summary_writer, summary_op, score_input, self.episode_reward, global_t) - + self.episode_reward = 0 self.game_state.reset() if USE_LSTM: @@ -182,32 +167,31 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): batch_td.append(td) batch_R.append(R) + cur_learning_rate = self._anneal_learning_rate(global_t) + if USE_LSTM: batch_si.reverse() batch_a.reverse() batch_td.reverse() batch_R.reverse() - sess.run( self.accum_gradients, + sess.run( self.apply_gradients, feed_dict = { self.local_network.s: batch_si, self.local_network.a: batch_a, self.local_network.td: batch_td, self.local_network.r: batch_R, self.local_network.initial_lstm_state: start_lstm_state, - self.local_network.step_size : [len(batch_a)] } ) + self.local_network.step_size : [len(batch_a)], + self.learning_rate_input: cur_learning_rate } ) else: - sess.run( self.accum_gradients, + sess.run( self.apply_gradients, feed_dict = { self.local_network.s: batch_si, self.local_network.a: batch_a, self.local_network.td: batch_td, - self.local_network.r: batch_R} ) - - cur_learning_rate = self._anneal_learning_rate(global_t) - - sess.run( self.apply_gradients, - feed_dict = { self.learning_rate_input: cur_learning_rate } ) + self.local_network.r: batch_R, + self.learning_rate_input: cur_learning_rate} ) if (self.thread_index == 0) and (self.local_t - self.prev_local_t >= PERFORMANCE_LOG_INTERVAL): self.prev_local_t += PERFORMANCE_LOG_INTERVAL @@ -219,4 +203,3 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): # return advanced local step size diff_local_t = self.local_t - start_local_t return diff_local_t - diff --git a/a3c_visualize.py b/a3c_visualize.py index babc6cf..1e461e6 100644 --- a/a3c_visualize.py +++ b/a3c_visualize.py @@ -27,7 +27,7 @@ if USE_LSTM: global_network = GameACLSTMNetwork(ACTION_SIZE, -1, device) else: - global_network = GameACFFNetwork(ACTION_SIZE, device) + global_network = GameACFFNetwork(ACTION_SIZE, -1, device) training_threads = [] @@ -40,15 +40,8 @@ clip_norm = GRAD_NORM_CLIP, device = device) -# for i in range(PARALLEL_SIZE): -# training_thread = A3CTrainingThread(i, global_network, 1.0, -# learning_rate_input, -# grad_applier, MAX_TIME_STEP, -# device = device) -# training_threads.append(training_thread) - sess = tf.Session() -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver() @@ -58,7 +51,7 @@ print("checkpoint loaded:", checkpoint.model_checkpoint_path) else: print("Could not find old checkpoint") - + W_conv1 = sess.run(global_network.W_conv1) # show graph of W_conv1 @@ -74,4 +67,3 @@ ax.set_title(str(inch) + "," + str(outch)) plt.show() - diff --git a/accum_trainer.py b/accum_trainer.py deleted file mode 100644 index 3036a61..0000000 --- a/accum_trainer.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- - -import tensorflow as tf - -class AccumTrainer(object): - def __init__(self, - device="/cpu:0", - name="AccumTrainer"): - self._name = name - self._device = device - - def _create_accum_grad(self, var): - """ - Create Variable where to accumulate gradients. - """ - zero = tf.zeros(var.get_shape().as_list(), dtype=var.dtype) - name = var.name.replace(":", "_") + "_accum_grad" - accum_grad = tf.Variable(zero, name=name, trainable=False) - return accum_grad - - def prepare_minimize(self, loss, var_list): - with tf.device(self._device): - var_refs = [v.ref() for v in var_list] - grads = tf.gradients( - loss, var_refs, - gate_gradients=False, - aggregation_method=None, - colocate_gradients_with_ops=False) - - self._var_list = var_list - self._grad_list = grads - self._accum_grad_list = [] - - with tf.control_dependencies(None): - for var in var_list: - accum_grad = self._create_accum_grad(var) - self._accum_grad_list.append(accum_grad) - - def get_accum_grad_list(self): - return self._accum_grad_list - - def accumulate_gradients(self, name=None): - with tf.device(self._device): - accumulate_ops = [] - - with tf.op_scope([], name, self._name) as name: - for var, grad, accum_grad in zip(self._var_list, self._grad_list, self._accum_grad_list): - with tf.name_scope("accum_" + var.op.name): - accumulate_ops.append( tf.assign_add(accum_grad, grad) ) - return tf.group(*accumulate_ops, name=name) - - def reset_gradients(self, name=None): - with tf.device(self._device): - reset_ops = [] - - with tf.op_scope([], name, self._name) as name: - for var, accum_grad in zip(self._var_list, self._accum_grad_list): - with tf.name_scope("reset_" + var.op.name): - zero = tf.zeros(accum_grad.get_shape()) - reset = accum_grad.assign(zero) - reset_ops.append(reset) - return tf.group(*reset_ops, name=name) diff --git a/accum_trainer_test.py b/accum_trainer_test.py deleted file mode 100644 index 109af0c..0000000 --- a/accum_trainer_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- - -import numpy as np -import tensorflow as tf -import accum_trainer - -class AccumTrainerTest(tf.test.TestCase): - def testAccum(self): - with self.test_session(): - var0 = tf.Variable([1.0, 2.0]) - trainer = accum_trainer.AccumTrainer() - - cost = tf.square(var0) - - trainer.prepare_minimize(cost, [var0]) - - accmulate_grad = trainer.accumulate_gradients() - reset = trainer.reset_gradients() - - tf.initialize_all_variables().run() - - # gradの加算を実行 - accmulate_grad.run() - - # accmulate_gradしても、var0の中身は変わらない - self.assertAllClose([1.0, 2.0], var0.eval()) - - accum_grads = trainer._accum_grad_list - accum_grad0 = accum_grads[0] - - # gradがaccum_gradへ加算されているのを確認 - self.assertAllClose([2.0, 4.0], accum_grad0.eval()) - - # gradの加算を再度実行 - accmulate_grad.run() - - # gradがaccum_gradへさらに加算されているのを確認 - self.assertAllClose([4.0, 8.0], accum_grad0.eval()) - - # resetを実行 - reset.run() - - # accum_gradがゼロになっているのを確認 - self.assertAllClose([0.0, 0.0], accum_grad0.eval()) - -<<<<<<< HEAD - def testBatchAccum(self): - with self.test_session(): - x = tf.placeholder("float", shape=(None,1)) - c = tf.constant( [1.0] ) - - var0 = tf.Variable( c ) - - mul = var0 * x - - trainer = accum_trainer.AccumTrainer() - - #cost = tf.square(mul) - cost = tf.reduce_sum( tf.square(mul) ) - - print(cost.get_shape()) - - trainer.prepare_minimize(cost, [var0]) - - accmulate_grad = trainer.accumulate_gradients() - reset = trainer.reset_gradients() - - tf.initialize_all_variables().run() - - si = [ [1.0], [2.0] ] - - # gradの加算を実行 - accmulate_grad.run( feed_dict = { x: si } ) - - # accmulate_gradしても、var0の中身は変わらない - self.assertAllClose([1.0], var0.eval()) - - accum_grads = trainer._accum_grad_list - accum_grad0 = accum_grads[0] - - # gradがaccum_gradへbatchで加算されているのを確認 - t = 2 * 1*1 * 1 + 2 * 2*2 * 1 - - self.assertAllClose([t], accum_grad0.eval()) - - # TODO: gradient clipping test - -if __name__ == "__main__": - tf.test.main() diff --git a/constants.py b/constants.py index 21b7c59..fd845db 100644 --- a/constants.py +++ b/constants.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import gym LOCAL_T_MAX = 5 # repeat step size RMSP_ALPHA = 0.99 # decay parameter for RMSProp @@ -8,9 +9,13 @@ INITIAL_ALPHA_LOW = 1e-4 # log_uniform low limit for learning rate INITIAL_ALPHA_HIGH = 1e-2 # log_uniform high limit for learning rate -PARALLEL_SIZE = 8 # parallel thread size +PARALLEL_SIZE = 16 # parallel thread size GYM_ENV = 'Pong-v0' -ACTION_SIZE = 6 # action size TODO: need to be retrieved from gym env +_env = gym.make(GYM_ENV) +_n_actions = _env.action_space.n +_env.close() +del _env +ACTION_SIZE = _n_actions INITIAL_ALPHA_LOG_RATE = 0.4226 # log_uniform interpolate rate for learning rate (around 7 * 10^-4) GAMMA = 0.99 # discount factor for rewards @@ -19,3 +24,5 @@ GRAD_NORM_CLIP = 40.0 # gradient norm clipping USE_GPU = False # To use GPU, set True USE_LSTM = True # True for A3C LSTM, False for A3C FF + +MNIH_2015 = True # Use Mnih et al [2015] architecture (3 conv layers) diff --git a/custom_lstm.py b/custom_lstm.py deleted file mode 100644 index c77f9f1..0000000 --- a/custom_lstm.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -import tensorflow as tf - -from tensorflow.python.ops.rnn_cell import RNNCell - -class CustomBasicLSTMCell(RNNCell): - """Custom Basic LSTM recurrent network cell. - (Modified to store matrix and bias as member variable.) - - The implementation is based on: http://arxiv.org/abs/1409.2329. - - We add forget_bias (default: 1) to the biases of the forget gate in order to - reduce the scale of forgetting in the beginning of the training. - - It does not allow cell clipping, a projection layer, and does not - use peep-hole connections: it is the basic baseline. - - For advanced models, please use the full LSTMCell that follows. - """ - - def __init__(self, num_units, forget_bias=1.0, input_size=None): - """Initialize the basic LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell. - forget_bias: float, The bias added to forget gates (see above). - input_size: Deprecated and unused. - """ - if input_size is not None: - logging.warn("%s: The input_size parameter is deprecated." % self) - self._num_units = num_units - self._forget_bias = forget_bias - - @property - def state_size(self): - return 2 * self._num_units - - @property - def output_size(self): - return self._num_units - - def __call__(self, inputs, state, scope=None): - """Long short-term memory cell (LSTM).""" - with tf.variable_scope(scope or type(self).__name__): # "BasicLSTMCell" - # Parameters of gates are concatenated into one multiply for efficiency. - c, h = tf.split(1, 2, state) - concat = self._linear([inputs, h], 4 * self._num_units, True) - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - i, j, f, o = tf.split(1, 4, concat) - - new_c = c * tf.sigmoid(f + self._forget_bias) + tf.sigmoid(i) * tf.tanh(j) - new_h = tf.tanh(new_c) * tf.sigmoid(o) - - return new_h, tf.concat(1, [new_c, new_h]) - - def _linear(self, args, output_size, bias, bias_start=0.0, scope=None): - """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. - - Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. - output_size: int, second dimension of W[i]. - bias: boolean, whether to add a bias term or not. - bias_start: starting value to initialize the bias; 0 by default. - scope: VariableScope for the created subgraph; defaults to "Linear". - - Returns: - A 2D Tensor with shape [batch x output_size] equal to - sum_i(args[i] * W[i]), where W[i]s are newly created matrices. - - Raises: - ValueError: if some of the arguments has unspecified or wrong shape. - """ - if args is None or (isinstance(args, (list, tuple)) and not args): - raise ValueError("`args` must be specified") - if not isinstance(args, (list, tuple)): - args = [args] - - # Calculate the total size of arguments on dimension 1. - total_arg_size = 0 - shapes = [a.get_shape().as_list() for a in args] - for shape in shapes: - if len(shape) != 2: - raise ValueError("Linear is expecting 2D arguments: %s" % str(shapes)) - if not shape[1]: - raise ValueError("Linear expects shape[1] of arguments: %s" % str(shapes)) - else: - total_arg_size += shape[1] - - # Now the computation. - with tf.variable_scope(scope or "Linear"): - matrix = tf.get_variable("Matrix", [total_arg_size, output_size]) - if len(args) == 1: - res = tf.matmul(args[0], matrix) - else: - res = tf.matmul(tf.concat(1, args), matrix) - if not bias: - return res - bias_term = tf.get_variable( - "Bias", [output_size], - initializer=tf.constant_initializer(bias_start)) - - # Store as a member for copying. (Customized here) - self.matrix = matrix - self.bias = bias_term - - return res + bias_term - diff --git a/game_ac_network.py b/game_ac_network.py index 7c9c887..68c931f 100644 --- a/game_ac_network.py +++ b/game_ac_network.py @@ -1,37 +1,40 @@ # -*- coding: utf-8 -*- import tensorflow as tf import numpy as np -from custom_lstm import CustomBasicLSTMCell + +from constants import MNIH_2015 # Actor-Critic Network Base Class # (Policy network and Value network) class GameACNetwork(object): def __init__(self, action_size, + thread_index, # -1 for global device="/cpu:0"): - self._device = device self._action_size = action_size + self._thread_index = thread_index + self._device = device def prepare_loss(self, entropy_beta): with tf.device(self._device): # taken action (input for policy) self.a = tf.placeholder("float", [None, self._action_size]) - + # temporary difference (R-V) (input for policy) self.td = tf.placeholder("float", [None]) # avoid NaN with clipping when value in pi becomes zero log_pi = tf.log(tf.clip_by_value(self.pi, 1e-20, 1.0)) - + # policy entropy entropy = -tf.reduce_sum(self.pi * log_pi, reduction_indices=1) - + # policy loss (output) (Adding minus, because the original paper's objective function is for gradient ascent, but we use gradient descent optimizer.) - policy_loss = - tf.reduce_sum( tf.reduce_sum( tf.mul( log_pi, self.a ), reduction_indices=1 ) * self.td + entropy * entropy_beta ) + policy_loss = - tf.reduce_sum( tf.reduce_sum( tf.multiply( log_pi, self.a ), reduction_indices=1 ) * self.td + entropy * entropy_beta ) # R (input for value) self.r = tf.placeholder("float", [None]) - + # value loss (output) # (Learning rate for Critic is half of Actor's, so multiply by 0.5) value_loss = 0.5 * tf.nn.l2_loss(self.r - self.v) @@ -41,12 +44,12 @@ def prepare_loss(self, entropy_beta): def run_policy_and_value(self, sess, s_t): raise NotImplementedError() - + def run_policy(self, sess, s_t): raise NotImplementedError() def run_value(self, sess, s_t): - raise NotImplementedError() + raise NotImplementedError() def get_vars(self): raise NotImplementedError() @@ -58,7 +61,7 @@ def sync_from(self, src_netowrk, name=None): sync_ops = [] with tf.device(self._device): - with tf.op_scope([], name, "GameACNetwork") as name: + with tf.name_scope(name, "GameACNetwork", []) as name: for(src_var, dst_var) in zip(src_vars, dst_vars): sync_op = tf.assign(dst_var, src_var) sync_ops.append(sync_op) @@ -67,29 +70,25 @@ def sync_from(self, src_netowrk, name=None): # weight initialization based on muupan's code # https://github.com/muupan/async-rl/blob/master/a3c_ale.py - def _fc_weight_variable(self, shape): - input_channels = shape[0] - d = 1.0 / np.sqrt(input_channels) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) - - def _fc_bias_variable(self, shape, input_channels): + def _fc_variable(self, weight_shape): + input_channels = weight_shape[0] + output_channels = weight_shape[1] d = 1.0 / np.sqrt(input_channels) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) - - def _conv_weight_variable(self, shape): - w = shape[0] - h = shape[1] - input_channels = shape[2] + bias_shape = [output_channels] + weight = tf.Variable(tf.random_uniform(weight_shape, minval=-d, maxval=d)) + bias = tf.Variable(tf.random_uniform(bias_shape, minval=-d, maxval=d)) + return weight, bias + + def _conv_variable(self, weight_shape): + w = weight_shape[0] + h = weight_shape[1] + input_channels = weight_shape[2] + output_channels = weight_shape[3] d = 1.0 / np.sqrt(input_channels * w * h) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) - - def _conv_bias_variable(self, shape, w, h, input_channels): - d = 1.0 / np.sqrt(input_channels * w * h) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) + bias_shape = [output_channels] + weight = tf.Variable(tf.random_uniform(weight_shape, minval=-d, maxval=d)) + bias = tf.Variable(tf.random_uniform(bias_shape, minval=-d, maxval=d)) + return weight, bias def _conv2d(self, x, W, stride): return tf.nn.conv2d(x, W, strides = [1, stride, stride, 1], padding = "VALID") @@ -98,35 +97,44 @@ def _conv2d(self, x, W, stride): class GameACFFNetwork(GameACNetwork): def __init__(self, action_size, + thread_index, # -1 for global device="/cpu:0"): - GameACNetwork.__init__(self, action_size, device) - - with tf.device(self._device): - self.W_conv1 = self._conv_weight_variable([8, 8, 4, 16]) # stride=4 - self.b_conv1 = self._conv_bias_variable([16], 8, 8, 4) - - self.W_conv2 = self._conv_weight_variable([4, 4, 16, 32]) # stride=2 - self.b_conv2 = self._conv_bias_variable([32], 4, 4, 16) - - self.W_fc1 = self._fc_weight_variable([2592, 256]) - self.b_fc1 = self._fc_bias_variable([256], 2592) + GameACNetwork.__init__(self, action_size, thread_index, device) + + scope_name = "net_" + str(self._thread_index) + with tf.device(self._device), tf.variable_scope(scope_name) as scope: + if MNIH_2015: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 32]) + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 32, 64]) + self.W_conv3, self.b_conv3 = self._conv_variable([3, 3, 64, 64]) + self.W_fc1, self.b_fc1 = self._fc_variable([3136, 256]) + else: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 16]) # stride=4 + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 16, 32]) # stride=2 + self.W_fc1, self.b_fc1 = self._fc_variable([2592, 256]) # weight for policy output layer - self.W_fc2 = self._fc_weight_variable([256, action_size]) - self.b_fc2 = self._fc_bias_variable([action_size], 256) + self.W_fc2, self.b_fc2 = self._fc_variable([256, action_size]) # weight for value output layer - self.W_fc3 = self._fc_weight_variable([256, 1]) - self.b_fc3 = self._fc_bias_variable([1], 256) + self.W_fc3, self.b_fc3 = self._fc_variable([256, 1]) # state (input) self.s = tf.placeholder("float", [None, 84, 84, 4]) - - h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) - h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) - h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) - h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) + if MNIH_2015: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + h_conv3 = tf.nn.relu(self._conv2d(h_conv2, self.W_conv3, 1) + self.b_conv3) + + h_conv3_flat = tf.reshape(h_conv3, [-1, 3136]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv3_flat, self.W_fc1) + self.b_fc1) + else: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + + h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) # policy (output) self.pi = tf.nn.softmax(tf.matmul(h_fc1, self.W_fc2) + self.b_fc2) @@ -159,48 +167,55 @@ def __init__(self, action_size, thread_index, # -1 for global device="/cpu:0" ): - GameACNetwork.__init__(self, action_size, device) - - with tf.device(self._device): - self.W_conv1 = self._conv_weight_variable([8, 8, 4, 16]) # stride=4 - self.b_conv1 = self._conv_bias_variable([16], 8, 8, 4) - - self.W_conv2 = self._conv_weight_variable([4, 4, 16, 32]) # stride=2 - self.b_conv2 = self._conv_bias_variable([32], 4, 4, 16) - - self.W_fc1 = self._fc_weight_variable([2592, 256]) - self.b_fc1 = self._fc_bias_variable([256], 2592) + GameACNetwork.__init__(self, action_size, thread_index, device) + + scope_name = "net_" + str(self._thread_index) + with tf.device(self._device), tf.variable_scope(scope_name) as scope: + if MNIH_2015: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 32]) + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 32, 64]) + self.W_conv3, self.b_conv3 = self._conv_variable([3, 3, 64, 64]) + self.W_fc1, self.b_fc1 = self._fc_variable([3136, 256]) + else: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 16]) # stride=4 + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 16, 32]) # stride=2 + self.W_fc1, self.b_fc1 = self._fc_variable([2592, 256]) # lstm - self.lstm = CustomBasicLSTMCell(256) + self.lstm = tf.contrib.rnn.BasicLSTMCell(256, state_is_tuple=True) # weight for policy output layer - self.W_fc2 = self._fc_weight_variable([256, action_size]) - self.b_fc2 = self._fc_bias_variable([action_size], 256) + self.W_fc2, self.b_fc2 = self._fc_variable([256, action_size]) # weight for value output layer - self.W_fc3 = self._fc_weight_variable([256, 1]) - self.b_fc3 = self._fc_bias_variable([1], 256) + self.W_fc3, self.b_fc3 = self._fc_variable([256, 1]) # state (input) self.s = tf.placeholder("float", [None, 84, 84, 4]) - - h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) - h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) - h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) - h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) - # h_fc1 shape=(5,256) + if MNIH_2015: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + h_conv3 = tf.nn.relu(self._conv2d(h_conv2, self.W_conv3, 1) + self.b_conv3) + + h_conv3_flat = tf.reshape(h_conv3, [-1, 3136]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv3_flat, self.W_fc1) + self.b_fc1) + else: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + + h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) h_fc1_reshaped = tf.reshape(h_fc1, [1,-1,256]) - # h_fc_reshaped = (1,5,256) # place holder for LSTM unrolling time step size. self.step_size = tf.placeholder(tf.float32, [1]) - self.initial_lstm_state = tf.placeholder(tf.float32, [1, self.lstm.state_size]) - - scope = "net_" + str(thread_index) + self.initial_lstm_state0 = tf.placeholder(tf.float32, [1, 256]) + self.initial_lstm_state1 = tf.placeholder(tf.float32, [1, 256]) + self.initial_lstm_state = tf.contrib.rnn.LSTMStateTuple(self.initial_lstm_state0, + self.initial_lstm_state1) # Unrolling LSTM up to LOCAL_T_MAX time steps. (= 5time steps.) # When episode terminates unrolling time steps becomes less than LOCAL_TIME_STEP. @@ -215,51 +230,59 @@ def __init__(self, scope = scope) # lstm_outputs: (1,5,256) for back prop, (1,1,256) for forward prop. - + lstm_outputs = tf.reshape(lstm_outputs, [-1,256]) # policy (output) self.pi = tf.nn.softmax(tf.matmul(lstm_outputs, self.W_fc2) + self.b_fc2) - + # value (output) v_ = tf.matmul(lstm_outputs, self.W_fc3) + self.b_fc3 self.v = tf.reshape( v_, [-1] ) + scope.reuse_variables() + self.W_lstm = tf.get_variable("basic_lstm_cell/weights") + self.b_lstm = tf.get_variable("basic_lstm_cell/biases") + self.reset_state() - + def reset_state(self): - self.lstm_state_out = np.zeros([1, self.lstm.state_size]) + self.lstm_state_out = tf.contrib.rnn.LSTMStateTuple(np.zeros([1, 256]), + np.zeros([1, 256])) def run_policy_and_value(self, sess, s_t): # This run_policy_and_value() is used when forward propagating. # so the step size is 1. pi_out, v_out, self.lstm_state_out = sess.run( [self.pi, self.v, self.lstm_state], feed_dict = {self.s : [s_t], - self.initial_lstm_state : self.lstm_state_out, + self.initial_lstm_state0 : self.lstm_state_out[0], + self.initial_lstm_state1 : self.lstm_state_out[1], self.step_size : [1]} ) # pi_out: (1,3), v_out: (1) return (pi_out[0], v_out[0]) def run_policy(self, sess, s_t): - # This run_policy() is used for displaying the result with display tool. + # This run_policy() is used for displaying the result with display tool. pi_out, self.lstm_state_out = sess.run( [self.pi, self.lstm_state], feed_dict = {self.s : [s_t], - self.initial_lstm_state : self.lstm_state_out, + self.initial_lstm_state0 : self.lstm_state_out[0], + self.initial_lstm_state1 : self.lstm_state_out[1], self.step_size : [1]} ) - + return pi_out[0] def run_value(self, sess, s_t): - # This run_value() is used for calculating V for bootstrapping at the + # This run_value() is used for calculating V for bootstrapping at the # end of LOCAL_T_MAX time step sequence. # When next sequcen starts, V will be calculated again with the same state using updated network weights, # so we don't update LSTM state here. prev_lstm_state_out = self.lstm_state_out v_out, _ = sess.run( [self.v, self.lstm_state], feed_dict = {self.s : [s_t], - self.initial_lstm_state : self.lstm_state_out, + self.initial_lstm_state0 : self.lstm_state_out[0], + self.initial_lstm_state1 : self.lstm_state_out[1], self.step_size : [1]} ) - + # roll back lstm state self.lstm_state_out = prev_lstm_state_out return v_out[0] @@ -268,6 +291,6 @@ def get_vars(self): return [self.W_conv1, self.b_conv1, self.W_conv2, self.b_conv2, self.W_fc1, self.b_fc1, - self.lstm.matrix, self.lstm.bias, + self.W_lstm, self.b_lstm, self.W_fc2, self.b_fc2, self.W_fc3, self.b_fc3] diff --git a/game_state.py b/game_state.py index a8e5ffe..9bbfdde 100644 --- a/game_state.py +++ b/game_state.py @@ -3,12 +3,33 @@ import numpy as np import gym -import skimage.color -import skimage.transform +import cv2 +import atari_py from constants import GYM_ENV from constants import ACTION_SIZE +class AtariEnvSkipping(gym.Wrapper): + def __init__(self, env, frameskip=4): + self.env = env + self.env.ale.setFloat('frame_skip'.encode('utf-8'), frameskip) + self.env.ale.setFloat('repeat_action_probability'.encode('utf-8'), 0.0) + self.env._seed() + + print ("lives={}".format(self.env.ale.lives())) + print ("frameskip={}".format(self.env.ale.getFloat(b'frame_skip'))) + print ("repeat_action_probability={}".format(self.env.ale.getFloat(b'repeat_action_probability'))) + print ("action space={}".format(self.env.action_space.n)) + + def _step(self, a): + reward = 0.0 + action = self.env._action_set[a] + + reward += self.env.ale.act(action) + ob = self.env._get_obs() + + return ob, reward, self.env.ale.game_over(), {"ale.lives": self.env.ale.lives()} + class GameState(object): def __init__(self, display=False, crop_screen=True, frame_skip=4, no_op_max=30): self._display = display @@ -17,43 +38,40 @@ def __init__(self, display=False, crop_screen=True, frame_skip=4, no_op_max=30): if self._frame_skip < 1: self._frame_skip = 1 self._no_op_max = no_op_max - + self.env_id = GYM_ENV + self.env = gym.make(GYM_ENV) + self.env = AtariEnvSkipping(self.env, frameskip=self._frame_skip) - #print "action space=", self.env.action_space - self.reset() - + def _process_frame(self, action, reshape): reward = 0 - for i in range(self._frame_skip): - observation, r, terminal, _ = self.env.step(action) - reward += r - if terminal: - break - # observation shape = (210, 160, 3) - - grayscale_observation = skimage.color.rgb2gray(observation) - # shape (210, 160) range = [0.0, 1.0] + observation, r, terminal, _ = self.env.step(action) + reward += r + + grayscale_observation = cv2.cvtColor(observation, cv2.COLOR_RGB2GRAY) if self._crop_screen: - # resize to height=110, width=84 - resized_observation = skimage.transform.resize(grayscale_observation, (110, 84)) - resized_observation = resized_observation.astype(np.float32) + resized_observation = grayscale_observation.astype(np.float32) # crop to fit 84x84 - x_t = resized_observation[18:102,:] + x_t = resized_observation[34:34+160, :160] + x_t = cv2.resize(x_t, (84, 84)) else: # resize to height=84, width=84 - resized_observation = skimage.transform.resize(grayscale_observation, (84, 84)) + resized_observation = cv2.resize(grayscale_observation, (84,84)) x_t = resized_observation.astype(np.float32) if reshape: x_t = np.reshape(x_t, (84, 84, 1)) + + # normalize + x_t *= (1.0/255.0) return reward, terminal, x_t - + def reset(self): self.env.reset() - + # randomize initial state if self._no_op_max > 0: no_op = np.random.randint(0, self._no_op_max + 1) @@ -61,20 +79,20 @@ def reset(self): self.env.step(0) _, _, x_t = self._process_frame(0, False) - + self.reward = 0 self.terminal = False self.s_t = np.stack((x_t, x_t, x_t, x_t), axis = 2) - + def process(self, action): if self._display: self.env.render() - + r, t, x_t1 = self._process_frame(action, True) self.reward = r self.terminal = t - self.s_t1 = np.append(self.s_t[:,:,1:], x_t1, axis = 2) + self.s_t1 = np.append(self.s_t[:,:,1:], x_t1, axis = 2) def update(self): self.s_t = self.s_t1 diff --git a/game_state_test.py b/game_state_test.py index 44750eb..ca3e78e 100644 --- a/game_state_test.py +++ b/game_state_test.py @@ -7,9 +7,9 @@ class TestSequenceFunctions(unittest.TestCase): def test_process(self): game_state = GameState(0) - + before_s_t = np.array( game_state.s_t ) - + for i in range(1000): bef1 = game_state.s_t[:,:,1] bef2 = game_state.s_t[:,:,2] @@ -17,7 +17,7 @@ def test_process(self): game_state.process(1) game_state.update() - + aft0 = game_state.s_t[:,:,0] aft1 = game_state.s_t[:,:,1] aft2 = game_state.s_t[:,:,2] diff --git a/rmsprop_applier.py b/rmsprop_applier.py index b73ad14..0d92149 100644 --- a/rmsprop_applier.py +++ b/rmsprop_applier.py @@ -75,7 +75,7 @@ def _zeros_slot(self, var, slot_name, op_name): # TODO: in RMSProp native code, memcpy() (for CPU) and # cudaMemcpyAsync() (for GPU) are used when updating values, # and values might tend to be overwritten with results from other threads. - # (Need to check the learning performance with replacing it) + # (Need to check the learning performance with replacing it) def _apply_dense(self, grad, var): rms = self.get_slot(var, "rms") mom = self.get_slot(var, "momentum") @@ -96,7 +96,7 @@ def apply_gradients(self, var_list, accum_grad_list, name=None): with tf.control_dependencies(None): self._create_slots(var_list) - with tf.op_scope([], name, self._name) as name: + with tf.name_scope(name, self._name, []) as name: self._prepare() for var, accum_grad in zip(var_list, accum_grad_list): with tf.name_scope("update_" + var.op.name), tf.device(var.device): diff --git a/rmsprop_applier_test.py b/rmsprop_applier_test.py index a5e1c06..00e9cf3 100644 --- a/rmsprop_applier_test.py +++ b/rmsprop_applier_test.py @@ -9,15 +9,15 @@ class RMSPropApplierTest(tf.test.TestCase): def testApply(self): with self.test_session(): var = tf.Variable([1.0, 2.0]) - + grad0 = tf.Variable([2.0, 4.0]) grad1 = tf.Variable([3.0, 6.0]) - + opt = rmsprop_applier.RMSPropApplier(learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0) - + apply_gradient0 = opt.apply_gradients([var], [grad0]) apply_gradient1 = opt.apply_gradients([var], [grad1]) @@ -49,8 +49,8 @@ def testApply(self): ms_y = ms_y + (dy * dy - ms_y) * (1.0 - 0.9) x = x - (2.0 * dx / math.sqrt(ms_x+1.0)) y = y - (2.0 * dy / math.sqrt(ms_y+1.0)) - + self.assertAllClose(np.array([x, y]), var.eval()) - + if __name__ == "__main__": tf.test.main()