diff --git a/a3c.py b/a3c.py index 841ee80..42ad5d0 100644 --- a/a3c.py +++ b/a3c.py @@ -49,7 +49,7 @@ def log_uniform(lo, hi, rate): if USE_LSTM: global_network = GameACLSTMNetwork(ACTION_SIZE, -1, device) else: - global_network = GameACFFNetwork(ACTION_SIZE, device) + global_network = GameACFFNetwork(ACTION_SIZE, -1, device) training_threads = [] @@ -74,15 +74,15 @@ def log_uniform(lo, hi, rate): sess = tf.Session(config=tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)) -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() sess.run(init) # summary for tensorboard score_input = tf.placeholder(tf.int32) -tf.scalar_summary("score", score_input) +tf.summary.scalar("score", score_input) -summary_op = tf.merge_all_summaries() -summary_writer = tf.train.SummaryWriter(LOG_FILE, sess.graph_def) +summary_op = tf.summary.merge_all() +summary_writer = tf.summary.FileWriter(LOG_FILE, sess.graph) # init or load checkpoint with saver saver = tf.train.Saver() @@ -106,7 +106,7 @@ def log_uniform(lo, hi, rate): def train_function(parallel_index): global global_t - + training_thread = training_threads[parallel_index] # set start_time start_time = time.time() - wall_t @@ -121,17 +121,17 @@ def train_function(parallel_index): diff_global_t = training_thread.process(sess, global_t, summary_writer, summary_op, score_input) global_t += diff_global_t - - + + def signal_handler(signal, frame): global stop_requested print('You pressed Ctrl+C!') stop_requested = True - + train_threads = [] for i in range(PARALLEL_SIZE): train_threads.append(threading.Thread(target=train_function, args=(i,))) - + signal.signal(signal.SIGINT, signal_handler) # set start time @@ -144,12 +144,12 @@ def signal_handler(signal, frame): signal.pause() print('Now saving data. Please wait') - + for t in train_threads: t.join() if not os.path.exists(CHECKPOINT_DIR): - os.mkdir(CHECKPOINT_DIR) + os.mkdir(CHECKPOINT_DIR) # write wall time wall_t = time.time() - start_time @@ -158,5 +158,3 @@ def signal_handler(signal, frame): f.write(str(wall_t)) saver.save(sess, CHECKPOINT_DIR + '/' + 'checkpoint', global_step = global_t) - - diff --git a/a3c_display.py b/a3c_display.py index a70856a..1461850 100644 --- a/a3c_display.py +++ b/a3c_display.py @@ -18,19 +18,7 @@ from constants import USE_LSTM def choose_action(pi_values): - values = [] - sum = 0.0 - for rate in pi_values: - sum = sum + rate - value = sum - values.append(value) - - r = random.random() * sum - for i in range(len(values)): - if values[i] >= r: - return i; - #fail safe - return len(values)-1 + return np.random.choice(range(len(pi_values)), p=pi_values) # use CPU for display tool device = "/cpu:0" @@ -38,7 +26,7 @@ def choose_action(pi_values): if USE_LSTM: global_network = GameACLSTMNetwork(ACTION_SIZE, -1, device) else: - global_network = GameACFFNetwork(ACTION_SIZE, device) + global_network = GameACFFNetwork(ACTION_SIZE, -1, device) learning_rate_input = tf.placeholder("float") @@ -49,17 +37,8 @@ def choose_action(pi_values): clip_norm = GRAD_NORM_CLIP, device = device) -# training_threads = [] -# for i in range(PARALLEL_SIZE): -# training_thread = A3CTrainingThread(i, global_network, 1.0, -# learning_rate_input, -# grad_applier, -# 8000000, -# device = device) -# training_threads.append(training_thread) - sess = tf.Session() -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver() @@ -70,7 +49,7 @@ def choose_action(pi_values): else: print("Could not find old checkpoint") -game_state = GameState(display=True, no_op_max=0) +game_state = GameState(0, display=True, no_op_max=0) while True: pi_values = global_network.run_policy(sess, game_state.s_t) @@ -82,4 +61,3 @@ def choose_action(pi_values): game_state.reset() else: game_state.update() - diff --git a/a3c_training_thread.py b/a3c_training_thread.py index 2383432..c7d2780 100644 --- a/a3c_training_thread.py +++ b/a3c_training_thread.py @@ -5,7 +5,6 @@ import time import sys -from accum_trainer import AccumTrainer from game_state import GameState from game_state import ACTION_SIZE from game_ac_network import GameACFFNetwork, GameACLSTMNetwork @@ -35,26 +34,26 @@ def __init__(self, if USE_LSTM: self.local_network = GameACLSTMNetwork(ACTION_SIZE, thread_index, device) else: - self.local_network = GameACFFNetwork(ACTION_SIZE, device) + self.local_network = GameACFFNetwork(ACTION_SIZE, thread_index, device) self.local_network.prepare_loss(ENTROPY_BETA) - # TODO: don't need accum trainer anymore with batch - self.trainer = AccumTrainer(device) - self.trainer.prepare_minimize( self.local_network.total_loss, - self.local_network.get_vars() ) - - self.accum_gradients = self.trainer.accumulate_gradients() - self.reset_gradients = self.trainer.reset_gradients() - + with tf.device(device): + var_refs = [v._ref() for v in self.local_network.get_vars()] + self.gradients = tf.gradients( + self.local_network.total_loss, var_refs, + gate_gradients=False, + aggregation_method=None, + colocate_gradients_with_ops=False) + self.apply_gradients = grad_applier.apply_gradients( global_network.get_vars(), - self.trainer.get_accum_grad_list() ) + self.gradients ) self.sync = self.local_network.sync_from(global_network) - + self.game_state = GameState() - + self.local_t = 0 self.initial_learning_rate = initial_learning_rate @@ -71,26 +70,15 @@ def _anneal_learning_rate(self, global_time_step): return learning_rate def choose_action(self, pi_values): - values = [] - sum = 0.0 - for rate in pi_values: - sum = sum + rate - value = sum - values.append(value) - - r = random.random() * sum - for i in range(len(values)): - if values[i] >= r: - return i; - #fail safe - return len(values)-1 + return np.random.choice(range(len(pi_values)), p=pi_values) def _record_score(self, sess, summary_writer, summary_op, score_input, score, global_t): summary_str = sess.run(summary_op, feed_dict={ score_input: score }) summary_writer.add_summary(summary_str, global_t) - + summary_writer.flush() + def set_start_time(self, start_time): self.start_time = start_time @@ -102,9 +90,6 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): terminal_end = False - # reset accumulated gradients - sess.run( self.reset_gradients ) - # copy weights from shared to local sess.run( self.sync ) @@ -112,7 +97,7 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): if USE_LSTM: start_lstm_state = self.local_network.lstm_state_out - + # t_max times loop for i in range(LOCAL_T_MAX): pi_, value_ = self.local_network.run_policy_and_value(sess, self.game_state.s_t) @@ -142,14 +127,14 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): # s_t1 -> s_t self.game_state.update() - + if terminal: terminal_end = True print("score={}".format(self.episode_reward)) self._record_score(sess, summary_writer, summary_op, score_input, self.episode_reward, global_t) - + self.episode_reward = 0 self.game_state.reset() if USE_LSTM: @@ -182,32 +167,31 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): batch_td.append(td) batch_R.append(R) + cur_learning_rate = self._anneal_learning_rate(global_t) + if USE_LSTM: batch_si.reverse() batch_a.reverse() batch_td.reverse() batch_R.reverse() - sess.run( self.accum_gradients, + sess.run( self.apply_gradients, feed_dict = { self.local_network.s: batch_si, self.local_network.a: batch_a, self.local_network.td: batch_td, self.local_network.r: batch_R, self.local_network.initial_lstm_state: start_lstm_state, - self.local_network.step_size : [len(batch_a)] } ) + self.local_network.step_size : [len(batch_a)], + self.learning_rate_input: cur_learning_rate } ) else: - sess.run( self.accum_gradients, + sess.run( self.apply_gradients, feed_dict = { self.local_network.s: batch_si, self.local_network.a: batch_a, self.local_network.td: batch_td, - self.local_network.r: batch_R} ) - - cur_learning_rate = self._anneal_learning_rate(global_t) - - sess.run( self.apply_gradients, - feed_dict = { self.learning_rate_input: cur_learning_rate } ) + self.local_network.r: batch_R, + self.learning_rate_input: cur_learning_rate} ) if (self.thread_index == 0) and (self.local_t - self.prev_local_t >= PERFORMANCE_LOG_INTERVAL): self.prev_local_t += PERFORMANCE_LOG_INTERVAL @@ -219,4 +203,3 @@ def process(self, sess, global_t, summary_writer, summary_op, score_input): # return advanced local step size diff_local_t = self.local_t - start_local_t return diff_local_t - diff --git a/a3c_visualize.py b/a3c_visualize.py index babc6cf..1e461e6 100644 --- a/a3c_visualize.py +++ b/a3c_visualize.py @@ -27,7 +27,7 @@ if USE_LSTM: global_network = GameACLSTMNetwork(ACTION_SIZE, -1, device) else: - global_network = GameACFFNetwork(ACTION_SIZE, device) + global_network = GameACFFNetwork(ACTION_SIZE, -1, device) training_threads = [] @@ -40,15 +40,8 @@ clip_norm = GRAD_NORM_CLIP, device = device) -# for i in range(PARALLEL_SIZE): -# training_thread = A3CTrainingThread(i, global_network, 1.0, -# learning_rate_input, -# grad_applier, MAX_TIME_STEP, -# device = device) -# training_threads.append(training_thread) - sess = tf.Session() -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() sess.run(init) saver = tf.train.Saver() @@ -58,7 +51,7 @@ print("checkpoint loaded:", checkpoint.model_checkpoint_path) else: print("Could not find old checkpoint") - + W_conv1 = sess.run(global_network.W_conv1) # show graph of W_conv1 @@ -74,4 +67,3 @@ ax.set_title(str(inch) + "," + str(outch)) plt.show() - diff --git a/accum_trainer.py b/accum_trainer.py deleted file mode 100644 index 3036a61..0000000 --- a/accum_trainer.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- - -import tensorflow as tf - -class AccumTrainer(object): - def __init__(self, - device="/cpu:0", - name="AccumTrainer"): - self._name = name - self._device = device - - def _create_accum_grad(self, var): - """ - Create Variable where to accumulate gradients. - """ - zero = tf.zeros(var.get_shape().as_list(), dtype=var.dtype) - name = var.name.replace(":", "_") + "_accum_grad" - accum_grad = tf.Variable(zero, name=name, trainable=False) - return accum_grad - - def prepare_minimize(self, loss, var_list): - with tf.device(self._device): - var_refs = [v.ref() for v in var_list] - grads = tf.gradients( - loss, var_refs, - gate_gradients=False, - aggregation_method=None, - colocate_gradients_with_ops=False) - - self._var_list = var_list - self._grad_list = grads - self._accum_grad_list = [] - - with tf.control_dependencies(None): - for var in var_list: - accum_grad = self._create_accum_grad(var) - self._accum_grad_list.append(accum_grad) - - def get_accum_grad_list(self): - return self._accum_grad_list - - def accumulate_gradients(self, name=None): - with tf.device(self._device): - accumulate_ops = [] - - with tf.op_scope([], name, self._name) as name: - for var, grad, accum_grad in zip(self._var_list, self._grad_list, self._accum_grad_list): - with tf.name_scope("accum_" + var.op.name): - accumulate_ops.append( tf.assign_add(accum_grad, grad) ) - return tf.group(*accumulate_ops, name=name) - - def reset_gradients(self, name=None): - with tf.device(self._device): - reset_ops = [] - - with tf.op_scope([], name, self._name) as name: - for var, accum_grad in zip(self._var_list, self._accum_grad_list): - with tf.name_scope("reset_" + var.op.name): - zero = tf.zeros(accum_grad.get_shape()) - reset = accum_grad.assign(zero) - reset_ops.append(reset) - return tf.group(*reset_ops, name=name) diff --git a/accum_trainer_test.py b/accum_trainer_test.py deleted file mode 100644 index 109af0c..0000000 --- a/accum_trainer_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- - -import numpy as np -import tensorflow as tf -import accum_trainer - -class AccumTrainerTest(tf.test.TestCase): - def testAccum(self): - with self.test_session(): - var0 = tf.Variable([1.0, 2.0]) - trainer = accum_trainer.AccumTrainer() - - cost = tf.square(var0) - - trainer.prepare_minimize(cost, [var0]) - - accmulate_grad = trainer.accumulate_gradients() - reset = trainer.reset_gradients() - - tf.initialize_all_variables().run() - - # gradの加算を実行 - accmulate_grad.run() - - # accmulate_gradしても、var0の中身は変わらない - self.assertAllClose([1.0, 2.0], var0.eval()) - - accum_grads = trainer._accum_grad_list - accum_grad0 = accum_grads[0] - - # gradがaccum_gradへ加算されているのを確認 - self.assertAllClose([2.0, 4.0], accum_grad0.eval()) - - # gradの加算を再度実行 - accmulate_grad.run() - - # gradがaccum_gradへさらに加算されているのを確認 - self.assertAllClose([4.0, 8.0], accum_grad0.eval()) - - # resetを実行 - reset.run() - - # accum_gradがゼロになっているのを確認 - self.assertAllClose([0.0, 0.0], accum_grad0.eval()) - -<<<<<<< HEAD - def testBatchAccum(self): - with self.test_session(): - x = tf.placeholder("float", shape=(None,1)) - c = tf.constant( [1.0] ) - - var0 = tf.Variable( c ) - - mul = var0 * x - - trainer = accum_trainer.AccumTrainer() - - #cost = tf.square(mul) - cost = tf.reduce_sum( tf.square(mul) ) - - print(cost.get_shape()) - - trainer.prepare_minimize(cost, [var0]) - - accmulate_grad = trainer.accumulate_gradients() - reset = trainer.reset_gradients() - - tf.initialize_all_variables().run() - - si = [ [1.0], [2.0] ] - - # gradの加算を実行 - accmulate_grad.run( feed_dict = { x: si } ) - - # accmulate_gradしても、var0の中身は変わらない - self.assertAllClose([1.0], var0.eval()) - - accum_grads = trainer._accum_grad_list - accum_grad0 = accum_grads[0] - - # gradがaccum_gradへbatchで加算されているのを確認 - t = 2 * 1*1 * 1 + 2 * 2*2 * 1 - - self.assertAllClose([t], accum_grad0.eval()) - - # TODO: gradient clipping test - -if __name__ == "__main__": - tf.test.main() diff --git a/constants.py b/constants.py index 21b7c59..fd845db 100644 --- a/constants.py +++ b/constants.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import gym LOCAL_T_MAX = 5 # repeat step size RMSP_ALPHA = 0.99 # decay parameter for RMSProp @@ -8,9 +9,13 @@ INITIAL_ALPHA_LOW = 1e-4 # log_uniform low limit for learning rate INITIAL_ALPHA_HIGH = 1e-2 # log_uniform high limit for learning rate -PARALLEL_SIZE = 8 # parallel thread size +PARALLEL_SIZE = 16 # parallel thread size GYM_ENV = 'Pong-v0' -ACTION_SIZE = 6 # action size TODO: need to be retrieved from gym env +_env = gym.make(GYM_ENV) +_n_actions = _env.action_space.n +_env.close() +del _env +ACTION_SIZE = _n_actions INITIAL_ALPHA_LOG_RATE = 0.4226 # log_uniform interpolate rate for learning rate (around 7 * 10^-4) GAMMA = 0.99 # discount factor for rewards @@ -19,3 +24,5 @@ GRAD_NORM_CLIP = 40.0 # gradient norm clipping USE_GPU = False # To use GPU, set True USE_LSTM = True # True for A3C LSTM, False for A3C FF + +MNIH_2015 = True # Use Mnih et al [2015] architecture (3 conv layers) diff --git a/custom_lstm.py b/custom_lstm.py deleted file mode 100644 index c77f9f1..0000000 --- a/custom_lstm.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -import tensorflow as tf - -from tensorflow.python.ops.rnn_cell import RNNCell - -class CustomBasicLSTMCell(RNNCell): - """Custom Basic LSTM recurrent network cell. - (Modified to store matrix and bias as member variable.) - - The implementation is based on: http://arxiv.org/abs/1409.2329. - - We add forget_bias (default: 1) to the biases of the forget gate in order to - reduce the scale of forgetting in the beginning of the training. - - It does not allow cell clipping, a projection layer, and does not - use peep-hole connections: it is the basic baseline. - - For advanced models, please use the full LSTMCell that follows. - """ - - def __init__(self, num_units, forget_bias=1.0, input_size=None): - """Initialize the basic LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell. - forget_bias: float, The bias added to forget gates (see above). - input_size: Deprecated and unused. - """ - if input_size is not None: - logging.warn("%s: The input_size parameter is deprecated." % self) - self._num_units = num_units - self._forget_bias = forget_bias - - @property - def state_size(self): - return 2 * self._num_units - - @property - def output_size(self): - return self._num_units - - def __call__(self, inputs, state, scope=None): - """Long short-term memory cell (LSTM).""" - with tf.variable_scope(scope or type(self).__name__): # "BasicLSTMCell" - # Parameters of gates are concatenated into one multiply for efficiency. - c, h = tf.split(1, 2, state) - concat = self._linear([inputs, h], 4 * self._num_units, True) - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - i, j, f, o = tf.split(1, 4, concat) - - new_c = c * tf.sigmoid(f + self._forget_bias) + tf.sigmoid(i) * tf.tanh(j) - new_h = tf.tanh(new_c) * tf.sigmoid(o) - - return new_h, tf.concat(1, [new_c, new_h]) - - def _linear(self, args, output_size, bias, bias_start=0.0, scope=None): - """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. - - Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. - output_size: int, second dimension of W[i]. - bias: boolean, whether to add a bias term or not. - bias_start: starting value to initialize the bias; 0 by default. - scope: VariableScope for the created subgraph; defaults to "Linear". - - Returns: - A 2D Tensor with shape [batch x output_size] equal to - sum_i(args[i] * W[i]), where W[i]s are newly created matrices. - - Raises: - ValueError: if some of the arguments has unspecified or wrong shape. - """ - if args is None or (isinstance(args, (list, tuple)) and not args): - raise ValueError("`args` must be specified") - if not isinstance(args, (list, tuple)): - args = [args] - - # Calculate the total size of arguments on dimension 1. - total_arg_size = 0 - shapes = [a.get_shape().as_list() for a in args] - for shape in shapes: - if len(shape) != 2: - raise ValueError("Linear is expecting 2D arguments: %s" % str(shapes)) - if not shape[1]: - raise ValueError("Linear expects shape[1] of arguments: %s" % str(shapes)) - else: - total_arg_size += shape[1] - - # Now the computation. - with tf.variable_scope(scope or "Linear"): - matrix = tf.get_variable("Matrix", [total_arg_size, output_size]) - if len(args) == 1: - res = tf.matmul(args[0], matrix) - else: - res = tf.matmul(tf.concat(1, args), matrix) - if not bias: - return res - bias_term = tf.get_variable( - "Bias", [output_size], - initializer=tf.constant_initializer(bias_start)) - - # Store as a member for copying. (Customized here) - self.matrix = matrix - self.bias = bias_term - - return res + bias_term - diff --git a/game_ac_network.py b/game_ac_network.py index 7c9c887..68c931f 100644 --- a/game_ac_network.py +++ b/game_ac_network.py @@ -1,37 +1,40 @@ # -*- coding: utf-8 -*- import tensorflow as tf import numpy as np -from custom_lstm import CustomBasicLSTMCell + +from constants import MNIH_2015 # Actor-Critic Network Base Class # (Policy network and Value network) class GameACNetwork(object): def __init__(self, action_size, + thread_index, # -1 for global device="/cpu:0"): - self._device = device self._action_size = action_size + self._thread_index = thread_index + self._device = device def prepare_loss(self, entropy_beta): with tf.device(self._device): # taken action (input for policy) self.a = tf.placeholder("float", [None, self._action_size]) - + # temporary difference (R-V) (input for policy) self.td = tf.placeholder("float", [None]) # avoid NaN with clipping when value in pi becomes zero log_pi = tf.log(tf.clip_by_value(self.pi, 1e-20, 1.0)) - + # policy entropy entropy = -tf.reduce_sum(self.pi * log_pi, reduction_indices=1) - + # policy loss (output) (Adding minus, because the original paper's objective function is for gradient ascent, but we use gradient descent optimizer.) - policy_loss = - tf.reduce_sum( tf.reduce_sum( tf.mul( log_pi, self.a ), reduction_indices=1 ) * self.td + entropy * entropy_beta ) + policy_loss = - tf.reduce_sum( tf.reduce_sum( tf.multiply( log_pi, self.a ), reduction_indices=1 ) * self.td + entropy * entropy_beta ) # R (input for value) self.r = tf.placeholder("float", [None]) - + # value loss (output) # (Learning rate for Critic is half of Actor's, so multiply by 0.5) value_loss = 0.5 * tf.nn.l2_loss(self.r - self.v) @@ -41,12 +44,12 @@ def prepare_loss(self, entropy_beta): def run_policy_and_value(self, sess, s_t): raise NotImplementedError() - + def run_policy(self, sess, s_t): raise NotImplementedError() def run_value(self, sess, s_t): - raise NotImplementedError() + raise NotImplementedError() def get_vars(self): raise NotImplementedError() @@ -58,7 +61,7 @@ def sync_from(self, src_netowrk, name=None): sync_ops = [] with tf.device(self._device): - with tf.op_scope([], name, "GameACNetwork") as name: + with tf.name_scope(name, "GameACNetwork", []) as name: for(src_var, dst_var) in zip(src_vars, dst_vars): sync_op = tf.assign(dst_var, src_var) sync_ops.append(sync_op) @@ -67,29 +70,25 @@ def sync_from(self, src_netowrk, name=None): # weight initialization based on muupan's code # https://github.com/muupan/async-rl/blob/master/a3c_ale.py - def _fc_weight_variable(self, shape): - input_channels = shape[0] - d = 1.0 / np.sqrt(input_channels) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) - - def _fc_bias_variable(self, shape, input_channels): + def _fc_variable(self, weight_shape): + input_channels = weight_shape[0] + output_channels = weight_shape[1] d = 1.0 / np.sqrt(input_channels) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) - - def _conv_weight_variable(self, shape): - w = shape[0] - h = shape[1] - input_channels = shape[2] + bias_shape = [output_channels] + weight = tf.Variable(tf.random_uniform(weight_shape, minval=-d, maxval=d)) + bias = tf.Variable(tf.random_uniform(bias_shape, minval=-d, maxval=d)) + return weight, bias + + def _conv_variable(self, weight_shape): + w = weight_shape[0] + h = weight_shape[1] + input_channels = weight_shape[2] + output_channels = weight_shape[3] d = 1.0 / np.sqrt(input_channels * w * h) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) - - def _conv_bias_variable(self, shape, w, h, input_channels): - d = 1.0 / np.sqrt(input_channels * w * h) - initial = tf.random_uniform(shape, minval=-d, maxval=d) - return tf.Variable(initial) + bias_shape = [output_channels] + weight = tf.Variable(tf.random_uniform(weight_shape, minval=-d, maxval=d)) + bias = tf.Variable(tf.random_uniform(bias_shape, minval=-d, maxval=d)) + return weight, bias def _conv2d(self, x, W, stride): return tf.nn.conv2d(x, W, strides = [1, stride, stride, 1], padding = "VALID") @@ -98,35 +97,44 @@ def _conv2d(self, x, W, stride): class GameACFFNetwork(GameACNetwork): def __init__(self, action_size, + thread_index, # -1 for global device="/cpu:0"): - GameACNetwork.__init__(self, action_size, device) - - with tf.device(self._device): - self.W_conv1 = self._conv_weight_variable([8, 8, 4, 16]) # stride=4 - self.b_conv1 = self._conv_bias_variable([16], 8, 8, 4) - - self.W_conv2 = self._conv_weight_variable([4, 4, 16, 32]) # stride=2 - self.b_conv2 = self._conv_bias_variable([32], 4, 4, 16) - - self.W_fc1 = self._fc_weight_variable([2592, 256]) - self.b_fc1 = self._fc_bias_variable([256], 2592) + GameACNetwork.__init__(self, action_size, thread_index, device) + + scope_name = "net_" + str(self._thread_index) + with tf.device(self._device), tf.variable_scope(scope_name) as scope: + if MNIH_2015: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 32]) + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 32, 64]) + self.W_conv3, self.b_conv3 = self._conv_variable([3, 3, 64, 64]) + self.W_fc1, self.b_fc1 = self._fc_variable([3136, 256]) + else: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 16]) # stride=4 + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 16, 32]) # stride=2 + self.W_fc1, self.b_fc1 = self._fc_variable([2592, 256]) # weight for policy output layer - self.W_fc2 = self._fc_weight_variable([256, action_size]) - self.b_fc2 = self._fc_bias_variable([action_size], 256) + self.W_fc2, self.b_fc2 = self._fc_variable([256, action_size]) # weight for value output layer - self.W_fc3 = self._fc_weight_variable([256, 1]) - self.b_fc3 = self._fc_bias_variable([1], 256) + self.W_fc3, self.b_fc3 = self._fc_variable([256, 1]) # state (input) self.s = tf.placeholder("float", [None, 84, 84, 4]) - - h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) - h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) - h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) - h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) + if MNIH_2015: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + h_conv3 = tf.nn.relu(self._conv2d(h_conv2, self.W_conv3, 1) + self.b_conv3) + + h_conv3_flat = tf.reshape(h_conv3, [-1, 3136]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv3_flat, self.W_fc1) + self.b_fc1) + else: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + + h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) # policy (output) self.pi = tf.nn.softmax(tf.matmul(h_fc1, self.W_fc2) + self.b_fc2) @@ -159,48 +167,55 @@ def __init__(self, action_size, thread_index, # -1 for global device="/cpu:0" ): - GameACNetwork.__init__(self, action_size, device) - - with tf.device(self._device): - self.W_conv1 = self._conv_weight_variable([8, 8, 4, 16]) # stride=4 - self.b_conv1 = self._conv_bias_variable([16], 8, 8, 4) - - self.W_conv2 = self._conv_weight_variable([4, 4, 16, 32]) # stride=2 - self.b_conv2 = self._conv_bias_variable([32], 4, 4, 16) - - self.W_fc1 = self._fc_weight_variable([2592, 256]) - self.b_fc1 = self._fc_bias_variable([256], 2592) + GameACNetwork.__init__(self, action_size, thread_index, device) + + scope_name = "net_" + str(self._thread_index) + with tf.device(self._device), tf.variable_scope(scope_name) as scope: + if MNIH_2015: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 32]) + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 32, 64]) + self.W_conv3, self.b_conv3 = self._conv_variable([3, 3, 64, 64]) + self.W_fc1, self.b_fc1 = self._fc_variable([3136, 256]) + else: + self.W_conv1, self.b_conv1 = self._conv_variable([8, 8, 4, 16]) # stride=4 + self.W_conv2, self.b_conv2 = self._conv_variable([4, 4, 16, 32]) # stride=2 + self.W_fc1, self.b_fc1 = self._fc_variable([2592, 256]) # lstm - self.lstm = CustomBasicLSTMCell(256) + self.lstm = tf.contrib.rnn.BasicLSTMCell(256, state_is_tuple=True) # weight for policy output layer - self.W_fc2 = self._fc_weight_variable([256, action_size]) - self.b_fc2 = self._fc_bias_variable([action_size], 256) + self.W_fc2, self.b_fc2 = self._fc_variable([256, action_size]) # weight for value output layer - self.W_fc3 = self._fc_weight_variable([256, 1]) - self.b_fc3 = self._fc_bias_variable([1], 256) + self.W_fc3, self.b_fc3 = self._fc_variable([256, 1]) # state (input) self.s = tf.placeholder("float", [None, 84, 84, 4]) - - h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) - h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) - h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) - h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) - # h_fc1 shape=(5,256) + if MNIH_2015: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + h_conv3 = tf.nn.relu(self._conv2d(h_conv2, self.W_conv3, 1) + self.b_conv3) + + h_conv3_flat = tf.reshape(h_conv3, [-1, 3136]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv3_flat, self.W_fc1) + self.b_fc1) + else: + h_conv1 = tf.nn.relu(self._conv2d(self.s, self.W_conv1, 4) + self.b_conv1) + h_conv2 = tf.nn.relu(self._conv2d(h_conv1, self.W_conv2, 2) + self.b_conv2) + + h_conv2_flat = tf.reshape(h_conv2, [-1, 2592]) + h_fc1 = tf.nn.relu(tf.matmul(h_conv2_flat, self.W_fc1) + self.b_fc1) h_fc1_reshaped = tf.reshape(h_fc1, [1,-1,256]) - # h_fc_reshaped = (1,5,256) # place holder for LSTM unrolling time step size. self.step_size = tf.placeholder(tf.float32, [1]) - self.initial_lstm_state = tf.placeholder(tf.float32, [1, self.lstm.state_size]) - - scope = "net_" + str(thread_index) + self.initial_lstm_state0 = tf.placeholder(tf.float32, [1, 256]) + self.initial_lstm_state1 = tf.placeholder(tf.float32, [1, 256]) + self.initial_lstm_state = tf.contrib.rnn.LSTMStateTuple(self.initial_lstm_state0, + self.initial_lstm_state1) # Unrolling LSTM up to LOCAL_T_MAX time steps. (= 5time steps.) # When episode terminates unrolling time steps becomes less than LOCAL_TIME_STEP. @@ -215,51 +230,59 @@ def __init__(self, scope = scope) # lstm_outputs: (1,5,256) for back prop, (1,1,256) for forward prop. - + lstm_outputs = tf.reshape(lstm_outputs, [-1,256]) # policy (output) self.pi = tf.nn.softmax(tf.matmul(lstm_outputs, self.W_fc2) + self.b_fc2) - + # value (output) v_ = tf.matmul(lstm_outputs, self.W_fc3) + self.b_fc3 self.v = tf.reshape( v_, [-1] ) + scope.reuse_variables() + self.W_lstm = tf.get_variable("basic_lstm_cell/weights") + self.b_lstm = tf.get_variable("basic_lstm_cell/biases") + self.reset_state() - + def reset_state(self): - self.lstm_state_out = np.zeros([1, self.lstm.state_size]) + self.lstm_state_out = tf.contrib.rnn.LSTMStateTuple(np.zeros([1, 256]), + np.zeros([1, 256])) def run_policy_and_value(self, sess, s_t): # This run_policy_and_value() is used when forward propagating. # so the step size is 1. pi_out, v_out, self.lstm_state_out = sess.run( [self.pi, self.v, self.lstm_state], feed_dict = {self.s : [s_t], - self.initial_lstm_state : self.lstm_state_out, + self.initial_lstm_state0 : self.lstm_state_out[0], + self.initial_lstm_state1 : self.lstm_state_out[1], self.step_size : [1]} ) # pi_out: (1,3), v_out: (1) return (pi_out[0], v_out[0]) def run_policy(self, sess, s_t): - # This run_policy() is used for displaying the result with display tool. + # This run_policy() is used for displaying the result with display tool. pi_out, self.lstm_state_out = sess.run( [self.pi, self.lstm_state], feed_dict = {self.s : [s_t], - self.initial_lstm_state : self.lstm_state_out, + self.initial_lstm_state0 : self.lstm_state_out[0], + self.initial_lstm_state1 : self.lstm_state_out[1], self.step_size : [1]} ) - + return pi_out[0] def run_value(self, sess, s_t): - # This run_value() is used for calculating V for bootstrapping at the + # This run_value() is used for calculating V for bootstrapping at the # end of LOCAL_T_MAX time step sequence. # When next sequcen starts, V will be calculated again with the same state using updated network weights, # so we don't update LSTM state here. prev_lstm_state_out = self.lstm_state_out v_out, _ = sess.run( [self.v, self.lstm_state], feed_dict = {self.s : [s_t], - self.initial_lstm_state : self.lstm_state_out, + self.initial_lstm_state0 : self.lstm_state_out[0], + self.initial_lstm_state1 : self.lstm_state_out[1], self.step_size : [1]} ) - + # roll back lstm state self.lstm_state_out = prev_lstm_state_out return v_out[0] @@ -268,6 +291,6 @@ def get_vars(self): return [self.W_conv1, self.b_conv1, self.W_conv2, self.b_conv2, self.W_fc1, self.b_fc1, - self.lstm.matrix, self.lstm.bias, + self.W_lstm, self.b_lstm, self.W_fc2, self.b_fc2, self.W_fc3, self.b_fc3] diff --git a/game_state.py b/game_state.py index a8e5ffe..9bbfdde 100644 --- a/game_state.py +++ b/game_state.py @@ -3,12 +3,33 @@ import numpy as np import gym -import skimage.color -import skimage.transform +import cv2 +import atari_py from constants import GYM_ENV from constants import ACTION_SIZE +class AtariEnvSkipping(gym.Wrapper): + def __init__(self, env, frameskip=4): + self.env = env + self.env.ale.setFloat('frame_skip'.encode('utf-8'), frameskip) + self.env.ale.setFloat('repeat_action_probability'.encode('utf-8'), 0.0) + self.env._seed() + + print ("lives={}".format(self.env.ale.lives())) + print ("frameskip={}".format(self.env.ale.getFloat(b'frame_skip'))) + print ("repeat_action_probability={}".format(self.env.ale.getFloat(b'repeat_action_probability'))) + print ("action space={}".format(self.env.action_space.n)) + + def _step(self, a): + reward = 0.0 + action = self.env._action_set[a] + + reward += self.env.ale.act(action) + ob = self.env._get_obs() + + return ob, reward, self.env.ale.game_over(), {"ale.lives": self.env.ale.lives()} + class GameState(object): def __init__(self, display=False, crop_screen=True, frame_skip=4, no_op_max=30): self._display = display @@ -17,43 +38,40 @@ def __init__(self, display=False, crop_screen=True, frame_skip=4, no_op_max=30): if self._frame_skip < 1: self._frame_skip = 1 self._no_op_max = no_op_max - + self.env_id = GYM_ENV + self.env = gym.make(GYM_ENV) + self.env = AtariEnvSkipping(self.env, frameskip=self._frame_skip) - #print "action space=", self.env.action_space - self.reset() - + def _process_frame(self, action, reshape): reward = 0 - for i in range(self._frame_skip): - observation, r, terminal, _ = self.env.step(action) - reward += r - if terminal: - break - # observation shape = (210, 160, 3) - - grayscale_observation = skimage.color.rgb2gray(observation) - # shape (210, 160) range = [0.0, 1.0] + observation, r, terminal, _ = self.env.step(action) + reward += r + + grayscale_observation = cv2.cvtColor(observation, cv2.COLOR_RGB2GRAY) if self._crop_screen: - # resize to height=110, width=84 - resized_observation = skimage.transform.resize(grayscale_observation, (110, 84)) - resized_observation = resized_observation.astype(np.float32) + resized_observation = grayscale_observation.astype(np.float32) # crop to fit 84x84 - x_t = resized_observation[18:102,:] + x_t = resized_observation[34:34+160, :160] + x_t = cv2.resize(x_t, (84, 84)) else: # resize to height=84, width=84 - resized_observation = skimage.transform.resize(grayscale_observation, (84, 84)) + resized_observation = cv2.resize(grayscale_observation, (84,84)) x_t = resized_observation.astype(np.float32) if reshape: x_t = np.reshape(x_t, (84, 84, 1)) + + # normalize + x_t *= (1.0/255.0) return reward, terminal, x_t - + def reset(self): self.env.reset() - + # randomize initial state if self._no_op_max > 0: no_op = np.random.randint(0, self._no_op_max + 1) @@ -61,20 +79,20 @@ def reset(self): self.env.step(0) _, _, x_t = self._process_frame(0, False) - + self.reward = 0 self.terminal = False self.s_t = np.stack((x_t, x_t, x_t, x_t), axis = 2) - + def process(self, action): if self._display: self.env.render() - + r, t, x_t1 = self._process_frame(action, True) self.reward = r self.terminal = t - self.s_t1 = np.append(self.s_t[:,:,1:], x_t1, axis = 2) + self.s_t1 = np.append(self.s_t[:,:,1:], x_t1, axis = 2) def update(self): self.s_t = self.s_t1 diff --git a/game_state_test.py b/game_state_test.py index 44750eb..ca3e78e 100644 --- a/game_state_test.py +++ b/game_state_test.py @@ -7,9 +7,9 @@ class TestSequenceFunctions(unittest.TestCase): def test_process(self): game_state = GameState(0) - + before_s_t = np.array( game_state.s_t ) - + for i in range(1000): bef1 = game_state.s_t[:,:,1] bef2 = game_state.s_t[:,:,2] @@ -17,7 +17,7 @@ def test_process(self): game_state.process(1) game_state.update() - + aft0 = game_state.s_t[:,:,0] aft1 = game_state.s_t[:,:,1] aft2 = game_state.s_t[:,:,2] diff --git a/rmsprop_applier.py b/rmsprop_applier.py index b73ad14..0d92149 100644 --- a/rmsprop_applier.py +++ b/rmsprop_applier.py @@ -75,7 +75,7 @@ def _zeros_slot(self, var, slot_name, op_name): # TODO: in RMSProp native code, memcpy() (for CPU) and # cudaMemcpyAsync() (for GPU) are used when updating values, # and values might tend to be overwritten with results from other threads. - # (Need to check the learning performance with replacing it) + # (Need to check the learning performance with replacing it) def _apply_dense(self, grad, var): rms = self.get_slot(var, "rms") mom = self.get_slot(var, "momentum") @@ -96,7 +96,7 @@ def apply_gradients(self, var_list, accum_grad_list, name=None): with tf.control_dependencies(None): self._create_slots(var_list) - with tf.op_scope([], name, self._name) as name: + with tf.name_scope(name, self._name, []) as name: self._prepare() for var, accum_grad in zip(var_list, accum_grad_list): with tf.name_scope("update_" + var.op.name), tf.device(var.device): diff --git a/rmsprop_applier_test.py b/rmsprop_applier_test.py index a5e1c06..00e9cf3 100644 --- a/rmsprop_applier_test.py +++ b/rmsprop_applier_test.py @@ -9,15 +9,15 @@ class RMSPropApplierTest(tf.test.TestCase): def testApply(self): with self.test_session(): var = tf.Variable([1.0, 2.0]) - + grad0 = tf.Variable([2.0, 4.0]) grad1 = tf.Variable([3.0, 6.0]) - + opt = rmsprop_applier.RMSPropApplier(learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0) - + apply_gradient0 = opt.apply_gradients([var], [grad0]) apply_gradient1 = opt.apply_gradients([var], [grad1]) @@ -49,8 +49,8 @@ def testApply(self): ms_y = ms_y + (dy * dy - ms_y) * (1.0 - 0.9) x = x - (2.0 * dx / math.sqrt(ms_x+1.0)) y = y - (2.0 * dy / math.sqrt(ms_y+1.0)) - + self.assertAllClose(np.array([x, y]), var.eval()) - + if __name__ == "__main__": tf.test.main()