Multi-Armed-Bandit-Problem/bandits.py at master · Sadiq0123/Multi-Armed-Bandit-Problem · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# -*- coding: utf-8 -*-
"""Bandits.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1Q8tuFYRXgSX9WCJbWLA_qdYCb5pRvi7k
"""

import numpy as np
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (20,10)

num_steps = 10000
num_band = 10         # NUMBER OF BANDITS
epsilon1 = 0.1         # EPSILON 1 IN EPSILON-GREEDY
epsilon2 = 0.01         # EPSILON 2 IN EPSILON-GREEDY
alpha = 0.05          # ALPHA IN EXPONENTIAL RECENCY WEIGHTED AVERAGE
temp_Q = np.zeros(num_band)
temp_N = np.zeros(num_band)
index_vec = np.arange(0,num_band)
temp_R = np.random.uniform(0,3,10)  # REWARD DISTRIBUTION
tot_Reward = np.zeros(num_steps)        # FINAL VALUE TO BE PLOTTED

temp_Q2 = np.zeros(num_band)+10
temp_N2 = np.zeros(num_band)
index_vec2 = np.arange(0,num_band)
tot_Reward2 = np.zeros(num_steps)      # FINAL VALUE TO BE PLOTTED

print(temp_R)
i = 1

for step in range(num_steps):

  #  FIRST GRAPH

  if np.random.uniform(0,1)>epsilon1:
      if len(np.argwhere(temp_Q==np.amax(temp_Q)).flatten())>1:
          index = np.random.choice(index_vec[np.argwhere(temp_Q==np.amax(temp_Q)).flatten()])
      else:
          index = index_vec[np.argwhere(temp_Q==np.amax(temp_Q)).flatten()]
  else:
      index = np.random.choice(index_vec)
  reward = np.random.normal(temp_R[index],1)
  tot_Reward[step] = tot_Reward[step-1] + reward
  temp_N[index] = temp_N[index] + 1
  temp_Q[index] = temp_Q[index] + (reward - temp_Q[index])/temp_N[index]

  # SECOND GRAPH

  if np.random.uniform(0,1)>epsilon2:
      if len(np.argwhere(temp_Q2==np.amax(temp_Q2)).flatten())>1:
          index = np.random.choice(index_vec2[np.argwhere(temp_Q2==np.amax(temp_Q2)).flatten()])
      else:
          index = index_vec2[np.argwhere(temp_Q2==np.amax(temp_Q2)).flatten()]
  else:
      index = np.random.choice(index_vec2)
  reward = np.random.normal(temp_R[index],1)
  tot_Reward2[step] = tot_Reward2[step-1] + reward
  temp_N2[index] = temp_N2[index] + 1
  temp_Q2[index] = temp_Q2[index] + (reward - temp_Q2[index])/temp_N2[index]


#   PLOTTING THE RATIO OF TOTAL REWARD OBTAINED TO MAXIMUM OBTAINABLE REWARD

plt.plot(np.divide(tot_Reward,np.amax(temp_R)*np.arange(1,num_steps+1,1)),'r',label = 'epsilon = 0.1 ')
plt.plot(np.divide(tot_Reward2,np.amax(temp_R)*np.arange(1,num_steps+1,1)),'k', label = 'epsilon = 0.01')
plt.legend(loc='best')
plt.show()

print(np.amax(tot_Reward))
print(np.divide(tot_Reward,np.amax(temp_R)*np.arange(0,num_steps,1))[-1])
print(np.divide(tot_Reward2,np.amax(temp_R)*np.arange(0,num_steps,1))[-1])