Skip to content

Improve stick game human mode and reporting #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
1marieCurie wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Improve stick game human mode and reporting #3

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 80 additions & 33 deletions rl/sticks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import random
import numpy as np


class StickGame(object):
"""
StickGame.
Expand All @@ -26,7 +27,7 @@ def reset(self):

def display(self):
# Display the state of the game
print ("| " * self.nb)
print("| " * self.nb)

def step(self, action):
# @action either 1, 2 or 3. Take an action into the environement
Expand All @@ -36,6 +37,7 @@ def step(self, action):
else:
return self.nb, 0


class StickPlayer(object):
"""
Stick Player
Expand All @@ -47,11 +49,12 @@ def __init__(self, is_human, size, trainable=True):
self.is_human = is_human
self.history = []
self.V = {}
for s in range(1, size+1):
for s in range(1, size + 1):
self.V[s] = 0.
self.win_nb = 0.
self.lose_nb = 0.
self.rewards = []
self.game_results = []
self.eps = 0.99
self.trainable = trainable

Expand All @@ -60,6 +63,7 @@ def reset_stat(self):
self.win_nb = 0
self.lose_nb = 0
self.rewards = []
self.game_results = []

def greedy_step(self, state):
# Greedy step
Expand All @@ -79,10 +83,28 @@ def play(self, state):
# Take random action
if random.uniform(0, 1) < self.eps:
action = randint(1, 3)
else: # Or greedy action
else: # Or greedy action
action = self.greedy_step(state)
else:
action = int(input("$>"))
while True:
try:
choice = input(f"$> Take 1, 2, or 3 sticks (remaining: {state}, Q to quit): ").strip()
if choice.lower() == "q":
raise SystemExit("Exiting the human game loop.")
action = int(choice)
except ValueError:
print("Please enter a whole number: 1, 2, or 3.")
continue
except (EOFError, KeyboardInterrupt):
raise SystemExit("\nInput cancelled. Exiting the human game loop.")

if action not in (1, 2, 3):
print("Invalid move. Choose 1, 2, or 3.")
continue
if action > state:
print(f"Invalid move. You cannot take more than {state} stick(s).")
continue
break
return action

def add_transition(self, n_tuple):
Expand All @@ -99,40 +121,60 @@ def train(self):
for transition in reversed(self.history):
s, a, r, sp = transition
if r == 0:
self.V[s] = self.V[s] + 0.001*(self.V[sp] - self.V[s])
self.V[s] = self.V[s] + 0.001 * (self.V[sp] - self.V[s])
else:
self.V[s] = self.V[s] + 0.001*(r - self.V[s])
self.V[s] = self.V[s] + 0.001 * (r - self.V[s])

self.history = []

def add_game_result(self, result):
self.game_results.append(result)


def should_play_human():
try:
choice = input("Press P to play against the agent, or press Enter to run the automated evaluation: ")
except (EOFError, KeyboardInterrupt):
return False

return choice.strip().lower() == "p"


def play(game, p1, p2, train=True):
state = game.reset()
players = [p1, p2]
random.shuffle(players)
p = 0
while game.is_finished() is False:

if players[p%2].is_human:
if players[p % 2].is_human:
game.display()

action = players[p%2].play(state)
action = players[p % 2].play(state)
n_state, reward = game.step(action)

# Game is over. Ass stat
if (reward != 0):
# Game is over. Add stat
if reward != 0:
# Update stat of the current player
players[p%2].lose_nb += 1. if reward == -1 else 0
players[p%2].win_nb += 1. if reward == 1 else 0
players[p % 2].lose_nb += 1. if reward == -1 else 0
players[p % 2].win_nb += 1. if reward == 1 else 0
# Update stat of the other player
players[(p+1)%2].lose_nb += 1. if reward == 1 else 0
players[(p+1)%2].win_nb += 1. if reward == -1 else 0
players[(p + 1) % 2].lose_nb += 1. if reward == 1 else 0
players[(p + 1) % 2].win_nb += 1. if reward == -1 else 0
players[p % 2].add_game_result(reward)
players[(p + 1) % 2].add_game_result(-reward)

if players[p % 2].is_human:
print("You lose. You took the last stick.")
elif players[(p + 1) % 2].is_human:
print("You win. The agent took the last stick.")

# Add the reversed reward and the new state to the other player
if p != 0:
s, a, r, sp = players[(p+1)%2].history[-1]
players[(p+1)%2].history[-1] = (s, a, reward * -1, n_state)
s, a, r, sp = players[(p + 1) % 2].history[-1]
players[(p + 1) % 2].history[-1] = (s, a, reward * -1, n_state)

players[p%2].add_transition((state, action, reward, None))
players[p % 2].add_transition((state, action, reward, None))

state = n_state
p += 1
Expand All @@ -141,7 +183,10 @@ def play(game, p1, p2, train=True):
p1.train()
p2.train()


if __name__ == '__main__':
play_human = should_play_human()

game = StickGame(12)

# PLayers to train
Expand All @@ -154,22 +199,24 @@ def play(game, p1, p2, train=True):
# Train the agent
for i in range(0, 10000):
if i % 10 == 0:
p1.eps = max(p1.eps*0.996, 0.05)
p2.eps = max(p2.eps*0.996, 0.05)
p1.eps = max(p1.eps * 0.996, 0.05)
p2.eps = max(p2.eps * 0.996, 0.05)
play(game, p1, p2)
p1.reset_stat()

# Display the value function
for key in p1.V:
print(key, p1.V[key])
print("--------------------------")

# Play agains a random player
for _ in range(0, 1000):
play(game, p1, random_player, train=False)
print("p1 win rate", p1.win_nb/(p1.win_nb + p1.lose_nb))
print("p1 win mean", np.mean(p1.rewards))

# Play agains us
while True:
play(game, p1, human, train=False)
if play_human:
while True:
play(game, p1, human, train=False)
else:
# Display the value function
for key in p1.V:
print(key, p1.V[key])
print("--------------------------")

# Play against a random player
for _ in range(0, 1000):
play(game, p1, random_player, train=False)
print("p1 win rate", p1.win_nb / (p1.win_nb + p1.lose_nb))
print("p1 move reward mean", np.mean(p1.rewards))
print("p1 game reward mean", np.mean(p1.game_results))
print("Running automated mode. Press P at startup if you want to play yourself.")