huangfu/InteractionImitation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

making option list indexable to visualize in tensorboard. noticing run from last night had much better performance under more long options, unclear if due to choice or environment factors, like episodes lasting longer. making stoponcollision and seed parameters. fixing ability to see reward metrics under ray/tune/, problem was they were being returned as tensors, not floats

Browse Source
This commit is contained in:
Arec Jamgochian
2022-02-25 16:36:14 -08:00
parent d1f23e6d25
commit 59083ebce3
2 changed files with 69 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
sgail-ppo-options-setobs2.py
View File

@@ -6,8 +6,8 @@ from src.safe_options.options import gail_ppo, Buffer
from src.core.value import SetValue from src.core.value import SetValue
from src.safe_options.policy import SetMaskedDiscretePolicy from src.safe_options.policy import SetMaskedDiscretePolicy
from src.core.discriminator import DeepsetDiscriminator from src.core.discriminator import DeepsetDiscriminator
import torch.optim import torch
import torch.nn
from intersim.envs import IntersimpleLidarFlatRandom from intersim.envs import IntersimpleLidarFlatRandom
from intersim.envs.intersimple import speed_reward from intersim.envs.intersimple import speed_reward
import functools import functools
@@ -18,25 +18,35 @@ from torch.utils.tensorboard import SummaryWriter
from ray import tune from ray import tune
DIR = os.path.dirname(os.path.abspath(__file__)) DIR = os.path.dirname(os.path.abspath(__file__))
option_list = [[(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5]],
[(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20]],
[(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5, 10, 20]], # was the best in training with single hidden layer, but very slow
[(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20, 40]],
[(vel, time) for vel in [0, 2, 5, 10] for time in [5, 10, 20]],
[(vel, time) for vel in [0, 3, 10] for time in [5, 20, 40]]
]
def training_function(config): obs_min = np.array([
obs_min = np.array([
[-1000, -1000, 0, -np.pi, -1e-1, 0.], [-1000, -1000, 0, -np.pi, -1e-1, 0.],
[0, -np.pi, -20, -20, -np.pi, -1e-1], [0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1], [0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1], [0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1], [0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1], [0, -np.pi, -20, -20, -np.pi, -1e-1],
]).reshape(-1) ]).reshape(-1)
obs_max = np.array([ obs_max = np.array([
[1000, 1000, 20, np.pi, 1e-1, 0.], [1000, 1000, 20, np.pi, 1e-1, 0.],
[50, np.pi, 20, 20, np.pi, 1e-1], [50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1], [50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1], [50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1], [50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1], [50, np.pi, 20, 20, np.pi, 1e-1],
]).reshape(-1) ]).reshape(-1)
def training_function(config):
np.random.seed(config['seed'])
torch.manual_seed(config['seed'])
envs = sum([[SafeOptionsEnv(Setobs( envs = sum([[SafeOptionsEnv(Setobs(
TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom( TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom(
@@ -45,9 +55,9 @@ def training_function(config):
speed_reward, speed_reward,
collision_penalty=0 collision_penalty=0
), ),
stop_on_collision=False, track=track, stop_on_collision=config['stop_on_collision'], track=track,
), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10))
), options=config['policy']['option'], safe_actions_collision_method='circle', ), options=option_list[config['policy']['option']], safe_actions_collision_method='circle',
abort_unsafe_collision_method='circle') for _ in range(20)] for track in range(4)],[]) abort_unsafe_collision_method='circle') for _ in range(20)] for track in range(4)],[])
env_fn = lambda i: envs[i] env_fn = lambda i: envs[i]
@@ -79,7 +89,9 @@ def training_function(config):
expert_data = Buffer(*expert_data) expert_data = Buffer(*expert_data)
def callback(info): def callback(info):
tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode']) tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'],
disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'],
mean_episode_length=info['gen/mean_episode_length'])
value, policy = gail_ppo( value, policy = gail_ppo(
env_fn=env_fn, env_fn=env_fn,
@@ -91,9 +103,9 @@ def training_function(config):
value=value, value=value,
v_opt=v_opt, v_opt=v_opt,
v_iters=config['value']['iterations_per_epoch'], v_iters=config['value']['iterations_per_epoch'],
epochs=200, epochs=2, # 200 FIXME
rollout_episodes=60, rollout_episodes=6, #60, FIXME
rollout_steps=60, rollout_steps=6, #60, FIXME
gamma=0.99, gamma=0.99,
gae_lambda=0.9, gae_lambda=0.9,
clip_ratio=config['policy']['clip_ratio'], clip_ratio=config['policy']['clip_ratio'],
@@ -104,33 +116,33 @@ def training_function(config):
lr_schedulers=[pi_lr_scheduler], lr_schedulers=[pi_lr_scheduler],
) )
# save value, policy
analysis = tune.run( analysis = tune.run(
training_function, training_function,
config={ config={
'stop_on_collision': tune.grid_search([True, False]),
'policy': { 'policy': {
'learning_rate': tune.grid_search([3e-4]), 'learning_rate': 3e-4, # tune.grid_search([3e-4]),
'learning_rate_decay': tune.grid_search([1.0]), 'learning_rate_decay': 1.0, #tune.grid_search([1.0]),
'clip_ratio': tune.grid_search([0.2]), 'clip_ratio': 0.2, #tune.grid_search([0.2]),
'iterations_per_epoch': tune.grid_search([100]), 'iterations_per_epoch': 100, #tune.grid_search([100]),
'hidden_layer_size': tune.grid_search([10, 20, 30]), 'hidden_layer_size': tune.grid_search([10, 20, 40]),
'n_hidden_layers': tune.grid_search([1, 2, 3]), 'n_hidden_layers': 1, #tune.grid_search([1, 2, 3]), #FIXME
'activation':tune.grid_search([torch.nn.LeakyReLU, torch.nn.Tanh]), 'activation':tune.grid_search([torch.nn.LeakyReLU, torch.nn.Tanh]),
'option': tune.grid_search([[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], 'option': tune.grid_search(list(range(len(option_list))))
[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5),
(0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10)],
[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5),
(0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10),
(0, 20), (1, 20), (2, 20), (4, 20), (6, 20), (8, 20), (20, 10)]])
}, },
'value': { 'value': {
'learning_rate': tune.grid_search([1e-3]), 'learning_rate': 1e-3, # tune.grid_search([1e-3]),
'iterations_per_epoch': tune.grid_search([1000]), 'iterations_per_epoch': 1000, #tune.grid_search([1000]),
}, },
'discriminator': { 'discriminator': {
'learning_rate': tune.grid_search([1e-3]), 'learning_rate': 1e-3, #tune.grid_search([1e-3]),
'weight_decay': tune.grid_search([1e-4]), 'weight_decay': 1e-4, #tune.grid_search([1e-4]),
'iterations_per_epoch': tune.grid_search([100]), 'iterations_per_epoch': 100, #tune.grid_search([100]),
} },
'seed': 0,
} }
) )

21
src/safe_options/options.py
View File

@@ -52,7 +52,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value
generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions)
logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0]
logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length , epoch)
gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0]
logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch)
logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch)
@@ -63,7 +64,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value
else: else:
generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions))
logger.add_scalar('disc/final_loss', loss, epoch) logger.add_scalar('disc/final_loss', loss, epoch)
logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0]
logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode , epoch)
#assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape
generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1)
@@ -76,7 +78,9 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value
'epoch': epoch, 'epoch': epoch,
'value': value, 'value': value,
'policy': policy, 'policy': policy,
'gen/mean_reward_per_episode': gen_mean_reward_per_episode, 'gen/mean_episode_length': gen_mean_episode_length.item(),
'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(),
'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(),
}) })
return value, policy return value, policy
@@ -93,8 +97,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v
generator_data = OptionsRollout(HLBuffer(*hl_data), Buffer(*ll_data)) generator_data = OptionsRollout(HLBuffer(*hl_data), Buffer(*ll_data))
generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions)
gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0]
logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length, epoch)
gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0]
logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch)
logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch)
@@ -105,7 +109,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v
else: else:
generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions))
logger.add_scalar('disc/final_loss', loss, epoch) logger.add_scalar('disc/final_loss', loss, epoch)
logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0]
logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode, epoch)
#assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape
generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1)
@@ -118,7 +123,9 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v
'epoch': epoch, 'epoch': epoch,
'value': value, 'value': value,
'policy': policy, 'policy': policy,
'gen/mean_reward_per_episode': gen_mean_reward_per_episode, 'gen/mean_episode_length': gen_mean_episode_length.item(),
'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(),
'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(),
}) })
for lr_scheduler in lr_schedulers: for lr_scheduler in lr_schedulers: