From 59083ebce38243afbe7bf1973352ffb85993d69f Mon Sep 17 00:00:00 2001 From: Arec Jamgochian Date: Fri, 25 Feb 2022 16:36:14 -0800 Subject: [PATCH] making option list indexable to visualize in tensorboard. noticing run from last night had much better performance under more long options, unclear if due to choice or environment factors, like episodes lasting longer. making stoponcollision and seed parameters. fixing ability to see reward metrics under ray/tune/, problem was they were being returned as tensors, not floats --- sgail-ppo-options-setobs2.py | 98 ++++++++++++++++++++---------------- src/safe_options/options.py | 21 +++++--- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/sgail-ppo-options-setobs2.py b/sgail-ppo-options-setobs2.py index 6f9c4fd..5d7f569 100644 --- a/sgail-ppo-options-setobs2.py +++ b/sgail-ppo-options-setobs2.py @@ -6,8 +6,8 @@ from src.safe_options.options import gail_ppo, Buffer from src.core.value import SetValue from src.safe_options.policy import SetMaskedDiscretePolicy from src.core.discriminator import DeepsetDiscriminator -import torch.optim -import torch.nn +import torch + from intersim.envs import IntersimpleLidarFlatRandom from intersim.envs.intersimple import speed_reward import functools @@ -18,25 +18,35 @@ from torch.utils.tensorboard import SummaryWriter from ray import tune DIR = os.path.dirname(os.path.abspath(__file__)) +option_list = [[(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5]], + [(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20]], + [(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5, 10, 20]], # was the best in training with single hidden layer, but very slow + [(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20, 40]], + [(vel, time) for vel in [0, 2, 5, 10] for time in [5, 10, 20]], + [(vel, time) for vel in [0, 3, 10] for time in [5, 20, 40]] +] + +obs_min = np.array([ + [-1000, -1000, 0, -np.pi, -1e-1, 0.], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], +]).reshape(-1) + +obs_max = np.array([ + [1000, 1000, 20, np.pi, 1e-1, 0.], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], +]).reshape(-1) def training_function(config): - obs_min = np.array([ - [-1000, -1000, 0, -np.pi, -1e-1, 0.], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - ]).reshape(-1) - - obs_max = np.array([ - [1000, 1000, 20, np.pi, 1e-1, 0.], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - ]).reshape(-1) + np.random.seed(config['seed']) + torch.manual_seed(config['seed']) envs = sum([[SafeOptionsEnv(Setobs( TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom( @@ -45,9 +55,9 @@ def training_function(config): speed_reward, collision_penalty=0 ), - stop_on_collision=False, track=track, + stop_on_collision=config['stop_on_collision'], track=track, ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) - ), options=config['policy']['option'], safe_actions_collision_method='circle', + ), options=option_list[config['policy']['option']], safe_actions_collision_method='circle', abort_unsafe_collision_method='circle') for _ in range(20)] for track in range(4)],[]) env_fn = lambda i: envs[i] @@ -79,7 +89,9 @@ def training_function(config): expert_data = Buffer(*expert_data) def callback(info): - tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode']) + tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'], + disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'], + mean_episode_length=info['gen/mean_episode_length']) value, policy = gail_ppo( env_fn=env_fn, @@ -91,9 +103,9 @@ def training_function(config): value=value, v_opt=v_opt, v_iters=config['value']['iterations_per_epoch'], - epochs=200, - rollout_episodes=60, - rollout_steps=60, + epochs=2, # 200 FIXME + rollout_episodes=6, #60, FIXME + rollout_steps=6, #60, FIXME gamma=0.99, gae_lambda=0.9, clip_ratio=config['policy']['clip_ratio'], @@ -104,33 +116,33 @@ def training_function(config): lr_schedulers=[pi_lr_scheduler], ) + # save value, policy + + analysis = tune.run( training_function, config={ + 'stop_on_collision': tune.grid_search([True, False]), 'policy': { - 'learning_rate': tune.grid_search([3e-4]), - 'learning_rate_decay': tune.grid_search([1.0]), - 'clip_ratio': tune.grid_search([0.2]), - 'iterations_per_epoch': tune.grid_search([100]), - 'hidden_layer_size': tune.grid_search([10, 20, 30]), - 'n_hidden_layers': tune.grid_search([1, 2, 3]), + 'learning_rate': 3e-4, # tune.grid_search([3e-4]), + 'learning_rate_decay': 1.0, #tune.grid_search([1.0]), + 'clip_ratio': 0.2, #tune.grid_search([0.2]), + 'iterations_per_epoch': 100, #tune.grid_search([100]), + 'hidden_layer_size': tune.grid_search([10, 20, 40]), + 'n_hidden_layers': 1, #tune.grid_search([1, 2, 3]), #FIXME 'activation':tune.grid_search([torch.nn.LeakyReLU, torch.nn.Tanh]), - 'option': tune.grid_search([[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], - [(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5), - (0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10)], - [(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5), - (0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10), - (0, 20), (1, 20), (2, 20), (4, 20), (6, 20), (8, 20), (20, 10)]]) + 'option': tune.grid_search(list(range(len(option_list)))) }, 'value': { - 'learning_rate': tune.grid_search([1e-3]), - 'iterations_per_epoch': tune.grid_search([1000]), + 'learning_rate': 1e-3, # tune.grid_search([1e-3]), + 'iterations_per_epoch': 1000, #tune.grid_search([1000]), }, 'discriminator': { - 'learning_rate': tune.grid_search([1e-3]), - 'weight_decay': tune.grid_search([1e-4]), - 'iterations_per_epoch': tune.grid_search([100]), - } + 'learning_rate': 1e-3, #tune.grid_search([1e-3]), + 'weight_decay': 1e-4, #tune.grid_search([1e-4]), + 'iterations_per_epoch': 100, #tune.grid_search([100]), + }, + 'seed': 0, } ) diff --git a/src/safe_options/options.py b/src/safe_options/options.py index b4cfd8a..75f9e33 100644 --- a/src/safe_options/options.py +++ b/src/safe_options/options.py @@ -52,7 +52,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) - logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length , epoch) gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) @@ -63,7 +64,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value else: generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode , epoch) #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) @@ -76,7 +78,9 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value 'epoch': epoch, 'value': value, 'policy': policy, - 'gen/mean_reward_per_episode': gen_mean_reward_per_episode, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), }) return value, policy @@ -93,8 +97,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v generator_data = OptionsRollout(HLBuffer(*hl_data), Buffer(*ll_data)) generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) - - logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length, epoch) gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) @@ -105,7 +109,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v else: generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode, epoch) #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) @@ -118,7 +123,9 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v 'epoch': epoch, 'value': value, 'policy': policy, - 'gen/mean_reward_per_episode': gen_mean_reward_per_episode, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), }) for lr_scheduler in lr_schedulers: