diff --git a/sgail-ppo-options-setobs2.py b/sgail-ppo-options-setobs2.py index 6f9c4fd..5d7f569 100644 --- a/sgail-ppo-options-setobs2.py +++ b/sgail-ppo-options-setobs2.py @@ -6,8 +6,8 @@ from src.safe_options.options import gail_ppo, Buffer from src.core.value import SetValue from src.safe_options.policy import SetMaskedDiscretePolicy from src.core.discriminator import DeepsetDiscriminator -import torch.optim -import torch.nn +import torch + from intersim.envs import IntersimpleLidarFlatRandom from intersim.envs.intersimple import speed_reward import functools @@ -18,25 +18,35 @@ from torch.utils.tensorboard import SummaryWriter from ray import tune DIR = os.path.dirname(os.path.abspath(__file__)) +option_list = [[(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5]], + [(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20]], + [(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5, 10, 20]], # was the best in training with single hidden layer, but very slow + [(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20, 40]], + [(vel, time) for vel in [0, 2, 5, 10] for time in [5, 10, 20]], + [(vel, time) for vel in [0, 3, 10] for time in [5, 20, 40]] +] + +obs_min = np.array([ + [-1000, -1000, 0, -np.pi, -1e-1, 0.], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], +]).reshape(-1) + +obs_max = np.array([ + [1000, 1000, 20, np.pi, 1e-1, 0.], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], +]).reshape(-1) def training_function(config): - obs_min = np.array([ - [-1000, -1000, 0, -np.pi, -1e-1, 0.], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - ]).reshape(-1) - - obs_max = np.array([ - [1000, 1000, 20, np.pi, 1e-1, 0.], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - ]).reshape(-1) + np.random.seed(config['seed']) + torch.manual_seed(config['seed']) envs = sum([[SafeOptionsEnv(Setobs( TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom( @@ -45,9 +55,9 @@ def training_function(config): speed_reward, collision_penalty=0 ), - stop_on_collision=False, track=track, + stop_on_collision=config['stop_on_collision'], track=track, ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) - ), options=config['policy']['option'], safe_actions_collision_method='circle', + ), options=option_list[config['policy']['option']], safe_actions_collision_method='circle', abort_unsafe_collision_method='circle') for _ in range(20)] for track in range(4)],[]) env_fn = lambda i: envs[i] @@ -79,7 +89,9 @@ def training_function(config): expert_data = Buffer(*expert_data) def callback(info): - tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode']) + tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'], + disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'], + mean_episode_length=info['gen/mean_episode_length']) value, policy = gail_ppo( env_fn=env_fn, @@ -91,9 +103,9 @@ def training_function(config): value=value, v_opt=v_opt, v_iters=config['value']['iterations_per_epoch'], - epochs=200, - rollout_episodes=60, - rollout_steps=60, + epochs=2, # 200 FIXME + rollout_episodes=6, #60, FIXME + rollout_steps=6, #60, FIXME gamma=0.99, gae_lambda=0.9, clip_ratio=config['policy']['clip_ratio'], @@ -104,33 +116,33 @@ def training_function(config): lr_schedulers=[pi_lr_scheduler], ) + # save value, policy + + analysis = tune.run( training_function, config={ + 'stop_on_collision': tune.grid_search([True, False]), 'policy': { - 'learning_rate': tune.grid_search([3e-4]), - 'learning_rate_decay': tune.grid_search([1.0]), - 'clip_ratio': tune.grid_search([0.2]), - 'iterations_per_epoch': tune.grid_search([100]), - 'hidden_layer_size': tune.grid_search([10, 20, 30]), - 'n_hidden_layers': tune.grid_search([1, 2, 3]), + 'learning_rate': 3e-4, # tune.grid_search([3e-4]), + 'learning_rate_decay': 1.0, #tune.grid_search([1.0]), + 'clip_ratio': 0.2, #tune.grid_search([0.2]), + 'iterations_per_epoch': 100, #tune.grid_search([100]), + 'hidden_layer_size': tune.grid_search([10, 20, 40]), + 'n_hidden_layers': 1, #tune.grid_search([1, 2, 3]), #FIXME 'activation':tune.grid_search([torch.nn.LeakyReLU, torch.nn.Tanh]), - 'option': tune.grid_search([[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], - [(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5), - (0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10)], - [(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5), - (0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10), - (0, 20), (1, 20), (2, 20), (4, 20), (6, 20), (8, 20), (20, 10)]]) + 'option': tune.grid_search(list(range(len(option_list)))) }, 'value': { - 'learning_rate': tune.grid_search([1e-3]), - 'iterations_per_epoch': tune.grid_search([1000]), + 'learning_rate': 1e-3, # tune.grid_search([1e-3]), + 'iterations_per_epoch': 1000, #tune.grid_search([1000]), }, 'discriminator': { - 'learning_rate': tune.grid_search([1e-3]), - 'weight_decay': tune.grid_search([1e-4]), - 'iterations_per_epoch': tune.grid_search([100]), - } + 'learning_rate': 1e-3, #tune.grid_search([1e-3]), + 'weight_decay': 1e-4, #tune.grid_search([1e-4]), + 'iterations_per_epoch': 100, #tune.grid_search([100]), + }, + 'seed': 0, } ) diff --git a/src/safe_options/options.py b/src/safe_options/options.py index b4cfd8a..75f9e33 100644 --- a/src/safe_options/options.py +++ b/src/safe_options/options.py @@ -52,7 +52,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) - logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length , epoch) gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) @@ -63,7 +64,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value else: generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode , epoch) #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) @@ -76,7 +78,9 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value 'epoch': epoch, 'value': value, 'policy': policy, - 'gen/mean_reward_per_episode': gen_mean_reward_per_episode, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), }) return value, policy @@ -93,8 +97,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v generator_data = OptionsRollout(HLBuffer(*hl_data), Buffer(*ll_data)) generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) - - logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length, epoch) gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) @@ -105,7 +109,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v else: generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode, epoch) #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) @@ -118,7 +123,9 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v 'epoch': epoch, 'value': value, 'policy': policy, - 'gen/mean_reward_per_episode': gen_mean_reward_per_episode, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), }) for lr_scheduler in lr_schedulers: