diff --git a/intersimple-expert-data-setobs2-loc0-track0.pt b/intersimple-expert-data-setobs2-loc0-track0.pt new file mode 100644 index 0000000..6192f97 Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track0.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track1.pt b/intersimple-expert-data-setobs2-loc0-track1.pt new file mode 100644 index 0000000..e8a881a Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track1.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track2.pt b/intersimple-expert-data-setobs2-loc0-track2.pt new file mode 100644 index 0000000..f099ecf Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track2.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track3.pt b/intersimple-expert-data-setobs2-loc0-track3.pt new file mode 100644 index 0000000..3c51278 Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track3.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track4.pt b/intersimple-expert-data-setobs2-loc0-track4.pt new file mode 100644 index 0000000..8834b90 Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track4.pt differ diff --git a/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py b/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py index 28139f2..0b24e6e 100644 --- a/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py +++ b/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py @@ -1,10 +1,13 @@ +import sys +sys.path.append('../../../../') + import torch import functools -from core.sampling import rollout_sb3 -from intersim.envs import IntersimpleLidarFlatRandom +from src.core.sampling import rollout_sb3 +from intersim.envs import IntersimpleLidarFlatIncrementingAgent from intersim.envs.intersimple import speed_reward from intersim.expert import NormalizedIntersimpleExpert -from util.wrappers import CollisionPenaltyWrapper, Setobs +from src.util.wrappers import CollisionPenaltyWrapper, Setobs import numpy as np from gym.wrappers import TransformObservation @@ -26,7 +29,9 @@ obs_max = np.array([ [50, np.pi, 20, 20, np.pi, 1e-1], ]).reshape(-1) -env = IntersimpleLidarFlatRandom( +env = IntersimpleLidarFlatIncrementingAgent( + loc=0, + track=4, n_rays=5, reward=functools.partial( speed_reward, @@ -42,7 +47,8 @@ env = Setobs(TransformObservation( collision_distance=6, collision_penalty=100 ), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) )) -expert_data = rollout_sb3(env, policy, n_episodes=2048, max_steps_per_episode=200) +print(env.nv, 'vehicles') +expert_data = rollout_sb3(env, policy, n_episodes=150, max_steps_per_episode=200) states, actions, rewards, dones = expert_data print(f'Expert mean episode length {(~dones).sum() / states.shape[0]}') @@ -50,4 +56,4 @@ print(f'Expert mean reward per episode {rewards[~dones].sum() / states.shape[0]} print(f'Observation mean', states[~dones].mean(0)) print(f'Observation std', states[~dones].std(0)) -torch.save(expert_data, 'intersimple-expert-data-setobs2.pt') +torch.save(expert_data, 'intersimple-expert-data-setobs2-loc0-track4.pt') diff --git a/sgail-ppo-options-setobs2.py b/sgail-ppo-options-setobs2.py index b004873..82f3588 100644 --- a/sgail-ppo-options-setobs2.py +++ b/sgail-ppo-options-setobs2.py @@ -42,7 +42,7 @@ def training_function(config): speed_reward, collision_penalty=0 ), - stop_on_collision=True, + stop_on_collision=False, ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) ), options=[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], safe_actions_collision_method='circle', abort_unsafe_collision_method='circle') for _ in range(60)] @@ -58,7 +58,17 @@ def training_function(config): discriminator = DeepsetDiscriminator() # config net architecture disc_opt = torch.optim.Adam(discriminator.parameters(), lr=config['discriminator']['learning_rate'], weight_decay=config['discriminator']['weight_decay']) - expert_data = torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2.pt')) + expert_data = [ + torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track0.pt')), + torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track1.pt')), + torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track2.pt')), + torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track3.pt')), + ] + d0 = [d[0] for d in expert_data] + d1 = [d[1] for d in expert_data] + d2 = [d[2] for d in expert_data] + d3 = [d[3] for d in expert_data] + expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3)) expert_data = Buffer(*expert_data) def callback(info): @@ -91,11 +101,11 @@ analysis = tune.run( training_function, config={ 'policy': { - 'learning_rate': tune.grid_search([1e-5, 7e-5, 3e-4]), - 'learning_rate_decay': tune.grid_search([1.0, 0.98]), - 'clip_ratio': tune.grid_search([0.2, 0.1]), + 'learning_rate': tune.grid_search([3e-4]), + 'learning_rate_decay': tune.grid_search([1.0]), + 'clip_ratio': tune.grid_search([0.2]), 'iterations_per_epoch': tune.grid_search([100]), - 'hidden_layer_size': tune.grid_search([10, 25, 50]) + 'hidden_layer_size': tune.grid_search([10]) }, 'value': { 'learning_rate': tune.grid_search([1e-3]), @@ -109,7 +119,7 @@ analysis = tune.run( } ) -print('Best config: ', analysis.get_best_config(metric='gen_mean_reward_per_episode', mode='min')) +print('Best config: ', analysis.get_best_config(metric='gen_mean_reward_per_episode', mode='max')) # %% # policy = SetMaskedDiscretePolicy(env_fn(0).action_space.n)