Incrementing agent expert data, smaller policy network

scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py

@@ -1,10 +1,13 @@
+import sys
+sys.path.append('../../../../')
+
 import torch
 import functools
-from core.sampling import rollout_sb3
-from intersim.envs import IntersimpleLidarFlatRandom
+from src.core.sampling import rollout_sb3
+from intersim.envs import IntersimpleLidarFlatIncrementingAgent
 from intersim.envs.intersimple import speed_reward
 from intersim.expert import NormalizedIntersimpleExpert
-from util.wrappers import CollisionPenaltyWrapper, Setobs
+from src.util.wrappers import CollisionPenaltyWrapper, Setobs
 import numpy as np
 from gym.wrappers import TransformObservation
 
@@ -26,7 +29,9 @@ obs_max = np.array([
     [50, np.pi, 20, 20, np.pi, 1e-1],
 ]).reshape(-1)
 
-env = IntersimpleLidarFlatRandom(
+env = IntersimpleLidarFlatIncrementingAgent(
+    loc=0,
+    track=4,
     n_rays=5,
     reward=functools.partial(
         speed_reward,
@@ -42,7 +47,8 @@ env = Setobs(TransformObservation(
         collision_distance=6, collision_penalty=100
     ), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
 ))
-expert_data = rollout_sb3(env, policy, n_episodes=2048, max_steps_per_episode=200)
+print(env.nv, 'vehicles')
+expert_data = rollout_sb3(env, policy, n_episodes=150, max_steps_per_episode=200)
 
 states, actions, rewards, dones = expert_data
 print(f'Expert mean episode length {(~dones).sum() / states.shape[0]}')
@@ -50,4 +56,4 @@ print(f'Expert mean reward per episode {rewards[~dones].sum() / states.shape[0]}
 print(f'Observation mean', states[~dones].mean(0))
 print(f'Observation std', states[~dones].std(0))
 
-torch.save(expert_data, 'intersimple-expert-data-setobs2.pt')
+torch.save(expert_data, 'intersimple-expert-data-setobs2-loc0-track4.pt')

sgail-ppo-options-setobs2.py

@@ -42,7 +42,7 @@ def training_function(config):
                 speed_reward,
                 collision_penalty=0
             ),
-            stop_on_collision=True,
+            stop_on_collision=False,
         ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10))
     ), options=[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], safe_actions_collision_method='circle', abort_unsafe_collision_method='circle') for _ in range(60)]
 
@@ -58,7 +58,17 @@ def training_function(config):
     discriminator = DeepsetDiscriminator() # config net architecture
     disc_opt = torch.optim.Adam(discriminator.parameters(), lr=config['discriminator']['learning_rate'], weight_decay=config['discriminator']['weight_decay'])
 
-    expert_data = torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2.pt'))
+    expert_data = [
+        torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track0.pt')),
+        torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track1.pt')),
+        torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track2.pt')),
+        torch.load(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'intersimple-expert-data-setobs2-loc0-track3.pt')),
+    ]
+    d0 = [d[0] for d in expert_data]
+    d1 = [d[1] for d in expert_data]
+    d2 = [d[2] for d in expert_data]
+    d3 = [d[3] for d in expert_data]
+    expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3))
     expert_data = Buffer(*expert_data)
 
     def callback(info):
@@ -91,11 +101,11 @@ analysis = tune.run(
     training_function,
     config={
         'policy': {
-            'learning_rate': tune.grid_search([1e-5, 7e-5, 3e-4]),
-            'learning_rate_decay': tune.grid_search([1.0, 0.98]),
-            'clip_ratio': tune.grid_search([0.2, 0.1]),
+            'learning_rate': tune.grid_search([3e-4]),
+            'learning_rate_decay': tune.grid_search([1.0]),
+            'clip_ratio': tune.grid_search([0.2]),
             'iterations_per_epoch': tune.grid_search([100]),
-            'hidden_layer_size': tune.grid_search([10, 25, 50])
+            'hidden_layer_size': tune.grid_search([10])
         },
         'value': {
             'learning_rate': tune.grid_search([1e-3]),
@@ -109,7 +119,7 @@ analysis = tune.run(
     }
 )
 
-print('Best config: ', analysis.get_best_config(metric='gen_mean_reward_per_episode', mode='min'))
+print('Best config: ', analysis.get_best_config(metric='gen_mean_reward_per_episode', mode='max'))
 
 # %%
 # policy = SetMaskedDiscretePolicy(env_fn(0).action_space.n)