commiting json files and intermediate settings used to run baseline methods around 02-26, 16ocklock. Next will set off big tuning runs for gail, gailtrpo, and bc (with committed values)

bc-experiment.py

@@ -44,6 +44,8 @@ def training_function(config):
     np.random.seed(config['seed'])
     torch.manual_seed(config['seed'])
 
+    # choose validation environment
+    if config['experiment'] == 'A': 
         envs = [Setobs(TransformObservation(CollisionPenaltyWrapper(
             IntersimpleLidarFlatRandom(
                 n_rays=5,
@@ -56,6 +58,22 @@ def training_function(config):
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
         )) for _ in range(60)]
+    elif config['experiment'] == 'B': 
+        envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper(
+            IntersimpleLidarFlatRandom(
+                n_rays=5,
+                reward=functools.partial(
+                    speed_reward,
+                    collision_penalty=0
+                ),
+                check_collisions=True,
+                stop_on_collision=config['trainenv']['stop_on_collision'],
+            ), collision_distance=6, collision_penalty=100),
+            lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
+        )) for _ in range(15)] for track in range(4)],[])
+    else:
+        raise NotImplementedError
+
     env_fn = lambda i: envs[i]
 
     # load expert data
@@ -99,7 +117,7 @@ def training_function(config):
         pi_opt.step()
         pi_lr_scheduler.step()
 
-        if epoch % 100 == 0:
+        if epoch % 10 == 0:
             gen_states, gen_actions, gen_rewards, gen_dones, gen_collisions = rollout(env_fn, policy.cpu(), n_episodes=60, max_steps_per_episode=200)
             gen_mean_episode_length = (~gen_dones).sum() / gen_states.shape[0]
             gen_mean_reward_per_episode = gen_rewards[~gen_dones].sum() / gen_states.shape[0]
@@ -114,7 +132,7 @@ def training_function(config):
         
         # save model checkpoints
         ep = epoch + 1
-        if (ep % 25 == 0):
+        if (ep % 50 == 0):
             torch.save(policy.state_dict(), f'policy_epoch{ep}.pt')
 
     # save model
@@ -124,7 +142,7 @@ if __name__ == '__main__':
     import argparse
     parser = argparse.ArgumentParser()
     parser.add_argument('--train', choices=['A', 'B'])
-    parser.add_argument('--epochs', type=int, default=200)
+    parser.add_argument('--epochs', type=int, default=1000)
     parser.add_argument('--test', type=str, help='path to config file to run final training on')
     parser.add_argument('--test_seeds', type=int, default=5)
     parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over')

best_configs/bc_expA.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 1000,
+    "seed": 0
+}

best_configs/bc_expB.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 1000,
+    "seed": 0
+}

best_configs/gail-trpo_expA.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "delta": 0.01,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/gail-trpo_expB.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "delta": 0.01,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/gail_expA.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/gail_expB.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

gail-experiment.py

@@ -187,8 +187,8 @@ if __name__ == '__main__':
                 },
                 'discriminator': {
                     'learning_rate': 1e-3, 
-                    'weight_decay': 1e-5, 
-                    'iterations_per_epoch': 500, 
+                    'weight_decay': 1e-4, # 1e-5 Fixme
+                    'iterations_per_epoch': 100, # 500 Fixme
                     'n_hidden_layers_element': tune.grid_search([3,4]),
                     'n_hidden_layers_global': tune.grid_search([1,2]),
                     'hidden_layer_size': 10, 

gail-trpo-experiment.py

@@ -184,8 +184,8 @@ if __name__ == '__main__':
                 },
                 'discriminator': {
                     'learning_rate': 1e-3, 
-                    'weight_decay': 1e-5, 
-                    'iterations_per_epoch': 500, 
+                    'weight_decay': 1e-4, # 1e-5 Fixme
+                    'iterations_per_epoch': 100, # 500 Fixme
                     'n_hidden_layers_element': tune.grid_search([3,4]),
                     'n_hidden_layers_global': tune.grid_search([1,2]),
                     'hidden_layer_size': 10,