updating experiment B vecenv

bc-experiment.py

@@ -162,9 +162,9 @@ if __name__ == '__main__':
                 },
                 'policy': {
                     'learning_rate': 3e-4, 
-                    'learning_rate_decay': 1.0, 
+                    'learning_rate_decay': tune.grid_search([0.001, 1.0]), 
-                    'hidden_layer_size': tune.grid_search([20, 40]),
+                    'hidden_layer_size': tune.grid_search([10, 20, 40, 80]),
-                    'n_hidden_layers': tune.grid_search([2, 3]), 
+                    'n_hidden_layers': tune.grid_search([2, 3, 4]), 
                     'activation':0,
                 },
                 'train_epochs': args.epochs,

best_configs/shail-etienne4_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 500,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 150,
+    "seed": 0
+}

eval_experiments.py

@@ -1,17 +1,23 @@
 import os
 from src.eval_main import eval_main
 from src.evaluation.utils import load_and_average
+import torch
+import json
+
+activations = [torch.nn.Tanh, torch.nn.LeakyReLU]
 
 def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=False):
     
+    exclude_keys_from_policy_kwargs = {'learning_rate', 'learning_rate_decay', 'clip_ratio', 'iterations_per_epoch', 'option'}
     policy_kwargs = {}
+
     if method in ['expert', 'idm']:
         env, env_kwargs ='NRasterizedRouteIncrementingAgent', {}
     elif method in ['bc','gail']:
         env='NormalizedContinuousEvalEnv' 
         env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
     elif method in ['hail']:
-        env = 'NormalizedOptionsEvalEnv'
+        env = 'NormalizedSafeOptionsEvalEnv'
         env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None}
     elif method in ['shail']:
         env = 'NormalizedSafeOptionsEvalEnv'
@@ -23,7 +29,18 @@ def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=F
 
     if folder is not None:
         files = [os.path.join(folder, f) for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
-        print('%i folders found in %s folder' %(len(files), folder))
+        files = [f for f in files if f.endswith('.pt')]
+        with open(os.path.join(folder, 'config.json'), 'rb') as f:
+            config = json.load(f)
+        print('%i policy files found in %s folder' %(len(files), folder))
+        print('found policy config', config['policy'])
+
+    policy_config = {k: v for k, v in config['policy'].items() if k not in exclude_keys_from_policy_kwargs}
+    policy_config['activation'] = activations[policy_config['activation']]
+    print('final policy config', policy_config)
+
+    policy_kwargs.update(policy_config)
+    print('final policy kwargs', policy_kwargs)
 
     if not skip_running:
         for policy_file in files:

shail-experiment.py

@@ -62,7 +62,7 @@ def training_function(config):
             ), options=option_list[config['policy']['option']],
             safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], 
             abort_unsafe_collision_method=config['trainenv']['abort_unsafe_collision_method'],
-            ) for _ in range(60)]
+            ) for _ in range(120)]
 
     elif config['experiment'] == 'B': 
         envs = sum([[SafeOptionsEnv(Setobs(
@@ -78,7 +78,7 @@ def training_function(config):
             ), options=option_list[config['policy']['option']],
             safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], 
             abort_unsafe_collision_method=config['trainenv']['abort_unsafe_collision_method'],
-            ) for _ in range(15)] for track in range(4)],[])
+            ) for _ in range(30)] for track in range(4)],[])
     
     else:
         raise NotImplementedError
@@ -142,7 +142,7 @@ def training_function(config):
         v_opt=v_opt,
         v_iters=config['value']['iterations_per_epoch'],
         epochs=config['train_epochs'],
-        rollout_episodes=60, 
+        rollout_episodes=120, 
         rollout_steps=60,
         gamma=0.99,
         gae_lambda=0.9,
@@ -185,11 +185,11 @@ if __name__ == '__main__':
                     'learning_rate': 3e-4, 
                     'learning_rate_decay': 1.0, 
                     'clip_ratio': 0.2, 
-                    'iterations_per_epoch': 100,
+                    'iterations_per_epoch': tune.grid_search([250, 500, 750]),
-                    'hidden_layer_size': tune.grid_search([20, 40]),
+                    'hidden_layer_size': 40, #tune.grid_search([20, 40]),
-                    'n_hidden_layers': tune.grid_search([2, 3]), 
+                    'n_hidden_layers': 3, #tune.grid_search([2, 3]), 
                     'activation':0,
-                    'option': tune.grid_search(list(range(len(option_list))))
+                    'option': 0, #tune.grid_search(list(range(len(option_list))))
                 },
                 'value': {
                     'learning_rate': 1e-3, 
@@ -199,8 +199,8 @@ if __name__ == '__main__':
                     'learning_rate': 1e-3, 
                     'weight_decay': 1e-4, 
                     'iterations_per_epoch': 100, 
-                    'n_hidden_layers_element': tune.grid_search([3,4]),
+                    'n_hidden_layers_element': 3, #tune.grid_search([3,4]),
-                    'n_hidden_layers_global': tune.grid_search([1,2]),
+                    'n_hidden_layers_global': 2, #tune.grid_search([1,2]),
                     'hidden_layer_size': 10, 
                     'activation': 0,
                 },
@@ -247,6 +247,13 @@ if __name__ == '__main__':
             os.makedirs(savepath)
         
         import shutil
+
+        # save config
+        shutil.copyfile(
+            args.test,
+            os.path.join(savepath, 'config.json')
+        )
+
         for i in range(args.test_seeds):
             s = analysis._checkpoints[i]['config']['seed']
             check_dir = analysis._checkpoints[i]['logdir']

src/eval_main.py

@@ -41,33 +41,33 @@ def load_policy(method:str,
     if method == 'idm':
         policy = IDMRulePolicy(env, **policy_kwargs)
     elif method == 'bc':
-        policy = SetPolicy(env.action_space.shape[-1])
+        policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'gail-trpo':
-        policy = SetPolicy(env.action_space.shape[-1])
+        policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs)
         policy(torch.zeros(env.observation_space.shape))
         policy = ReparamPolicy(policy)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'gail':
-        policy = SetPolicy(env.action_space.shape[-1])
+        policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'rail':
         raise NotImplementedError
     elif method == 'hail-trpo':
-        policy = SetDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy(torch.zeros(env.observation_space.shape))
         policy = ReparamPolicy(policy)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'hail':
-        policy = SetDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'shail-trpo':
-        policy = SetMaskedDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy(
             torch.zeros(env.observation_space['observation'].shape),
             torch.zeros(env.observation_space['safe_actions'].shape)
@@ -76,7 +76,7 @@ def load_policy(method:str,
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'shail':
-        policy = SetMaskedDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     else:

test_policies/bc/expA/config.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 300,
+    "seed": 0
+}

test_policies/bc/expB/config.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 300,
+    "seed": 0
+}

test_policies/gail/expA/config.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/gail/expB/config.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/hail-etienne/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/hail-etienne/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/hail/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 90,
+    "seed": 0
+}

test_policies/hail/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 85,
+    "seed": 0
+}

test_policies/shail-etienne/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/shail-etienne/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/shail/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 90,
+    "seed": 0
+}

test_policies/shail/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 85,
+    "seed": 0
+}