diff --git a/eval_experiments.py b/eval_experiments.py index f9f1c8e..53e3684 100644 --- a/eval_experiments.py +++ b/eval_experiments.py @@ -1,17 +1,23 @@ import os from src.eval_main import eval_main from src.evaluation.utils import load_and_average +import torch +import json + +activations = [torch.nn.Tanh, torch.nn.LeakyReLU] def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=False): + exclude_keys_from_policy_kwargs = {'learning_rate', 'learning_rate_decay', 'clip_ratio', 'iterations_per_epoch', 'option'} policy_kwargs = {} + if method in ['expert', 'idm']: env, env_kwargs ='NRasterizedRouteIncrementingAgent', {} elif method in ['bc','gail']: env='NormalizedContinuousEvalEnv' env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000} elif method in ['hail']: - env = 'NormalizedOptionsEvalEnv' + env = 'NormalizedSafeOptionsEvalEnv' env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None} elif method in ['shail']: env = 'NormalizedSafeOptionsEvalEnv' @@ -23,7 +29,18 @@ def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=F if folder is not None: files = [os.path.join(folder, f) for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))] - print('%i folders found in %s folder' %(len(files), folder)) + files = [f for f in files if f.endswith('.pt')] + with open(os.path.join(folder, 'config.json'), 'rb') as f: + config = json.load(f) + print('%i policy files found in %s folder' %(len(files), folder)) + print('found policy config', config['policy']) + + policy_config = {k: v for k, v in config['policy'].items() if k not in exclude_keys_from_policy_kwargs} + policy_config['activation'] = activations[policy_config['activation']] + print('final policy config', policy_config) + + policy_kwargs.update(policy_config) + print('final policy kwargs', policy_kwargs) if not skip_running: for policy_file in files: diff --git a/shail-experiment.py b/shail-experiment.py index 4195759..598efa9 100644 --- a/shail-experiment.py +++ b/shail-experiment.py @@ -247,6 +247,13 @@ if __name__ == '__main__': os.makedirs(savepath) import shutil + + # save config + shutil.copyfile( + args.test, + os.path.join(savepath, 'config.json') + ) + for i in range(args.test_seeds): s = analysis._checkpoints[i]['config']['seed'] check_dir = analysis._checkpoints[i]['logdir'] diff --git a/src/eval_main.py b/src/eval_main.py index 2e50286..37e78bd 100644 --- a/src/eval_main.py +++ b/src/eval_main.py @@ -41,33 +41,33 @@ def load_policy(method:str, if method == 'idm': policy = IDMRulePolicy(env, **policy_kwargs) elif method == 'bc': - policy = SetPolicy(env.action_space.shape[-1]) + policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs) policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'gail-trpo': - policy = SetPolicy(env.action_space.shape[-1]) + policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs) policy(torch.zeros(env.observation_space.shape)) policy = ReparamPolicy(policy) policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'gail': - policy = SetPolicy(env.action_space.shape[-1]) + policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs) policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'rail': raise NotImplementedError elif method == 'hail-trpo': - policy = SetDiscretePolicy(env.action_space.n) + policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs) policy(torch.zeros(env.observation_space.shape)) policy = ReparamPolicy(policy) policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'hail': - policy = SetDiscretePolicy(env.action_space.n) + policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs) policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'shail-trpo': - policy = SetMaskedDiscretePolicy(env.action_space.n) + policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs) policy( torch.zeros(env.observation_space['observation'].shape), torch.zeros(env.observation_space['safe_actions'].shape) @@ -76,7 +76,7 @@ def load_policy(method:str, policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'shail': - policy = SetMaskedDiscretePolicy(env.action_space.n) + policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs) policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() else: diff --git a/test_policies/bc/expA/config.json b/test_policies/bc/expA/config.json new file mode 100644 index 0000000..42fee47 --- /dev/null +++ b/test_policies/bc/expA/config.json @@ -0,0 +1,15 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "train_epochs": 300, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/bc/expB/config.json b/test_policies/bc/expB/config.json new file mode 100644 index 0000000..9d81a2f --- /dev/null +++ b/test_policies/bc/expB/config.json @@ -0,0 +1,15 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "train_epochs": 300, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/gail/expA/config.json b/test_policies/gail/expA/config.json new file mode 100644 index 0000000..e254f61 --- /dev/null +++ b/test_policies/gail/expA/config.json @@ -0,0 +1,31 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/gail/expB/config.json b/test_policies/gail/expB/config.json new file mode 100644 index 0000000..b05e446 --- /dev/null +++ b/test_policies/gail/expB/config.json @@ -0,0 +1,31 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/hail-etienne/expA/config.json b/test_policies/hail-etienne/expA/config.json new file mode 100644 index 0000000..dbf59ca --- /dev/null +++ b/test_policies/hail-etienne/expA/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/hail-etienne/expB/config.json b/test_policies/hail-etienne/expB/config.json new file mode 100644 index 0000000..0fae03f --- /dev/null +++ b/test_policies/hail-etienne/expB/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/hail/expA/config.json b/test_policies/hail/expA/config.json new file mode 100644 index 0000000..ca9fe11 --- /dev/null +++ b/test_policies/hail/expA/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 90, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/hail/expB/config.json b/test_policies/hail/expB/config.json new file mode 100644 index 0000000..a93d5e1 --- /dev/null +++ b/test_policies/hail/expB/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 85, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/shail-etienne/expA/config.json b/test_policies/shail-etienne/expA/config.json new file mode 100644 index 0000000..6de9706 --- /dev/null +++ b/test_policies/shail-etienne/expA/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/shail-etienne/expB/config.json b/test_policies/shail-etienne/expB/config.json new file mode 100644 index 0000000..04139c2 --- /dev/null +++ b/test_policies/shail-etienne/expB/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/shail/expA/config.json b/test_policies/shail/expA/config.json new file mode 100644 index 0000000..76472e0 --- /dev/null +++ b/test_policies/shail/expA/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 90, + "seed": 0 +} \ No newline at end of file diff --git a/test_policies/shail/expB/config.json b/test_policies/shail/expB/config.json new file mode 100644 index 0000000..3a35200 --- /dev/null +++ b/test_policies/shail/expB/config.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 85, + "seed": 0 +} \ No newline at end of file