diff --git a/.gitignore b/.gitignore index e8de66b..ddd1d71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.png *.pkl *.pt *.zip diff --git a/bc-experiment.py b/bc-experiment.py new file mode 100644 index 0000000..b50489b --- /dev/null +++ b/bc-experiment.py @@ -0,0 +1,212 @@ +# %% +import os + +from tqdm import tqdm +from src.core.sampling import rollout +from src.core.gail import gail_ppo, Buffer +from src.core.value import SetValue +from src.core.policy import SetPolicy +from src.core.discriminator import DeepsetDiscriminator +import torch + +from intersim.envs import IntersimpleLidarFlatRandom +from intersim.envs.intersimple import speed_reward +import functools +from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Setobs +import numpy as np +from torch.utils.tensorboard import SummaryWriter +from ray import tune +from datetime import datetime +import json + +DIR = os.path.dirname(os.path.abspath(__file__)) +activations = [torch.nn.Tanh, torch.nn.LeakyReLU] + +obs_min = np.array([ + [-1000, -1000, 0, -np.pi, -1e-1, 0.], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], +]).reshape(-1) + +obs_max = np.array([ + [1000, 1000, 20, np.pi, 1e-1, 0.], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], +]).reshape(-1) + +def training_function(config): + np.random.seed(config['seed']) + torch.manual_seed(config['seed']) + + # choose validation environment + if config['experiment'] == 'A': + envs = [Setobs(TransformObservation(CollisionPenaltyWrapper( + IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], + ), collision_distance=6, collision_penalty=100), + lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) + )) for _ in range(60)] + elif config['experiment'] == 'B': + envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper( + IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], + ), collision_distance=6, collision_penalty=100), + lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) + )) for _ in range(15)] for track in range(4)],[]) + else: + raise NotImplementedError + + env_fn = lambda i: envs[i] + + # load expert data + + if config['experiment'] == 'A': + expert_data = torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')) + elif config['experiment'] == 'B': + expert_data = [ + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track1.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track2.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track3.pt')), + ] + d0 = [d[0] for d in expert_data] + d1 = [d[1] for d in expert_data] + d2 = [d[2] for d in expert_data] + d3 = [d[3] for d in expert_data] + expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3)) + + expert_data = Buffer(*expert_data) + + # configure and train policy + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + policy = SetPolicy(expert_data.actions.shape[-1], + n_hidden_layers=config['policy']['n_hidden_layers'], + hidden_layer_size=config['policy']['hidden_layer_size'], + activation=activations[config['policy']['activation']] ) # config net architecture + policy = policy.to(device) + + pi_opt = torch.optim.Adam(policy.parameters(), lr=config['policy']['learning_rate']) + pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=config['policy']['learning_rate_decay']) + + expert_states = expert_data.states[~expert_data.dones].to(device) + expert_actions = expert_data.actions[~expert_data.dones].to(device) + + for epoch in range(config['train_epochs']): + pi_opt.zero_grad() + loss = -policy.log_prob(policy(expert_states), expert_actions).mean() + loss.backward() + pi_opt.step() + pi_lr_scheduler.step() + + if epoch % 25 == 0: + gen_states, gen_actions, gen_rewards, gen_dones, gen_collisions = rollout(env_fn, policy.cpu(), n_episodes=60, max_steps_per_episode=200) + gen_mean_episode_length = (~gen_dones).sum() / gen_states.shape[0] + gen_mean_reward_per_episode = gen_rewards[~gen_dones].sum() / gen_states.shape[0] + gen_collision_rate = (1. * gen_collisions.any(-1)).mean() + + tune.report( + gen_mean_reward_per_episode=gen_mean_reward_per_episode.item(), + mean_episode_length=gen_mean_episode_length.item(), + gen_collision_rate=gen_collision_rate.item(), + loss=loss.item(), + ) + + # save model checkpoints + ep = epoch + 1 + if (ep % 50 == 0): + torch.save(policy.state_dict(), f'policy_epoch{ep}.pt') + + # save model + torch.save(policy.state_dict(), 'policy_final.pt') + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--train', choices=['A', 'B']) + parser.add_argument('--epochs', type=int, default=1000) + parser.add_argument('--test', type=str, help='path to config file to run final training on') + parser.add_argument('--test_seeds', type=int, default=5) + parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over') + args = parser.parse_args() + + assert (args.train is None) ^ (args.test is None), 'Must either train on an experiment or test with a config file' + + # if no test config specified, train + if args.test is None: + print('Running Tuning for Experiment %s'%(args.train)) + analysis = tune.run( + training_function, + config={ + 'experiment': args.train, + 'trainenv': { + 'stop_on_collision': False, + }, + 'policy': { + 'learning_rate': 3e-4, + 'learning_rate_decay': 1.0, + 'hidden_layer_size': tune.grid_search([20, 40]), + 'n_hidden_layers': tune.grid_search([2, 3]), + 'activation':0, + }, + 'train_epochs': args.epochs, + 'seed': 0, + } + # TODO resources_per_trial={'gpu': 1} + ) + best_config = analysis.get_best_config(metric='gen_collision_rate', mode='min') + print('Best config: ', best_config) + + # safe best_config + if not os.path.isdir(os.path.join(DIR, 'best_configs')): + os.mkdir(os.path.join(DIR, 'best_configs')) + + # save gail + with open(os.path.join(DIR, 'best_configs',f'bc_exp{args.train}.json'), 'w', encoding='utf-8') as f: + json.dump(best_config, f, ensure_ascii=False, indent=4) + + # if config file specified, rerun it with appropriate number of seeds + else: + with open(args.test, 'rb') as f: + config = json.load(f) + + print(f'Retraining {args.test} with {args.test_seeds} seeds on experiment {config["experiment"]}') + + # rerun with appropriate number of seeds + rpt = {'cpu': int(args.test_cpus/args.test_seeds)} if (args.test_cpus is not None) else None + config['seed'] = tune.grid_search(list(range(1,args.test_seeds+1))) + analysis = tune.run(training_function, config=config, resources_per_trial=rpt) + + # move final policies to appropriate directory + split_ = os.path.basename(args.test).split('_') + model = split_[0] + exper = split_[-1].split('.')[0] + savepath = os.path.join('test_policies',model,exper) + + if not os.path.isdir(savepath): + os.makedirs(savepath) + + import shutil + for i in range(args.test_seeds): + s = analysis._checkpoints[i]['config']['seed'] + check_dir = analysis._checkpoints[i]['logdir'] + shutil.copyfile(os.path.join(check_dir,'policy_final.pt'), + os.path.join(savepath, f'policy_seed{s}.pt')) \ No newline at end of file diff --git a/best_configs/bc_expA.json b/best_configs/bc_expA.json new file mode 100644 index 0000000..42fee47 --- /dev/null +++ b/best_configs/bc_expA.json @@ -0,0 +1,15 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "train_epochs": 300, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/bc_expB.json b/best_configs/bc_expB.json new file mode 100644 index 0000000..9d81a2f --- /dev/null +++ b/best_configs/bc_expB.json @@ -0,0 +1,15 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "train_epochs": 300, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail-trpo_expA.json b/best_configs/gail-trpo_expA.json new file mode 100644 index 0000000..ff708f1 --- /dev/null +++ b/best_configs/gail-trpo_expA.json @@ -0,0 +1,31 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "delta": 0.01, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail-trpo_expB.json b/best_configs/gail-trpo_expB.json new file mode 100644 index 0000000..ae0fa67 --- /dev/null +++ b/best_configs/gail-trpo_expB.json @@ -0,0 +1,31 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "delta": 0.01, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail_expA.json b/best_configs/gail_expA.json new file mode 100644 index 0000000..e254f61 --- /dev/null +++ b/best_configs/gail_expA.json @@ -0,0 +1,31 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail_expB.json b/best_configs/gail_expB.json new file mode 100644 index 0000000..b05e446 --- /dev/null +++ b/best_configs/gail_expB.json @@ -0,0 +1,31 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail-etienne2_expA.json b/best_configs/hail-etienne2_expA.json new file mode 100644 index 0000000..a43762d --- /dev/null +++ b/best_configs/hail-etienne2_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 10, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail-etienne2_expB.json b/best_configs/hail-etienne2_expB.json new file mode 100644 index 0000000..6425d4f --- /dev/null +++ b/best_configs/hail-etienne2_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 10, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail-etienne3_expA.json b/best_configs/hail-etienne3_expA.json new file mode 100644 index 0000000..91775ab --- /dev/null +++ b/best_configs/hail-etienne3_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail-etienne3_expB.json b/best_configs/hail-etienne3_expB.json new file mode 100644 index 0000000..c75cfb5 --- /dev/null +++ b/best_configs/hail-etienne3_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail-etienne_expA.json b/best_configs/hail-etienne_expA.json new file mode 100644 index 0000000..dbf59ca --- /dev/null +++ b/best_configs/hail-etienne_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail-etienne_expB.json b/best_configs/hail-etienne_expB.json new file mode 100644 index 0000000..0fae03f --- /dev/null +++ b/best_configs/hail-etienne_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail_expA.json b/best_configs/hail_expA.json new file mode 100644 index 0000000..ca9fe11 --- /dev/null +++ b/best_configs/hail_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 90, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/hail_expB.json b/best_configs/hail_expB.json new file mode 100644 index 0000000..a93d5e1 --- /dev/null +++ b/best_configs/hail_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": null, + "abort_unsafe_collision_method": null + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 85, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail-etienne2_expA.json b/best_configs/shail-etienne2_expA.json new file mode 100644 index 0000000..a3f6b90 --- /dev/null +++ b/best_configs/shail-etienne2_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 10, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail-etienne2_expB.json b/best_configs/shail-etienne2_expB.json new file mode 100644 index 0000000..096f2ea --- /dev/null +++ b/best_configs/shail-etienne2_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 10, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail-etienne3_expA.json b/best_configs/shail-etienne3_expA.json new file mode 100644 index 0000000..385d63d --- /dev/null +++ b/best_configs/shail-etienne3_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail-etienne3_expB.json b/best_configs/shail-etienne3_expB.json new file mode 100644 index 0000000..d0d0b69 --- /dev/null +++ b/best_configs/shail-etienne3_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 3, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail-etienne_expA.json b/best_configs/shail-etienne_expA.json new file mode 100644 index 0000000..6de9706 --- /dev/null +++ b/best_configs/shail-etienne_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail-etienne_expB.json b/best_configs/shail-etienne_expB.json new file mode 100644 index 0000000..04139c2 --- /dev/null +++ b/best_configs/shail-etienne_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 4, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 3, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail_expA.json b/best_configs/shail_expA.json new file mode 100644 index 0000000..76472e0 --- /dev/null +++ b/best_configs/shail_expA.json @@ -0,0 +1,33 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 90, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/shail_expB.json b/best_configs/shail_expB.json new file mode 100644 index 0000000..3a35200 --- /dev/null +++ b/best_configs/shail_expB.json @@ -0,0 +1,33 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false, + "safe_actions_collision_method": "circle", + "abort_unsafe_collision_method": "circle" + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 20, + "n_hidden_layers": 2, + "activation": 0, + "option": 0 + }, + "value": { + "learning_rate": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 2, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 85, + "seed": 0 +} \ No newline at end of file diff --git a/checkpoints/gail-ppo-intersimple-setobs2.pt b/checkpoints/gail-ppo-intersimple-setobs2.pt new file mode 100644 index 0000000..800c679 Binary files /dev/null and b/checkpoints/gail-ppo-intersimple-setobs2.pt differ diff --git a/checkpoints/sgail-options-setobs2-Feb21_13-30-45.pt b/checkpoints/sgail-options-setobs2-Feb21_13-30-45.pt new file mode 100644 index 0000000..4f02fe6 Binary files /dev/null and b/checkpoints/sgail-options-setobs2-Feb21_13-30-45.pt differ diff --git a/checkpoints/sgail-options-setobs2.pt b/checkpoints/sgail-options-setobs2.pt deleted file mode 100644 index 5bc3e99..0000000 Binary files a/checkpoints/sgail-options-setobs2.pt and /dev/null differ diff --git a/checkpoints/sgail-ppo-options-setobs2-Feb18_12-53-23.pt b/checkpoints/sgail-ppo-options-setobs2-Feb18_12-53-23.pt new file mode 100644 index 0000000..76cfb53 Binary files /dev/null and b/checkpoints/sgail-ppo-options-setobs2-Feb18_12-53-23.pt differ diff --git a/checkpoints/sgail-ppo-options-setobs2-Feb18_16-25-08.pt b/checkpoints/sgail-ppo-options-setobs2-Feb18_16-25-08.pt new file mode 100644 index 0000000..7013a8d Binary files /dev/null and b/checkpoints/sgail-ppo-options-setobs2-Feb18_16-25-08.pt differ diff --git a/eval_experiments.py b/eval_experiments.py new file mode 100644 index 0000000..3eed2ed --- /dev/null +++ b/eval_experiments.py @@ -0,0 +1,83 @@ +import os +from src.eval_main import eval_main +from src.evaluation.utils import load_and_average + +def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=False): + + policy_kwargs = {} + if method in ['expert', 'idm']: + env, env_kwargs ='NRasterizedRouteIncrementingAgent', {} + elif method in ['bc','gail']: + env='NormalizedContinuousEvalEnv' + env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000} + elif method in ['hail']: + env = 'NormalizedOptionsEvalEnv' + env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000} + elif method in ['shail']: + env = 'NormalizedSafeOptionsEvalEnv' + env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000} + else: + raise NotImplementedError + + files = [''] + + if folder is not None: + files = [os.path.join(folder, f) for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))] + print('%i folders found in %s folder' %(len(files), folder)) + + if not skip_running: + for policy_file in files: + # run metrics on that file + outbase = eval_main(locations=locations, + method=method, + policy_file=policy_file, + policy_kwargs=policy_kwargs, + env=env, + env_kwargs=env_kwargs) + outfolder = os.path.dirname(outbase) + else: + locstr = 'loc_'+'_'.join([f'r{ro}t{tr}' for (ro,tr) in locations]) + if folder is None: + outfolder = os.path.join('out',method,locstr) + else: + path_items = folder.split('/') + outfolder = os.path.join('out', '/'.join(path_items[1:]), locstr) + + # load metrics from save_path + average_metrics = load_and_average(outfolder) + if method in ['expert', 'idm']: + latex_print(average_metrics, light=True) + else: + latex_print(average_metrics) + +def latex_print(am, light=False): + """ + print latex line + + am (Dict[str,tuple]): dict mapping metric_name to (mean, std) + """ + + print('success rate, distance travelled, RWSE_10, |DeltaV|, AccelJSD') + if light: + if 'rwse_10s' in am.keys(): + print("%2.1f& %2.1f & %1.2f & %2.1f& " + "%0.3f \\\\" %( 100*am['success rate'][0], am['mean travel distance'][0], am['rwse_10s'][0], + am['average absolute average velocity'][0],am['acceleration distribution divergence'][0] )) + return + + + print("%2.1f& %2.1f & $---$ & $---$ & " + "$---$ \\\\" %( 100*am['success rate'][0], am['mean travel distance'][0])) + return + + print("%2.1f \\scriptstyle\\pm %2.1f & %2.1f \\scriptstyle\\pm %2.1f & " + "%1.2f \\scriptstyle\\pm %1.2f & %2.1f \\scriptstyle\\pm %1.1f & " + "%0.3f \\scriptstyle\\pm %0.3f \\\\" %( 100*am['success rate'][0], 100*am['success rate'][1], + am['mean travel distance'][0] , am['mean travel distance'][1] , + am['rwse_10s'][0] , am['rwse_10s'][1] , + am['average absolute average velocity'][0] , am['average absolute average velocity'][1] , + am['acceleration distribution divergence'][0] , am['acceleration distribution divergence'][1] )) + +if __name__=='__main__': + import fire + fire.Fire(main) \ No newline at end of file diff --git a/evaluate_models.sh b/evaluate_models.sh index a1ae239..85c0add 100755 --- a/evaluate_models.sh +++ b/evaluate_models.sh @@ -1,42 +1,19 @@ -# eval_main inputs -# locations: List[Tuple[int,int]]= [(0,0)], -# method: str='expert', -# policy_file: str='', -# policy_kwargs: dict={}, -# env: str='NRasterizedRouteIncrementingAgent', -# env_kwargs: dict={}, -# seed: int=0 +# can add --skip_running if you've run the runs before on the saved policies -# expert -python -m src.eval_main +python -m eval_experiments +python -m eval_experiments --locations='[(0,4)]' +python -m eval_experiments --method idm +python -m eval_experiments --method idm --locations='[(0,4)]' +python -m eval_experiments --method bc --folder='test_policies/bc/expA' +python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]' +python -m eval_experiments --method gail --folder='test_policies/gail/expA' +python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]' +python -m eval_experiments --method hail --folder='test_policies/hail/expA' --env_kwargs='{safe_actions_collision_method=None,abort_unsafe_collision_method=None}' +python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]' --env_kwargs='{safe_actions_collision_method=None,abort_unsafe_collision_method=None}' +python -m eval_experiments --method shail --folder='test_policies/shail/expA' +python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]' -# idm -python -m src.eval_main --method=idm - -# behavior cloning -python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=0 -python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=1 -python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=2 -python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=3 -python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=4 -python -m src.evaluation.utils load_and_average out/bc - -# GAIL -python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=0 -python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=1 -python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=2 -python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=3 -python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=4 -python -m src.evaluation.utils load_and_average out/gail - -# options GAIL -python -m src.eval_main --method=ogail --policy_file='checkpoints/gail-options-setobs2-Feb15_18-49-05.pt' --env='NormalizedOptionsEvalEnv' --env_kwargs='{stop_on_collision:True}' - -# options GAIL-PPO -python -m src.eval_main --method=ogail-ppo --policy_file='checkpoints/gail-ppo-options-setobs2-Feb15_22-05-38.pt' --env='NormalizedOptionsEvalEnv' --env_kwargs='{stop_on_collision:True}' - -# SHAIL -python -m src.eval_main --method=sgail --policy_file='checkpoints/sgail-options-setobs2.pt' --env='NormalizedSafeOptionsEvalEnv' --env_kwargs='{stop_on_collision:True,max_episode_steps:1000}' - -# SHAIL-PPO -python -m src.eval_main --method=sgail-ppo --policy_file='checkpoints/sgail-ppo-options-setobs2-17-02-2022.pt' --env='NormalizedSafeOptionsEvalEnv' --env_kwargs='{stop_on_collision:True,max_episode_steps:1000}' +python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expA' +python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expB' --locations='[(0,4)]' +python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expA' +python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expB' --locations='[(0,4)]' \ No newline at end of file diff --git a/expert_data/DR_USA_Roundabout_FT/track0000/joint_expert_actions.pt b/expert_data/DR_USA_Roundabout_FT/track0000/joint_expert_actions.pt new file mode 100644 index 0000000..ec15dd9 Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0000/joint_expert_actions.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0000/joint_expert_states.pt b/expert_data/DR_USA_Roundabout_FT/track0000/joint_expert_states.pt new file mode 100644 index 0000000..147a964 Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0000/joint_expert_states.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0001/joint_expert_actions.pt b/expert_data/DR_USA_Roundabout_FT/track0001/joint_expert_actions.pt new file mode 100644 index 0000000..bc2357f Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0001/joint_expert_actions.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0001/joint_expert_states.pt b/expert_data/DR_USA_Roundabout_FT/track0001/joint_expert_states.pt new file mode 100644 index 0000000..889282b Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0001/joint_expert_states.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0002/joint_expert_actions.pt b/expert_data/DR_USA_Roundabout_FT/track0002/joint_expert_actions.pt new file mode 100644 index 0000000..07f67ec Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0002/joint_expert_actions.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0002/joint_expert_states.pt b/expert_data/DR_USA_Roundabout_FT/track0002/joint_expert_states.pt new file mode 100644 index 0000000..11619c4 Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0002/joint_expert_states.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0003/joint_expert_actions.pt b/expert_data/DR_USA_Roundabout_FT/track0003/joint_expert_actions.pt new file mode 100644 index 0000000..b5a3866 Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0003/joint_expert_actions.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0003/joint_expert_states.pt b/expert_data/DR_USA_Roundabout_FT/track0003/joint_expert_states.pt new file mode 100644 index 0000000..7359003 Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0003/joint_expert_states.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0004/joint_expert_actions.pt b/expert_data/DR_USA_Roundabout_FT/track0004/joint_expert_actions.pt new file mode 100644 index 0000000..438e1a3 Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0004/joint_expert_actions.pt differ diff --git a/expert_data/DR_USA_Roundabout_FT/track0004/joint_expert_states.pt b/expert_data/DR_USA_Roundabout_FT/track0004/joint_expert_states.pt new file mode 100644 index 0000000..f71964f Binary files /dev/null and b/expert_data/DR_USA_Roundabout_FT/track0004/joint_expert_states.pt differ diff --git a/gail-experiment.py b/gail-experiment.py new file mode 100644 index 0000000..68c9ae9 --- /dev/null +++ b/gail-experiment.py @@ -0,0 +1,238 @@ +# %% +import os + +import gym +from src.core.gail import gail_ppo, Buffer +from src.core.value import SetValue +from src.core.policy import SetPolicy +from src.core.discriminator import DeepsetDiscriminator +import torch + +from intersim.envs import IntersimpleLidarFlatRandom +from intersim.envs.intersimple import speed_reward +import functools +from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Setobs +import numpy as np +from torch.utils.tensorboard import SummaryWriter +from ray import tune +from datetime import datetime +import json + +DIR = os.path.dirname(os.path.abspath(__file__)) +activations = [torch.nn.Tanh, torch.nn.LeakyReLU] + +obs_min = np.array([ + [-1000, -1000, 0, -np.pi, -1e-1, 0.], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], +]).reshape(-1) + +obs_max = np.array([ + [1000, 1000, 20, np.pi, 1e-1, 0.], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], +]).reshape(-1) + +def training_function(config): + np.random.seed(config['seed']) + torch.manual_seed(config['seed']) + + if config['experiment'] == 'A': + envs = [Setobs(TransformObservation(CollisionPenaltyWrapper( + IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], + ), collision_distance=6, collision_penalty=100), + lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) + )) for _ in range(60)] + + elif config['experiment'] == 'B': + envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper( + IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], + track=track, + ), collision_distance=6, collision_penalty=100), + lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) + )) for _ in range(15)] for track in range(4)],[]) + + else: + raise NotImplementedError + + env_fn = lambda i: envs[i] + + policy = SetPolicy(env_fn(0).action_space.shape[0], + n_hidden_layers=config['policy']['n_hidden_layers'], + hidden_layer_size=config['policy']['hidden_layer_size'], + activation=activations[config['policy']['activation']] ) # config net architecture + pi_opt = torch.optim.Adam(policy.parameters(), lr=config['policy']['learning_rate']) + pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=config['policy']['learning_rate_decay']) + + value = SetValue() # config net architecture + v_opt = torch.optim.Adam(value.parameters(), lr=config['value']['learning_rate'], weight_decay=config['value']['weight_decay']) + + discriminator = DeepsetDiscriminator( + n_hidden_layers_element=config['discriminator']['n_hidden_layers_element'], + n_hidden_layers_global=config['discriminator']['n_hidden_layers_global'], + hidden_layer_size=config['discriminator']['hidden_layer_size'], + activation=activations[config['discriminator']['activation']], + ) + disc_opt = torch.optim.Adam(discriminator.parameters(), lr=config['discriminator']['learning_rate'], weight_decay=config['discriminator']['weight_decay']) + + if config['experiment'] == 'A': + expert_data = torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')) + elif config['experiment'] == 'B': + expert_data = [ + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track1.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track2.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track3.pt')), + ] + d0 = [d[0] for d in expert_data] + d1 = [d[1] for d in expert_data] + d2 = [d[2] for d in expert_data] + d3 = [d[3] for d in expert_data] + expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3)) + + expert_data = Buffer(*expert_data) + + def callback(info): + tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'], + disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'], + mean_episode_length=info['gen/mean_episode_length'], + gen_collision_rate=info['gen/collision_rate']) + + # save model checkpoints + ep = info['epoch'] + 1 + if (ep % 25 == 0): + torch.save(info['policy'].state_dict(), f'policy_epoch{ep}.pt') + + value, policy = gail_ppo( + env_fn=env_fn, + expert_data=expert_data, + discriminator=discriminator, + disc_opt=disc_opt, + disc_iters=config['discriminator']['iterations_per_epoch'], + policy=policy, + value=value, + v_opt=v_opt, + v_iters=config['value']['iterations_per_epoch'], + epochs=config['train_epochs'], + rollout_episodes=60, + rollout_steps=200, + gamma=0.99, + gae_lambda=0.9, + clip_ratio=config['policy']['clip_ratio'], + pi_opt=pi_opt, + pi_iters=config['policy']['iterations_per_epoch'], + logger=SummaryWriter(comment='gail-ppo-options-setobs2'), + callback=callback, + lr_schedulers=[pi_lr_scheduler], + ) + + # save model + torch.save(policy.state_dict(), 'policy_final.pt') + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--train', choices=['A', 'B']) + parser.add_argument('--epochs', type=int, default=200) + parser.add_argument('--test', type=str, help='path to config file to run final training on') + parser.add_argument('--test_seeds', type=int, default=5) + parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over') + args = parser.parse_args() + + assert (args.train is None) ^ (args.test is None), 'Must either train on an experiment or test with a config file' + + # if no test config specified, train + if args.test is None: + print('Running Tuning for Experiment %s'%(args.train)) + analysis = tune.run( + training_function, + config={ + 'experiment': args.train, + 'trainenv': { + 'stop_on_collision': False, + }, + 'policy': { + 'learning_rate': 3e-4, + 'learning_rate_decay': 1.0, + 'clip_ratio': 0.2, + 'iterations_per_epoch': 100, + 'hidden_layer_size': tune.grid_search([20, 40]), + 'n_hidden_layers': tune.grid_search([2, 3]), + 'activation':0, + }, + 'value': { + 'learning_rate': 1e-4, + 'weight_decay': 1e-3, + 'iterations_per_epoch': 1000, + }, + 'discriminator': { + 'learning_rate': 1e-3, + 'weight_decay': 1e-4, + 'iterations_per_epoch': 100, + 'n_hidden_layers_element': tune.grid_search([3,4]), + 'n_hidden_layers_global': tune.grid_search([1,2]), + 'hidden_layer_size': 10, + 'activation': 0, + }, + 'train_epochs': args.epochs, + 'seed': 0, + } + ) + best_config = analysis.get_best_config(metric='gen_collision_rate', mode='min') + print('Best config: ', best_config) + + # safe best_config + if not os.path.isdir(os.path.join(DIR, 'best_configs')): + os.mkdir(os.path.join(DIR, 'best_configs')) + + # save gail + with open(os.path.join(DIR, 'best_configs',f'gail_exp{args.train}.json'), 'w', encoding='utf-8') as f: + json.dump(best_config, f, ensure_ascii=False, indent=4) + + # if config file specified, rerun it with appropriate number of seeds + else: + with open(args.test, 'rb') as f: + config = json.load(f) + + print(f'Retraining {args.test} with {args.test_seeds} seeds on experiment {config["experiment"]}') + + # rerun with appropriate number of seeds + rpt = {'cpu': int(args.test_cpus/args.test_seeds)} if (args.test_cpus is not None) else None + config['seed'] = tune.grid_search(list(range(1,args.test_seeds+1))) + analysis = tune.run(training_function, config=config, resources_per_trial=rpt) + + # move final policies to appropriate directory + split_ = os.path.basename(args.test).split('_') + model = split_[0] + exper = split_[-1].split('.')[0] + savepath = os.path.join('test_policies',model,exper) + + if not os.path.isdir(savepath): + os.makedirs(savepath) + + import shutil + for i in range(args.test_seeds): + s = analysis._checkpoints[i]['config']['seed'] + check_dir = analysis._checkpoints[i]['logdir'] + shutil.copyfile(os.path.join(check_dir,'policy_final.pt'), + os.path.join(savepath, f'policy_seed{s}.pt')) \ No newline at end of file diff --git a/intersimple-expert-data-setobs2-loc0-track0.pt b/intersimple-expert-data-setobs2-loc0-track0.pt new file mode 100644 index 0000000..6192f97 Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track0.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track1.pt b/intersimple-expert-data-setobs2-loc0-track1.pt new file mode 100644 index 0000000..e8a881a Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track1.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track2.pt b/intersimple-expert-data-setobs2-loc0-track2.pt new file mode 100644 index 0000000..f099ecf Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track2.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track3.pt b/intersimple-expert-data-setobs2-loc0-track3.pt new file mode 100644 index 0000000..3c51278 Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track3.pt differ diff --git a/intersimple-expert-data-setobs2-loc0-track4.pt b/intersimple-expert-data-setobs2-loc0-track4.pt new file mode 100644 index 0000000..8834b90 Binary files /dev/null and b/intersimple-expert-data-setobs2-loc0-track4.pt differ diff --git a/intersimple-expert-data-setobs2.pt b/intersimple-expert-data-setobs2.pt new file mode 100644 index 0000000..62323d4 Binary files /dev/null and b/intersimple-expert-data-setobs2.pt differ diff --git a/out/bc/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl b/out/bc/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl new file mode 100644 index 0000000..5625c55 Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl b/out/bc/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl new file mode 100644 index 0000000..5d1b323 Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl b/out/bc/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl new file mode 100644 index 0000000..d87b59c Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl b/out/bc/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl new file mode 100644 index 0000000..a55c966 Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl b/out/bc/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl new file mode 100644 index 0000000..5b6b715 Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl b/out/bc/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl new file mode 100644 index 0000000..3bfbbcd Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl b/out/bc/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl new file mode 100644 index 0000000..c47b17e Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl b/out/bc/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl new file mode 100644 index 0000000..8ebbdf8 Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl b/out/bc/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl new file mode 100644 index 0000000..f46c7f2 Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl differ diff --git a/out/bc/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl b/out/bc/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl new file mode 100644 index 0000000..8d95541 Binary files /dev/null and b/out/bc/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl b/out/bc/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl new file mode 100644 index 0000000..10b4c44 Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl b/out/bc/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl new file mode 100644 index 0000000..a5bf427 Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl b/out/bc/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl new file mode 100644 index 0000000..29d6b07 Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl b/out/bc/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl new file mode 100644 index 0000000..c3fd85a Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl b/out/bc/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl new file mode 100644 index 0000000..d875914 Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl b/out/bc/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl new file mode 100644 index 0000000..f0f393e Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl b/out/bc/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl new file mode 100644 index 0000000..95b3a84 Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl b/out/bc/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl new file mode 100644 index 0000000..9415130 Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl b/out/bc/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl new file mode 100644 index 0000000..1ae2a7b Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl differ diff --git a/out/bc/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl b/out/bc/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl new file mode 100644 index 0000000..01a350b Binary files /dev/null and b/out/bc/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl differ diff --git a/out/expert/loc_r0t0/expert_tseed0_summary.pkl b/out/expert/loc_r0t0/expert_tseed0_summary.pkl new file mode 100644 index 0000000..f4f1fd8 Binary files /dev/null and b/out/expert/loc_r0t0/expert_tseed0_summary.pkl differ diff --git a/out/expert/loc_r0t4/expert_tseed0_summary.pkl b/out/expert/loc_r0t4/expert_tseed0_summary.pkl new file mode 100644 index 0000000..294b756 Binary files /dev/null and b/out/expert/loc_r0t4/expert_tseed0_summary.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl b/out/gail/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl new file mode 100644 index 0000000..97a3947 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl b/out/gail/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl new file mode 100644 index 0000000..5ed53a8 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl b/out/gail/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl new file mode 100644 index 0000000..54ef594 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl b/out/gail/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl new file mode 100644 index 0000000..9587401 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl b/out/gail/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl new file mode 100644 index 0000000..f0bbd04 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl b/out/gail/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl new file mode 100644 index 0000000..727ab01 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl b/out/gail/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl new file mode 100644 index 0000000..71956a8 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl b/out/gail/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl new file mode 100644 index 0000000..9807193 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl b/out/gail/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl new file mode 100644 index 0000000..023a738 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl differ diff --git a/out/gail/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl b/out/gail/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl new file mode 100644 index 0000000..da38427 Binary files /dev/null and b/out/gail/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl b/out/gail/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl new file mode 100644 index 0000000..01c1e67 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl b/out/gail/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl new file mode 100644 index 0000000..908dbd6 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl b/out/gail/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl new file mode 100644 index 0000000..6871842 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl b/out/gail/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl new file mode 100644 index 0000000..164b855 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl b/out/gail/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl new file mode 100644 index 0000000..33ce257 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl b/out/gail/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl new file mode 100644 index 0000000..c97c831 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl b/out/gail/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl new file mode 100644 index 0000000..af5badf Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl b/out/gail/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl new file mode 100644 index 0000000..b309989 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl b/out/gail/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl new file mode 100644 index 0000000..fce6b7f Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl differ diff --git a/out/gail/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl b/out/gail/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl new file mode 100644 index 0000000..17efe68 Binary files /dev/null and b/out/gail/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl differ diff --git a/out/idm/loc_r0t0/idm_tseed0_comparison.pkl b/out/idm/loc_r0t0/idm_tseed0_comparison.pkl new file mode 100644 index 0000000..972d918 Binary files /dev/null and b/out/idm/loc_r0t0/idm_tseed0_comparison.pkl differ diff --git a/out/idm/loc_r0t0/idm_tseed0_summary.pkl b/out/idm/loc_r0t0/idm_tseed0_summary.pkl new file mode 100644 index 0000000..d8496af Binary files /dev/null and b/out/idm/loc_r0t0/idm_tseed0_summary.pkl differ diff --git a/out/idm/loc_r0t4/idm_tseed0_comparison.pkl b/out/idm/loc_r0t4/idm_tseed0_comparison.pkl new file mode 100644 index 0000000..8548abc Binary files /dev/null and b/out/idm/loc_r0t4/idm_tseed0_comparison.pkl differ diff --git a/out/idm/loc_r0t4/idm_tseed0_summary.pkl b/out/idm/loc_r0t4/idm_tseed0_summary.pkl new file mode 100644 index 0000000..5a0511f Binary files /dev/null and b/out/idm/loc_r0t4/idm_tseed0_summary.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl b/out/shail/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl new file mode 100644 index 0000000..bcda5b9 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed1_tseed0_comparison.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl b/out/shail/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl new file mode 100644 index 0000000..855e8d2 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed1_tseed0_summary.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl b/out/shail/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl new file mode 100644 index 0000000..4a1f097 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed2_tseed0_comparison.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl b/out/shail/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl new file mode 100644 index 0000000..7e6426d Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed2_tseed0_summary.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl b/out/shail/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl new file mode 100644 index 0000000..1875021 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed3_tseed0_comparison.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl b/out/shail/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl new file mode 100644 index 0000000..32e6928 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed3_tseed0_summary.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl b/out/shail/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl new file mode 100644 index 0000000..c169389 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed4_tseed0_comparison.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl b/out/shail/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl new file mode 100644 index 0000000..ef10af4 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed4_tseed0_summary.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl b/out/shail/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl new file mode 100644 index 0000000..73cb398 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed5_tseed0_comparison.pkl differ diff --git a/out/shail/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl b/out/shail/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl new file mode 100644 index 0000000..623fcc8 Binary files /dev/null and b/out/shail/expA/loc_r0t0/policy_seed5_tseed0_summary.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl b/out/shail/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl new file mode 100644 index 0000000..91d0614 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed1_tseed0_comparison.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl b/out/shail/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl new file mode 100644 index 0000000..632d7f7 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed1_tseed0_summary.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl b/out/shail/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl new file mode 100644 index 0000000..4a7f99d Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed2_tseed0_comparison.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl b/out/shail/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl new file mode 100644 index 0000000..494aec0 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed2_tseed0_summary.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl b/out/shail/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl new file mode 100644 index 0000000..ac29c3b Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed3_tseed0_comparison.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl b/out/shail/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl new file mode 100644 index 0000000..a7f9226 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed3_tseed0_summary.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl b/out/shail/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl new file mode 100644 index 0000000..0fc17b6 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed4_tseed0_comparison.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl b/out/shail/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl new file mode 100644 index 0000000..5ba57e1 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed4_tseed0_summary.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl b/out/shail/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl new file mode 100644 index 0000000..3763165 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed5_tseed0_comparison.pkl differ diff --git a/out/shail/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl b/out/shail/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl new file mode 100644 index 0000000..addec93 Binary files /dev/null and b/out/shail/expB/loc_r0t4/policy_seed5_tseed0_summary.pkl differ diff --git a/requirements.txt b/requirements.txt index 62870bb..0dc31a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,6 @@ tqdm ray[tune] hyperopt psutil -fire \ No newline at end of file +fire +stable_baselines3 +tensorboard \ No newline at end of file diff --git a/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py b/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py index 28139f2..0b24e6e 100644 --- a/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py +++ b/scratch/etienne/trpo/experiments/intersimple-expert-rollout-setobs2.py @@ -1,10 +1,13 @@ +import sys +sys.path.append('../../../../') + import torch import functools -from core.sampling import rollout_sb3 -from intersim.envs import IntersimpleLidarFlatRandom +from src.core.sampling import rollout_sb3 +from intersim.envs import IntersimpleLidarFlatIncrementingAgent from intersim.envs.intersimple import speed_reward from intersim.expert import NormalizedIntersimpleExpert -from util.wrappers import CollisionPenaltyWrapper, Setobs +from src.util.wrappers import CollisionPenaltyWrapper, Setobs import numpy as np from gym.wrappers import TransformObservation @@ -26,7 +29,9 @@ obs_max = np.array([ [50, np.pi, 20, 20, np.pi, 1e-1], ]).reshape(-1) -env = IntersimpleLidarFlatRandom( +env = IntersimpleLidarFlatIncrementingAgent( + loc=0, + track=4, n_rays=5, reward=functools.partial( speed_reward, @@ -42,7 +47,8 @@ env = Setobs(TransformObservation( collision_distance=6, collision_penalty=100 ), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) )) -expert_data = rollout_sb3(env, policy, n_episodes=2048, max_steps_per_episode=200) +print(env.nv, 'vehicles') +expert_data = rollout_sb3(env, policy, n_episodes=150, max_steps_per_episode=200) states, actions, rewards, dones = expert_data print(f'Expert mean episode length {(~dones).sum() / states.shape[0]}') @@ -50,4 +56,4 @@ print(f'Expert mean reward per episode {rewards[~dones].sum() / states.shape[0]} print(f'Observation mean', states[~dones].mean(0)) print(f'Observation std', states[~dones].std(0)) -torch.save(expert_data, 'intersimple-expert-data-setobs2.pt') +torch.save(expert_data, 'intersimple-expert-data-setobs2-loc0-track4.pt') diff --git a/scratch/etienne/trpo/experiments/sgail-ppo-options-setobs2.py b/scratch/etienne/trpo/experiments/sgail-ppo-options-setobs2.py index a7b01e7..efbe7dc 100644 --- a/scratch/etienne/trpo/experiments/sgail-ppo-options-setobs2.py +++ b/scratch/etienne/trpo/experiments/sgail-ppo-options-setobs2.py @@ -15,79 +15,87 @@ from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Set import numpy as np from src.safe_options.options import SafeOptionsEnv from torch.utils.tensorboard import SummaryWriter +from ray import tune -obs_min = np.array([ - [-1000, -1000, 0, -np.pi, -1e-1, 0.], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], - [0, -np.pi, -20, -20, -np.pi, -1e-1], -]).reshape(-1) +def training_function(config): + obs_min = np.array([ + [-1000, -1000, 0, -np.pi, -1e-1, 0.], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + ]).reshape(-1) -obs_max = np.array([ - [1000, 1000, 20, np.pi, 1e-1, 0.], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], - [50, np.pi, 20, 20, np.pi, 1e-1], -]).reshape(-1) + obs_max = np.array([ + [1000, 1000, 20, np.pi, 1e-1, 0.], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + ]).reshape(-1) -envs = [SafeOptionsEnv(Setobs( - TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom( - n_rays=5, - reward=functools.partial( - speed_reward, - collision_penalty=0 - ), - stop_on_collision=True, - ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) -), options=[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], safe_actions_collision_method='circle', abort_unsafe_collision_method='circle') for _ in range(60)] + envs = [SafeOptionsEnv(Setobs( + TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + stop_on_collision=True, + ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) + ), options=[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], safe_actions_collision_method='circle', abort_unsafe_collision_method='circle') for _ in range(60)] -env_fn = lambda i: envs[i] + env_fn = lambda i: envs[i] -policy = SetMaskedDiscretePolicy(env_fn(0).action_space.n) -pi_opt = torch.optim.Adam(policy.parameters(), lr=3e-4) + policy = SetMaskedDiscretePolicy(env_fn(0).action_space.n) # config net architecture + pi_opt = torch.optim.Adam(policy.parameters(), lr=3e-5) # config learning rate + pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=0.98) # config lr decay -value = SetValue() -v_opt = torch.optim.Adam(value.parameters(), lr=1e-3) + value = SetValue() # config net architecture + v_opt = torch.optim.Adam(value.parameters(), lr=1e-3) # config lr -discriminator = DeepsetDiscriminator() -disc_opt = torch.optim.Adam(discriminator.parameters(), lr=1e-3, weight_decay=1e-4) + discriminator = DeepsetDiscriminator() # config net architecture + disc_opt = torch.optim.Adam(discriminator.parameters(), lr=1e-3, weight_decay=1e-3) # config lr, weight decay -expert_data = torch.load('intersimple-expert-data-setobs2.pt') -expert_data = Buffer(*expert_data) + expert_data = torch.load('intersimple-expert-data-setobs2.pt') + expert_data = Buffer(*expert_data) -# %% -def callback(epoch, value, policy): - if not epoch % 10: - torch.save(policy.state_dict(), f'sgail-ppo-options-setobs2-{epoch}.pt') - torch.save(value.state_dict(), f'sgail-ppo-options-setobs2-value-{epoch}.pt') + def callback(info): + tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode']) -value, policy = gail_ppo( - env_fn=env_fn, - expert_data=expert_data, - discriminator=discriminator, - disc_opt=disc_opt, - disc_iters=100, - policy=policy, - value=value, - v_opt=v_opt, - v_iters=1000, - epochs=200, - rollout_episodes=60, - rollout_steps=60, - gamma=0.99, - gae_lambda=0.9, - clip_ratio=0.2, - pi_opt=pi_opt, - pi_iters=100, - logger=SummaryWriter(comment='sgail-ppo-options-setobs2'), - callback=callback, + value, policy = gail_ppo( + env_fn=env_fn, + expert_data=expert_data, + discriminator=discriminator, + disc_opt=disc_opt, + disc_iters=100, # config + policy=policy, + value=value, + v_opt=v_opt, + v_iters=1000, # config + epochs=200, + rollout_episodes=60, + rollout_steps=60, + gamma=0.99, + gae_lambda=0.9, + clip_ratio=0.2, # config + pi_opt=pi_opt, + pi_iters=100, # config + logger=SummaryWriter(comment='sgail-ppo-options-setobs2'), + callback=callback, + lr_schedulers=[pi_lr_scheduler], + ) + +analysis = tune.run( + training_function, + config={ + 'dummy': tune.grid_search([0.001, 0.01, 0.1]), + } ) -torch.save(policy.state_dict(), 'sgail-ppo-options-setobs2.pt') +print('Best config: ', analysis.get_best_config(metric='gen_mean_reward_per_episode', mode='min')) # %% policy = SetMaskedDiscretePolicy(env_fn(0).action_space.n) diff --git a/scratch/johannes/horner_scheme.py b/scratch/johannes/horner_scheme.py new file mode 100644 index 0000000..c454dfe --- /dev/null +++ b/scratch/johannes/horner_scheme.py @@ -0,0 +1,90 @@ +# %% + +import numpy as np +import torch +from timeit import default_timer as timer + +# %% + +def powerseries(x, deg): + return torch.stack([x**i for i in range(deg+1)],dim=-1) + +def improved_powerseries(x, deg): + r = torch.ones(*x.shape, deg+1, dtype=torch.float64) + for i in range(1,deg+1): + r[:, :, i] = r[:, :, i-1] * x + return r + +def horner_scheme(x, poly): + deg = poly.shape[-1] + nsteps = x.shape[-1] + r = poly[:, -1:].repeat(1, nsteps) + for i in range(2, deg+1): + r *= x + r += poly[:, -i:1-i] + return r + +# %% + +nv = 151 +delta = 10 +n = 20 + +state_s = torch.rand((nv, 1)) +nan_idx = np.random.choice([True, False], 151) +state_s[nan_idx] = np.nan + +# %% + +n_coef = 21 +xpoly = torch.rand((nv, n_coef),dtype=torch.float64) +ypoly = torch.rand((nv, n_coef),dtype=torch.float64) + +ds = delta * torch.arange(1,n+1).repeat(nv,1) + +s = ds + state_s +s = s.type(torch.float64) + +smax = s[:, 0] +smax = smax.unsqueeze(-1) + + +start = timer() +for _ in range(100): + deg = xpoly.shape[-1] - 1 + expand_sims = powerseries(s, deg) # (nv, n, deg+1) + # print(expand_sims.shape) + y = (ypoly.unsqueeze(1) * expand_sims).sum(dim=-1) + x = (xpoly.unsqueeze(1) * expand_sims).sum(dim=-1) +end = timer() +print("Powerseries: {}".format((end-start)*1)) + +start = timer() +for _ in range(100): + deg = xpoly.shape[-1] - 1 + expand_sims = improved_powerseries(s, deg) # (nv, n, deg+1) + # print(expand_sims.shape) + yp = (ypoly.unsqueeze(1) * expand_sims).sum(dim=-1) + xp = (xpoly.unsqueeze(1) * expand_sims).sum(dim=-1) +end = timer() +print("Improved Powerseries: {}".format((end-start)*1)) + +start = timer() +for _ in range(100): + x_horner = horner_scheme(s, xpoly) + y_horner = horner_scheme(s, ypoly) +end = timer() +print("Horner: {}".format((end-start)*1)) + +start = timer() +for _ in range(100): + x_max = horner_scheme(smax, xpoly) + y_max = horner_scheme(smax, ypoly) +end = timer() +# print("Horner smax: {}".format((end-start)*1)) + +assert np.all(np.isclose(xp,x)[~nan_idx]) +assert np.all(np.isclose(yp,y)[~nan_idx]) +assert np.all(np.isclose(x_horner,x)[~nan_idx]) +assert np.all(np.isclose(y_horner,y)[~nan_idx]) +# %% diff --git a/shail-experiment.py b/shail-experiment.py new file mode 100644 index 0000000..4195759 --- /dev/null +++ b/shail-experiment.py @@ -0,0 +1,254 @@ +# %% +import os + +import gym +from src.safe_options.options import gail_ppo, Buffer +from src.core.value import SetValue +from src.safe_options.policy import SetMaskedDiscretePolicy +from src.core.discriminator import DeepsetDiscriminator +import torch + +from intersim.envs import IntersimpleLidarFlatRandom +from intersim.envs.intersimple import speed_reward +import functools +from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Setobs +import numpy as np +from src.safe_options.options import SafeOptionsEnv +from torch.utils.tensorboard import SummaryWriter +from ray import tune +from datetime import datetime +import json + +DIR = os.path.dirname(os.path.abspath(__file__)) +option_list = [[(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5]], + [(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 10]], + [(vel, time) for vel in [0, 3, 10] for time in [5, 10, 20]] +] +activations = [torch.nn.Tanh, torch.nn.LeakyReLU] + +obs_min = np.array([ + [-1000, -1000, 0, -np.pi, -1e-1, 0.], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], + [0, -np.pi, -20, -20, -np.pi, -1e-1], +]).reshape(-1) + +obs_max = np.array([ + [1000, 1000, 20, np.pi, 1e-1, 0.], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], + [50, np.pi, 20, 20, np.pi, 1e-1], +]).reshape(-1) + +def training_function(config): + np.random.seed(config['seed']) + torch.manual_seed(config['seed']) + + if config['experiment'] == 'A': + envs = [SafeOptionsEnv(Setobs( + TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], + ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) + ), options=option_list[config['policy']['option']], + safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], + abort_unsafe_collision_method=config['trainenv']['abort_unsafe_collision_method'], + ) for _ in range(60)] + + elif config['experiment'] == 'B': + envs = sum([[SafeOptionsEnv(Setobs( + TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], track=track, + ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) + ), options=option_list[config['policy']['option']], + safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], + abort_unsafe_collision_method=config['trainenv']['abort_unsafe_collision_method'], + ) for _ in range(15)] for track in range(4)],[]) + + else: + raise NotImplementedError + + env_fn = lambda i: envs[i] + + policy = SetMaskedDiscretePolicy(env_fn(0).action_space.n, + n_hidden_layers=config['policy']['n_hidden_layers'], + hidden_layer_size=config['policy']['hidden_layer_size'], + activation=activations[config['policy']['activation']] ) # config net architecture + pi_opt = torch.optim.Adam(policy.parameters(), lr=config['policy']['learning_rate']) + pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=config['policy']['learning_rate_decay']) + + value = SetValue() # config net architecture + v_opt = torch.optim.Adam(value.parameters(), lr=config['value']['learning_rate']) + + discriminator = DeepsetDiscriminator( + n_hidden_layers_element=config['discriminator']['n_hidden_layers_element'], + n_hidden_layers_global=config['discriminator']['n_hidden_layers_global'], + hidden_layer_size=config['discriminator']['hidden_layer_size'], + activation=activations[config['discriminator']['activation']], + ) + disc_opt = torch.optim.Adam(discriminator.parameters(), lr=config['discriminator']['learning_rate'], weight_decay=config['discriminator']['weight_decay']) + + if config['experiment'] == 'A': + expert_data = torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')) + elif config['experiment'] == 'B': + expert_data = [ + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track1.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track2.pt')), + torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track3.pt')), + ] + d0 = [d[0] for d in expert_data] + d1 = [d[1] for d in expert_data] + d2 = [d[2] for d in expert_data] + d3 = [d[3] for d in expert_data] + expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3)) + + expert_data = Buffer(*expert_data) + + def callback(info): + tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'], + disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'], + mean_episode_length=info['gen/mean_episode_length'], + gen_collision_rate=info['gen/collision_rate']) + + # save model checkpoints + ep = info['epoch'] + 1 + if (ep % 25 == 0): + torch.save(info['policy'].state_dict(), f'policy_epoch{ep}.pt') + + value, policy = gail_ppo( + env_fn=env_fn, + expert_data=expert_data, + discriminator=discriminator, + disc_opt=disc_opt, + disc_iters=config['discriminator']['iterations_per_epoch'], + policy=policy, + value=value, + v_opt=v_opt, + v_iters=config['value']['iterations_per_epoch'], + epochs=config['train_epochs'], + rollout_episodes=60, + rollout_steps=60, + gamma=0.99, + gae_lambda=0.9, + clip_ratio=config['policy']['clip_ratio'], + pi_opt=pi_opt, + pi_iters=config['policy']['iterations_per_epoch'], + logger=SummaryWriter(comment='sgail-ppo-options-setobs2'), + callback=callback, + lr_schedulers=[pi_lr_scheduler], + ) + + # save model + torch.save(policy.state_dict(), 'policy_final.pt') + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--train', choices=['A', 'B']) + parser.add_argument('--epochs', type=int, default=200) + parser.add_argument('--test', type=str, help='path to config file to run final training on') + parser.add_argument('--test_seeds', type=int, default=5) + parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over') + args = parser.parse_args() + + assert (args.train is None) ^ (args.test is None), 'Must either train on an experiment or test with a config file' + + # if no test config specified, train + if args.test is None: + print('Running Tuning for Experiment %s'%(args.train)) + analysis = tune.run( + training_function, + config={ + 'experiment': args.train, + 'trainenv': { + 'stop_on_collision': False, + 'safe_actions_collision_method': 'circle', + 'abort_unsafe_collision_method': 'circle', + }, + 'policy': { + 'learning_rate': 3e-4, + 'learning_rate_decay': 1.0, + 'clip_ratio': 0.2, + 'iterations_per_epoch': 100, + 'hidden_layer_size': tune.grid_search([20, 40]), + 'n_hidden_layers': tune.grid_search([2, 3]), + 'activation':0, + 'option': tune.grid_search(list(range(len(option_list)))) + }, + 'value': { + 'learning_rate': 1e-3, + 'iterations_per_epoch': 1000, + }, + 'discriminator': { + 'learning_rate': 1e-3, + 'weight_decay': 1e-4, + 'iterations_per_epoch': 100, + 'n_hidden_layers_element': tune.grid_search([3,4]), + 'n_hidden_layers_global': tune.grid_search([1,2]), + 'hidden_layer_size': 10, + 'activation': 0, + }, + 'train_epochs': args.epochs, + 'seed': 0, + } + ) + best_config = analysis.get_best_config(metric='gen_collision_rate', mode='min') + print('Best config: ', best_config) + + # safe best_config + if not os.path.isdir(os.path.join(DIR, 'best_configs')): + os.mkdir(os.path.join(DIR, 'best_configs')) + + # save shail + with open(os.path.join(DIR, 'best_configs',f'shail_exp{args.train}.json'), 'w', encoding='utf-8') as f: + json.dump(best_config, f, ensure_ascii=False, indent=4) + + # save hail + best_config['trainenv']['safe_actions_collision_method']=None + best_config['trainenv']['abort_unsafe_collision_method']=None + with open(os.path.join(DIR, 'best_configs',f'hail_exp{args.train}.json'), 'w', encoding='utf-8') as f: + json.dump(best_config, f, ensure_ascii=False, indent=4) + + # if config file specified, rerun it with appropriate number of seeds + else: + with open(args.test, 'rb') as f: + config = json.load(f) + + print(f'Retraining {args.test} with {args.test_seeds} seeds on experiment {config["experiment"]}') + + # rerun with appropriate number of seeds + rpt = {'cpu': int(args.test_cpus/args.test_seeds)} if (args.test_cpus is not None) else None + config['seed'] = tune.grid_search(list(range(1,args.test_seeds+1))) + analysis = tune.run(training_function, config=config, resources_per_trial=rpt) + + # move final policies to appropriate directory + split_ = os.path.basename(args.test).split('_') + model = split_[0] + exper = split_[-1].split('.')[0] + savepath = os.path.join('test_policies',model,exper) + + if not os.path.isdir(savepath): + os.makedirs(savepath) + + import shutil + for i in range(args.test_seeds): + s = analysis._checkpoints[i]['config']['seed'] + check_dir = analysis._checkpoints[i]['logdir'] + shutil.copyfile(os.path.join(check_dir,'policy_final.pt'), + os.path.join(savepath, f'policy_seed{s}.pt')) \ No newline at end of file diff --git a/src/core/discriminator.py b/src/core/discriminator.py index 8074c32..f4aaa4e 100644 --- a/src/core/discriminator.py +++ b/src/core/discriminator.py @@ -18,23 +18,17 @@ class Discriminator(nn.Module): class DeepsetDiscriminator(nn.Module): - def __init__(self): + def __init__(self, n_hidden_layers_element=3, n_hidden_layers_global=2, hidden_layer_size=10, activation=nn.Tanh): super().__init__() - self.elem = nn.Sequential( - nn.LazyLinear(10), - nn.Tanh(), - nn.LazyLinear(10), - nn.Tanh(), - nn.LazyLinear(10), - ) - self.glob = nn.Sequential( - nn.LazyLinear(10), - nn.Tanh(), - nn.LazyLinear(10), - nn.Tanh(), - nn.LazyLinear(1), - ) - + + layers_elem = sum([[nn.LazyLinear(hidden_layer_size), + activation()] for _ in range(n_hidden_layers_element)], []) + self.elem = nn.Sequential(*layers_elem) + + layers_glob = sum([[nn.LazyLinear(hidden_layer_size), + activation()] for _ in range(n_hidden_layers_global)], []) + self.glob = nn.Sequential(*layers_glob, nn.LazyLinear(1)) + def forward(self, states, actions): actions = actions.unsqueeze(-2) actions = actions.expand(*actions.shape[:-2], states.shape[-2], actions.shape[-1]) diff --git a/src/core/gail.py b/src/core/gail.py index 2e34333..a9e5333 100644 --- a/src/core/gail.py +++ b/src/core/gail.py @@ -30,7 +30,7 @@ def roll_buffer(buffer, *args, **kwargs): def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value, v_opt, v_iters, epochs, rollout_episodes, rollout_steps, gamma, - gae_lambda, delta, backtrack_coeff, backtrack_iters, cg_iters=10, cg_damping=0.1, wasserstein=False, wasserstein_c=None, logger=TerminalLogger()): + gae_lambda, delta, backtrack_coeff, backtrack_iters, cg_iters=10, cg_damping=0.1, wasserstein=False, wasserstein_c=None, logger=TerminalLogger(), callback=None, lr_schedulers=[]): policy(torch.zeros(env_fn(0).observation_space.shape)) policy = ReparamPolicy(policy) @@ -38,11 +38,16 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value logger.add_scalar('expert/mean_episode_length', (~expert_data.dones).sum() / expert_data.states.shape[0]) logger.add_scalar('expert/mean_reward_per_episode', expert_data.rewards[~expert_data.dones].sum() / expert_data.states.shape[0]) - for epoch in tqdm(range(epochs)): - generator_data = Buffer(*rollout(env_fn, policy, rollout_episodes, rollout_steps)) + for epoch in range(epochs): + states, actions, rewards, dones, collisions = rollout(env_fn, policy, rollout_episodes, rollout_steps) + generator_data = Buffer(states, actions, rewards, dones) - logger.add_scalar('gen/mean_episode_length', (~generator_data.dones).sum() / generator_data.states.shape[0], epoch) - logger.add_scalar('gen/mean_reward_per_episode', generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.dones).sum() / generator_data.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length, epoch) + gen_mean_reward_per_episode = generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0] + logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) + gen_collision_rate = (1. * collisions.any(-1)).mean() + logger.add_scalar('gen/collision_rate', gen_collision_rate, epoch) discriminator, loss = train_discriminator(expert_data, generator_data, discriminator, disc_opt, disc_iters, wasserstein, wasserstein_c) if wasserstein: @@ -50,25 +55,45 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value else: generator_data.rewards = -F.logsigmoid(discriminator(generator_data.states, generator_data.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode, epoch) value, policy = trpo_step(value, policy, generator_data.states, generator_data.actions, generator_data.rewards, generator_data.dones, gamma, gae_lambda, delta, backtrack_coeff, backtrack_iters, v_opt, v_iters, cg_iters, cg_damping) expert_data = roll_buffer(expert_data, shifts=-3, dims=0) + + if callback is not None: + callback({ + 'epoch': epoch, + 'value': value, + 'policy': policy, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'gen/collision_rate': gen_collision_rate.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), + }) + + for lr_scheduler in lr_schedulers: + lr_scheduler.step() return value, policy def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value, v_opt, v_iters, epochs, rollout_episodes, rollout_steps, gamma, - gae_lambda, clip_ratio, pi_opt, pi_iters, target_kl=None, max_grad_norm=None, wasserstein=False, wasserstein_c=None, logger=TerminalLogger()): + gae_lambda, clip_ratio, pi_opt, pi_iters, target_kl=None, max_grad_norm=None, wasserstein=False, wasserstein_c=None, logger=TerminalLogger(), callback=None, lr_schedulers=[]): logger.add_scalar('expert/mean_episode_length', (~expert_data.dones).sum() / expert_data.states.shape[0]) logger.add_scalar('expert/mean_reward_per_episode', expert_data.rewards[~expert_data.dones].sum() / expert_data.states.shape[0]) for epoch in range(epochs): - generator_data = Buffer(*rollout(env_fn, policy, rollout_episodes, rollout_steps)) + states, actions, rewards, dones, collisions = rollout(env_fn, policy, rollout_episodes, rollout_steps) + generator_data = Buffer(states, actions, rewards, dones) - logger.add_scalar('gen/mean_episode_length', (~generator_data.dones).sum() / generator_data.states.shape[0], epoch) - logger.add_scalar('gen/mean_reward_per_episode', generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.dones).sum() / generator_data.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length, epoch) + gen_mean_reward_per_episode = generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0] + logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) + gen_collision_rate = (1. * collisions.any(-1)).mean() + logger.add_scalar('gen/collision_rate', gen_collision_rate, epoch) discriminator, loss = train_discriminator(expert_data, generator_data, discriminator, disc_opt, disc_iters, wasserstein, wasserstein_c) if wasserstein: @@ -76,10 +101,25 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v else: generator_data.rewards = -F.logsigmoid(discriminator(generator_data.states, generator_data.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.rewards[~generator_data.dones].sum() / generator_data.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode, epoch) value, policy = ppo_step(value, policy, generator_data.states, generator_data.actions, generator_data.rewards, generator_data.dones, clip_ratio, gamma, gae_lambda, pi_opt, pi_iters, v_opt, v_iters, target_kl, max_grad_norm) expert_data = roll_buffer(expert_data, shifts=-3, dims=0) + + if callback is not None: + callback({ + 'epoch': epoch, + 'value': value, + 'policy': policy, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'gen/collision_rate': gen_collision_rate.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), + }) + + for lr_scheduler in lr_schedulers: + lr_scheduler.step() return value, policy diff --git a/src/core/policy.py b/src/core/policy.py index 5485159..ba4b3f6 100644 --- a/src/core/policy.py +++ b/src/core/policy.py @@ -36,30 +36,31 @@ class BasePolicy(nn.Module): class Policy(BasePolicy): - def __init__(self, *args, **kwargs): + def __init__(self, *args, hidden_layer_size=50, n_hidden_layers=2, activation=nn.Tanh, **kwargs): super().__init__(*args, **kwargs) - self.nn = nn.Sequential( - nn.LazyLinear(50), - nn.Tanh(), - nn.LazyLinear(50), - nn.Tanh(), - nn.LazyLinear(2 * self.action_dim), - ) + layers = sum([[nn.LazyLinear(hidden_layer_size), + activation()] for _ in range(n_hidden_layers)],[]) + self.nn = nn.Sequential(*layers, nn.LazyLinear(2 *self.action_dim)) + + # old + # self.nn = nn.Sequential( + # nn.LazyLinear(50), + # nn.Tanh(), + # nn.LazyLinear(50), + # nn.Tanh(), + # nn.LazyLinear(2 * self.action_dim), + #) def forward(self, states): return self.nn(states) class DiscretePolicy(BasePolicy): - def __init__(self, *args, **kwargs): + def __init__(self, *args, hidden_layer_size=50, n_hidden_layers=2, activation=nn.Tanh, **kwargs): super().__init__(*args, **kwargs) - self.nn = nn.Sequential( - nn.LazyLinear(50), - nn.Tanh(), - nn.LazyLinear(50), - nn.Tanh(), - nn.LazyLinear(self.action_dim), - ) + layers = sum([[nn.LazyLinear(hidden_layer_size), + activation()] for _ in range(n_hidden_layers)],[]) + self.nn = nn.Sequential(*layers, nn.LazyLinear(self.action_dim)) def forward(self, states): return self.nn(states) diff --git a/src/core/sampling.py b/src/core/sampling.py index 66fbbef..f4b441d 100644 --- a/src/core/sampling.py +++ b/src/core/sampling.py @@ -2,6 +2,7 @@ import torch import gym from stable_baselines3.common.vec_env import DummyVecEnv as VecEnv from tqdm import tqdm +import numpy as np def rollout(env_fn, policy, n_episodes, max_steps_per_episode): env = env_fn(0) @@ -9,23 +10,27 @@ def rollout(env_fn, policy, n_episodes, max_steps_per_episode): actions = torch.zeros(n_episodes, max_steps_per_episode + 1, *env.action_space.shape) rewards = torch.zeros(n_episodes, max_steps_per_episode + 1) dones = torch.ones(n_episodes, max_steps_per_episode + 1, dtype=bool) + collisions = torch.zeros(n_episodes, max_steps_per_episode, dtype=bool) env = VecEnv(list(map(lambda i: (lambda: env_fn(i)), range(n_episodes)))) states[:, 0] = torch.tensor(env.reset()).clone().detach() dones[:, 0] = False - for s in range(max_steps_per_episode): + for s in tqdm(range(max_steps_per_episode), 'Rollout'): actions[:, s] = policy.sample(policy(states[:, s])).clone().detach() clipped_actions = actions[:, s] if isinstance(env.action_space, gym.spaces.Box): clipped_actions = torch.clamp(clipped_actions, torch.from_numpy(env.action_space.low), torch.from_numpy(env.action_space.high)) - o, r, d, _ = env.step(clipped_actions) + o, r, d, info = env.step(clipped_actions) states[:, s + 1] = torch.tensor(o).clone().detach() rewards[:, s] = torch.tensor(r).clone().detach() dones[:, s + 1] = torch.tensor(d).clone().detach() + collisions[:, s] = torch.from_numpy(np.stack([ + i['collision'] for i in info + ])).detach().clone() dones = dones.cumsum(1) > 0 @@ -34,7 +39,7 @@ def rollout(env_fn, policy, n_episodes, max_steps_per_episode): rewards = rewards[:, :max_steps_per_episode] dones = dones[:, :max_steps_per_episode] - return states, actions, rewards, dones + return states, actions, rewards, dones, collisions def rollout_sb3(env, policy, n_episodes, max_steps_per_episode): diff --git a/src/eval_main.py b/src/eval_main.py index 2e38932..2e50286 100644 --- a/src/eval_main.py +++ b/src/eval_main.py @@ -14,7 +14,8 @@ from src.core.reparam_module import ReparamPolicy, ReparamSafePolicy from src.options import envs as options_envs2 from src.safe_options.policy import SetMaskedDiscretePolicy from src.safe_options import options as options_envs3 - +from src.util.wrappers import IntersimpleTimeLimit +import os from typing import Optional, List, Dict, Tuple import torch import numpy as np @@ -36,46 +37,47 @@ def load_policy(method:str, Returns: policy (Optional[BaseAlgorithm]): the policy to evaluate """ + ml = torch.device('cpu') if not torch.cuda.is_available() else None if method == 'idm': policy = IDMRulePolicy(env, **policy_kwargs) elif method == 'bc': policy = SetPolicy(env.action_space.shape[-1]) - policy.load_state_dict(torch.load(policy_file)) + policy.load_state_dict(torch.load(policy_file, map_location=ml)) + policy.eval() + elif method == 'gail-trpo': + policy = SetPolicy(env.action_space.shape[-1]) + policy(torch.zeros(env.observation_space.shape)) + policy = ReparamPolicy(policy) + policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'gail': policy = SetPolicy(env.action_space.shape[-1]) - policy(torch.zeros(env.observation_space.shape)) - policy = ReparamPolicy(policy) - policy.load_state_dict(torch.load(policy_file)) - policy.eval() - elif method == 'gail-ppo': - policy = SetPolicy(env.action_space.shape[-1]) - policy.load_state_dict(torch.load(policy_file)) + policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() elif method == 'rail': raise NotImplementedError - elif method == 'ogail': + elif method == 'hail-trpo': policy = SetDiscretePolicy(env.action_space.n) policy(torch.zeros(env.observation_space.shape)) policy = ReparamPolicy(policy) - policy.load_state_dict(torch.load(policy_file)) + policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() - elif method == 'ogail-ppo': + elif method == 'hail': policy = SetDiscretePolicy(env.action_space.n) - policy.load_state_dict(torch.load(policy_file)) + policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() - elif method == 'sgail': + elif method == 'shail-trpo': policy = SetMaskedDiscretePolicy(env.action_space.n) policy( torch.zeros(env.observation_space['observation'].shape), torch.zeros(env.observation_space['safe_actions'].shape) ) policy = ReparamSafePolicy(policy) - policy.load_state_dict(torch.load(policy_file)) + policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() - elif method == 'sgail-ppo': + elif method == 'shail': policy = SetMaskedDiscretePolicy(env.action_space.n) - policy.load_state_dict(torch.load(policy_file)) + policy.load_state_dict(torch.load(policy_file, map_location=ml)) policy.eval() else: raise NotImplementedError @@ -201,7 +203,7 @@ def evaluate_policy(locations:List[Tuple[int,int]], # iterate through vehicles for i, location in tqdm(enumerate(locations)): - + # add roundabout and track to environent iround, track = location rname = intersim.LOCATIONS[iround] @@ -214,7 +216,14 @@ def evaluate_policy(locations:List[Tuple[int,int]], # initialize environment Env = envs_dict[env_class] - eval_env = Env(**env_kwargs) + + # wrap in TimeLimit + if 'max_episode_steps' in it_env_kwargs.keys(): + steps = it_env_kwargs.pop('max_episode_steps') + eval_env = IntersimpleTimeLimit(Env(**it_env_kwargs), + max_episode_steps=steps) + else: + eval_env = Env(**it_env_kwargs) evaluator = IntersimpleEvaluation(eval_env) # load policy @@ -238,6 +247,14 @@ def summary_metrics(metrics:List[Dict[str,list]]) -> Dict[str,float]: """ # keys = ['col_all','v_all', 'a_all','j_all', 'v_avg', 'a_avg', 'col', 'brake', 't'] summary_metrics = {} + + # mean travel distance + dt = 0.1 + travel_ds = [] + for iRound in range(len(metrics)): + for iTraj in range(len(metrics[iRound]['v_all'])): + travel_ds.append(dt*sum(metrics[iRound]['v_all'][iTraj])) + summary_metrics['mean travel distance'] = sum(travel_ds)/len(travel_ds) # average average-velocity all_vavgs = sum([d['v_avg'] for d in metrics],[]) # aggregate to single list @@ -265,6 +282,7 @@ def summary_metrics(metrics:List[Dict[str,list]]) -> Dict[str,float]: # collision rate all_collisions = sum([d['col'] for d in metrics],[]) # aggregate to single list summary_metrics['collision rate'] = sum(all_collisions)/len(all_collisions) + summary_metrics['success rate'] = 1 - summary_metrics['collision rate'] # hard brake rate all_hard_brakes = sum([d['brake'] for d in metrics],[]) # aggregate to single list @@ -273,6 +291,8 @@ def summary_metrics(metrics:List[Dict[str,list]]) -> Dict[str,float]: # average number of timesteps all_ts = sum([d['t'] for d in metrics],[]) # aggregate to single list summary_metrics['mean episode length'] = sum(all_ts)/len(all_ts) + summary_metrics['mean episode time'] = summary_metrics['mean episode length'] * dt + for key in summary_metrics.keys(): print(f'{key}: {summary_metrics[key]}') @@ -303,7 +323,7 @@ def comparison_metrics(policy_metrics:List[Dict[str,list]], expert_traj.append(np.vstack((expert_metrics[iR]['x_all'][iTraj], expert_metrics[iR]['y_all'][iTraj]))) policy_traj.append(np.vstack((policy_metrics[iR]['x_all'][iTraj], policy_metrics[iR]['y_all'][iTraj]))) assert len(expert_traj)==len(policy_traj) - comparison_metrics['rwse'] = rwse(expert_traj, policy_traj) + comparison_metrics.update(rwse(expert_traj, policy_traj)) # average velocity shortfall expert_vavg = np.array(sum([d['v_avg'] for d in expert_metrics],[])) @@ -355,14 +375,28 @@ def eval_main( policy_file (str): path to saved policy env (str): environment class method (str): method (expert, bc, gail, rail, hgail, hrail) + + Returns: + outbase (str): string to outbase """ - print(f'Evaluating {method} on {env}') + print(f'#############################################################################') + print(f'Evaluating {method} from file {policy_file} on {env} at locations {locations}') + print(f'#############################################################################') # set seed np.random.seed(seed) torch.manual_seed(seed) - pfilename = policy_file.split('/')[-1].split('.')[0] - outbase = f'out/{method}/{pfilename}_seed{seed}' + locstr = 'loc_'+'_'.join([f'r{ro}t{tr}' for (ro,tr) in locations]) + if policy_file == '': + method_path = method + name_base = method + else: + path_items = policy_file.split('/') + name_base = path_items[-1].split('.')[0] + method_path = ('/').join(path_items[1:-1]) + outfolder = os.path.join('out',method_path,locstr) + filebase = name_base + f'_tseed{seed}' + outbase = os.path.join(outfolder,filebase) # load expert metrics expert_metrics = generate_expert_metrics(locations) @@ -381,7 +415,9 @@ def eval_main( save_metrics(smetrics, outbase+'_summary.pkl') cmetrics = comparison_metrics(policy_metrics, expert_metrics, outbase=outbase) save_metrics(cmetrics, outbase+'_comparison.pkl') + + return outbase if __name__=='__main__': import fire - fire.Fire(eval_main) \ No newline at end of file + fire.Fire(eval_main) diff --git a/src/evaluation/evaluation.py b/src/evaluation/evaluation.py index b63d8cd..abc6b34 100644 --- a/src/evaluation/evaluation.py +++ b/src/evaluation/evaluation.py @@ -6,8 +6,9 @@ from typing import Callable, Dict, Optional import os import pickle from tqdm import tqdm +from src.util.wrappers import IntersimpleTimeLimit from src.options.envs import OptionsEnv -from src.util.wrappers import OptionsTimeLimit +from src.safe_options.options import SafeOptionsEnv class IntersimpleEvaluation: """ @@ -36,7 +37,10 @@ class IntersimpleEvaluation: self.env = eval_env self.n_episodes = eval_env.nv self.use_pbar = use_pbar - self.is_options_env = isinstance(self.env, (OptionsEnv, OptionsTimeLimit)) + if isinstance(self.env, IntersimpleTimeLimit): + self.is_options_env = isinstance(self.env.env, (OptionsEnv, SafeOptionsEnv)) + else: + self.is_options_env = isinstance(self.env, (OptionsEnv, SafeOptionsEnv)) # metrics present on every step of every episode self.metric_keys_all = ['x_all', 'y_all', 'v_all', 'a_all', 'col_all'] @@ -135,8 +139,8 @@ class IntersimpleEvaluation: self._metrics['a_all'][_agent].append(info['action_taken'][_agent,0].item()) col = info['collision'] - if col: - assert done + # if col: # commenting out if we dont want to end on collision + # assert done self._metrics['col_all'][_agent].append(col) if done and self.use_pbar: diff --git a/src/evaluation/metrics.py b/src/evaluation/metrics.py index 4098229..3b40d47 100644 --- a/src/evaluation/metrics.py +++ b/src/evaluation/metrics.py @@ -4,10 +4,58 @@ import numpy as np import matplotlib.pyplot as plt from torch.utils.data import DataLoader from intersim import collisions -from typing import List +from typing import List, Dict # import tikzplotlib -def rwse(expert:List[np.ndarray], policy:List[np.ndarray], dt:float=0.1) -> float: +def rwse(expert:List[np.ndarray], policy:List[np.ndarray], dt:float=0.1) -> Dict[str,float]: + """ + Calculate average mean squared displacement error + + Args: + expert (List[np.ndarray]): all position trajectories for all expert rollouts + policy (List[np.ndarray]): all position trajectories for all policy rollouts + + each trajectory in the list should have shape (2, T). however expert[i] might have a + different T than policy[i] + + Returns + rwse_dict (Dict[str,float]): dict of different RWSEs + """ + assert len(expert) == len(policy) + + # calculate rwse + times = [1,2,5,10,15,20,25,30] + time_indices = [int(t/dt) for t in times] + rwse_dict_keys = [f'rwse_{t}s' for t in times]+['rwse_end'] + se_dict = {key:[] for key in rwse_dict_keys} + for expert_trajectory, policy_trajectory in zip(expert, policy): + _, T1 = expert_trajectory.shape + _, T2 = policy_trajectory.shape + minT = min(T1, T2) + + crop_expert_trajectory = expert_trajectory[:, :minT] + crop_policy_trajectory = policy_trajectory[:, :minT] + + # square error along every time + se = ((crop_policy_trajectory - crop_expert_trajectory)**2).sum(0) + + # add to dict with appropriate indexing + for time, idx in zip(times, time_indices): + if minT >= idx: + se_dict[f'rwse_{time}s'].append(se[idx-1]) + se_dict['rwse_end'].append(se[-1]) + + assert len(se_dict['rwse_end']) == len(expert) + + # print how many trajectories of each time: + for key in rwse_dict_keys: + print('%s has %i elements'%(key, len(se_dict[key]))) + + rwse_dict = {key:np.mean(np.array(se_dict[key]))**0.5 for key in rwse_dict_keys} + + return rwse_dict + +def rwse_basic(expert:List[np.ndarray], policy:List[np.ndarray], dt:float=0.1) -> float: """ Calculate average mean squared displacement error @@ -42,8 +90,6 @@ def rwse(expert:List[np.ndarray], policy:List[np.ndarray], dt:float=0.1) -> floa return avg_rwse - - def visualize_distribution(expert, policy, filestr): """ Visualize two distributions diff --git a/src/evaluation/utils.py b/src/evaluation/utils.py index 7c78ca5..2e7c4fe 100644 --- a/src/evaluation/utils.py +++ b/src/evaluation/utils.py @@ -2,6 +2,7 @@ import pickle import os import numpy as np from typing import List,Dict + def save_metrics(metrics:dict, filestr:str): """ Save metric dict to filestr @@ -33,13 +34,20 @@ def load_metrics(filestr:str): metrics = pickle.load(f) return metrics -def average_metrics(metric_list:List[Dict[str,float]]): +def average_metrics(metric_list:List[Dict[str,float]], verbose:bool=True) ->Dict[str, tuple]: """ Average all the metrics in the list Args: metric_list (list of dicts): list of metric dicts which each map a string to a float + verbose (bool): whether to print avg metrics + Returns: + average_metrics (Dict[str, tuple]) """ + average_metrics = {} + if len(metric_list) == 0: + return average_metrics + keys = list(metric_list[0].keys()) N = len(metric_list) master_dict = {key:[] for key in keys} @@ -47,31 +55,41 @@ def average_metrics(metric_list:List[Dict[str,float]]): for i in range(N): master_dict[key].append(metric_list[i][key]) master_dict[key] = np.array(master_dict[key]) - mu = np.mean(master_dict[key]) - std2 = np.std(master_dict[key])*2 - print(f'{key}: {mu} \pm {std2}') + mu = np.nanmean(master_dict[key]) + std2 = np.nanstd(master_dict[key])*2 + if verbose: + print(f'{key}: {mu} \pm {std2}') + average_metrics[key] = (mu, std2) + return average_metrics -def load_and_average(path:str): +def load_and_average(path:str, verbose:bool=True): """ Load and average all metric files in a particular folder Args: path (str) + verbose (bool): whether to print avg metrics + Returns: + avg_metrics (Dict[str, tuple]) """ assert os.path.isdir(path) # summary metrics summary_files = [os.path.join(path,f) for f in os.listdir(path) if f.endswith('summary.pkl')] - print(*summary_files, sep='\n') + if verbose: + print(*summary_files, sep='\n') all_summary_metrics = [load_metrics(f) for f in summary_files] - average_metrics(all_summary_metrics) + avg_metrics = average_metrics(all_summary_metrics, verbose=verbose) # comparison metrics comp_files = [os.path.join(path,f) for f in os.listdir(path) if f.endswith('comparison.pkl')] - print(*comp_files, sep='\n') + if verbose: + print(*comp_files, sep='\n') all_comp_metrics = [load_metrics(f) for f in comp_files] - average_metrics(all_comp_metrics) + comp_avg = average_metrics(all_comp_metrics, verbose=verbose) + avg_metrics.update(comp_avg) + return avg_metrics if __name__=='__main__': import fire diff --git a/src/safe_options/options.py b/src/safe_options/options.py index 87ab047..7e160af 100644 --- a/src/safe_options/options.py +++ b/src/safe_options/options.py @@ -14,7 +14,7 @@ from src.options.envs import OptionsEnv from src.safe_options.collisions import feasible from intersim.envs import IntersimpleLidarFlatIncrementingAgent -from src.util.wrappers import OptionsTimeLimit, Setobs, TransformObservation +from src.util.wrappers import Setobs, TransformObservation @dataclass class Buffer: @@ -47,14 +47,18 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value logger.add_scalar('expert/mean_reward_per_episode', expert_data.rewards[~expert_data.dones].sum() / expert_data.states.shape[0]) for epoch in tqdm(range(epochs)): - hl_data, ll_data = rollout(env_fn, policy, rollout_episodes, rollout_steps) + hl_data, ll_data, collisions = rollout(env_fn, policy, rollout_episodes, rollout_steps) generator_data = OptionsRollout(HLBuffer(*hl_data), Buffer(*ll_data)) generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) - logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) - logger.add_scalar('gen/mean_reward_per_episode', generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length , epoch) + gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] + logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) + gen_collision_rate = (1. * collisions.any(-1)).mean() + logger.add_scalar('gen/collision_rate', gen_collision_rate, epoch) discriminator, loss = train_discriminator(expert_data, generator_data.ll, discriminator, disc_opt, disc_iters, wasserstein, wasserstein_c) if wasserstein: @@ -62,7 +66,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value else: generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode , epoch) #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) @@ -71,26 +76,37 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value expert_data = roll_buffer(expert_data, shifts=-3, dims=0) if callback is not None: - callback(epoch, value, policy) + callback({ + 'epoch': epoch, + 'value': value, + 'policy': policy, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'gen/collision_rate': gen_collision_rate.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), + }) return value, policy def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value, v_opt, v_iters, epochs, rollout_episodes, rollout_steps, gamma, - gae_lambda, clip_ratio, pi_opt, pi_iters, target_kl=None, max_grad_norm=None, wasserstein=False, wasserstein_c=None, logger=TerminalLogger(), callback=None): + gae_lambda, clip_ratio, pi_opt, pi_iters, target_kl=None, max_grad_norm=None, wasserstein=False, wasserstein_c=None, logger=TerminalLogger(), callback=None, lr_schedulers=[]): logger.add_scalar('expert/mean_episode_length', (~expert_data.dones).sum() / expert_data.states.shape[0]) logger.add_scalar('expert/mean_reward_per_episode', expert_data.rewards[~expert_data.dones].sum() / expert_data.states.shape[0]) for epoch in range(epochs): - hl_data, ll_data = rollout(env_fn, policy, rollout_episodes, rollout_steps) + hl_data, ll_data, collisions = rollout(env_fn, policy, rollout_episodes, rollout_steps) generator_data = OptionsRollout(HLBuffer(*hl_data), Buffer(*ll_data)) generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions) - - logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch) - logger.add_scalar('gen/mean_reward_per_episode', generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0], epoch) + gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0] + logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length, epoch) + gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0] + logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch) logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch) + gen_collision_rate = (1. * collisions.any(-1)).mean() + logger.add_scalar('gen/collision_rate', gen_collision_rate, epoch) discriminator, loss = train_discriminator(expert_data, generator_data.ll, discriminator, disc_opt, disc_iters, wasserstein, wasserstein_c) if wasserstein: @@ -98,7 +114,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v else: generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions)) logger.add_scalar('disc/final_loss', loss, epoch) - logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch) + disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0] + logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode, epoch) #assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1) @@ -107,7 +124,18 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v expert_data = roll_buffer(expert_data, shifts=-3, dims=0) if callback is not None: - callback(epoch, value, policy) + callback({ + 'epoch': epoch, + 'value': value, + 'policy': policy, + 'gen/mean_episode_length': gen_mean_episode_length.item(), + 'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(), + 'gen/collision_rate': gen_collision_rate.item(), + 'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(), + }) + + for lr_scheduler in lr_schedulers: + lr_scheduler.step() return value, policy @@ -119,6 +147,7 @@ def rollout(env_fn, policy, n_episodes, max_steps_per_episode): actions = torch.zeros(n_episodes, max_steps_per_episode + 1, *env.action_space.shape) rewards = torch.zeros(n_episodes, max_steps_per_episode + 1) dones = torch.ones(n_episodes, max_steps_per_episode + 1, dtype=bool) + collisions = torch.zeros(n_episodes, max_steps_per_episode, dtype=bool) ll_states = torch.zeros(n_episodes, max_steps_per_episode, env.max_plan_length + 1, *env.observation_space['observation'].shape) ll_actions = torch.zeros(n_episodes, max_steps_per_episode, env.max_plan_length + 1, *env.ll_action_space.shape) @@ -144,6 +173,9 @@ def rollout(env_fn, policy, n_episodes, max_steps_per_episode): safe_actions[:, s + 1] = torch.tensor(o['safe_actions']).clone().detach() rewards[:, s] = torch.tensor(r).clone().detach() dones[:, s + 1] = torch.tensor(d).clone().detach() + collisions[:, s] = torch.from_numpy(np.stack([ + any(k['collision'] for k in i['ll']['infos']) for i in info + ])).detach().clone() ll_states[:, s] = torch.from_numpy(np.stack([i['ll']['observations'] for i in info])).clone().detach() ll_actions[:, s] = torch.from_numpy(np.stack([i['ll']['actions'] for i in info])).clone().detach() @@ -158,7 +190,7 @@ def rollout(env_fn, policy, n_episodes, max_steps_per_episode): rewards = rewards[:, :max_steps_per_episode] dones = dones[:, :max_steps_per_episode] - return (states, safe_actions, actions, rewards, dones), (ll_states, ll_actions, ll_rewards, ll_dones) + return (states, safe_actions, actions, rewards, dones), (ll_states, ll_actions, ll_rewards, ll_dones), collisions class SafeOptionsEnv(OptionsEnv): @@ -176,6 +208,7 @@ class SafeOptionsEnv(OptionsEnv): return np.ones(len(self.options), dtype=bool) plans = [self.plan(o) for o in self.options] + plans = [np.pad(p, (0, self.max_plan_length - len(p)), constant_values=np.nan) for p in plans] plans = np.stack(plans) safe = feasible(self.env, plans, method=self.safe_actions_collision_method) if not safe.any(): @@ -226,9 +259,9 @@ class SafeOptionsEnv(OptionsEnv): if d: break - if self.abort_unsafe_collision_method is not None and \ - not feasible(self.env, plan[k:], method=self.abort_unsafe_collision_method): - break + if self.abort_unsafe_collision_method is not None: + if not feasible(self.env, plan[k:], method=self.abort_unsafe_collision_method): + break n_steps = k + 1 return observations, actions, rewards, env_done, plan_done, infos, n_steps @@ -251,10 +284,10 @@ obs_max = np.array([ [50, np.pi, 20, 20, np.pi, 1e-1], ]).reshape(-1) -def NormalizedSafeOptionsEvalEnv(max_episode_steps=float('inf'), safe_actions_collision_method=None, abort_unsafe_collision_method=None, **kwargs): - return OptionsTimeLimit(SafeOptionsEnv(Setobs( +def NormalizedSafeOptionsEvalEnv(safe_actions_collision_method='circle', abort_unsafe_collision_method='circle', **kwargs): + return SafeOptionsEnv(Setobs( TransformObservation(IntersimpleLidarFlatIncrementingAgent( n_rays=5, **kwargs, ), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)) - ), options=[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], safe_actions_collision_method=safe_actions_collision_method, abort_unsafe_collision_method=abort_unsafe_collision_method), max_episode_steps=max_episode_steps) + ), options=[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)], safe_actions_collision_method=safe_actions_collision_method, abort_unsafe_collision_method=abort_unsafe_collision_method) diff --git a/src/util/wrappers.py b/src/util/wrappers.py index 3a088f9..d2d62ea 100644 --- a/src/util/wrappers.py +++ b/src/util/wrappers.py @@ -9,7 +9,7 @@ class TransformObservation(gym.wrappers.TransformObservation): def __getattr__(self, name): return getattr(self.env, name) -class OptionsTimeLimit(gym.wrappers.TimeLimit): +class IntersimpleTimeLimit(gym.wrappers.TimeLimit): def __getattr__(self, name): return getattr(self.env, name) diff --git a/test_policies/bc/expA/policy_seed1.pt b/test_policies/bc/expA/policy_seed1.pt new file mode 100644 index 0000000..a23bb84 Binary files /dev/null and b/test_policies/bc/expA/policy_seed1.pt differ diff --git a/test_policies/bc/expA/policy_seed2.pt b/test_policies/bc/expA/policy_seed2.pt new file mode 100644 index 0000000..dadcdbc Binary files /dev/null and b/test_policies/bc/expA/policy_seed2.pt differ diff --git a/test_policies/bc/expA/policy_seed3.pt b/test_policies/bc/expA/policy_seed3.pt new file mode 100644 index 0000000..0fc6651 Binary files /dev/null and b/test_policies/bc/expA/policy_seed3.pt differ diff --git a/test_policies/bc/expA/policy_seed4.pt b/test_policies/bc/expA/policy_seed4.pt new file mode 100644 index 0000000..a00f1bb Binary files /dev/null and b/test_policies/bc/expA/policy_seed4.pt differ diff --git a/test_policies/bc/expA/policy_seed5.pt b/test_policies/bc/expA/policy_seed5.pt new file mode 100644 index 0000000..443e3e3 Binary files /dev/null and b/test_policies/bc/expA/policy_seed5.pt differ diff --git a/test_policies/bc/expB/policy_seed1.pt b/test_policies/bc/expB/policy_seed1.pt new file mode 100644 index 0000000..6e44cf8 Binary files /dev/null and b/test_policies/bc/expB/policy_seed1.pt differ diff --git a/test_policies/bc/expB/policy_seed2.pt b/test_policies/bc/expB/policy_seed2.pt new file mode 100644 index 0000000..cc49704 Binary files /dev/null and b/test_policies/bc/expB/policy_seed2.pt differ diff --git a/test_policies/bc/expB/policy_seed3.pt b/test_policies/bc/expB/policy_seed3.pt new file mode 100644 index 0000000..715b217 Binary files /dev/null and b/test_policies/bc/expB/policy_seed3.pt differ diff --git a/test_policies/bc/expB/policy_seed4.pt b/test_policies/bc/expB/policy_seed4.pt new file mode 100644 index 0000000..84ee90b Binary files /dev/null and b/test_policies/bc/expB/policy_seed4.pt differ diff --git a/test_policies/bc/expB/policy_seed5.pt b/test_policies/bc/expB/policy_seed5.pt new file mode 100644 index 0000000..ce52202 Binary files /dev/null and b/test_policies/bc/expB/policy_seed5.pt differ diff --git a/test_policies/gail/expA/policy_seed1.pt b/test_policies/gail/expA/policy_seed1.pt new file mode 100644 index 0000000..0530c10 Binary files /dev/null and b/test_policies/gail/expA/policy_seed1.pt differ diff --git a/test_policies/gail/expA/policy_seed2.pt b/test_policies/gail/expA/policy_seed2.pt new file mode 100644 index 0000000..9d85579 Binary files /dev/null and b/test_policies/gail/expA/policy_seed2.pt differ diff --git a/test_policies/gail/expA/policy_seed3.pt b/test_policies/gail/expA/policy_seed3.pt new file mode 100644 index 0000000..6d48834 Binary files /dev/null and b/test_policies/gail/expA/policy_seed3.pt differ diff --git a/test_policies/gail/expA/policy_seed4.pt b/test_policies/gail/expA/policy_seed4.pt new file mode 100644 index 0000000..709f468 Binary files /dev/null and b/test_policies/gail/expA/policy_seed4.pt differ diff --git a/test_policies/gail/expA/policy_seed5.pt b/test_policies/gail/expA/policy_seed5.pt new file mode 100644 index 0000000..a524ac9 Binary files /dev/null and b/test_policies/gail/expA/policy_seed5.pt differ diff --git a/test_policies/gail/expB/policy_seed1.pt b/test_policies/gail/expB/policy_seed1.pt new file mode 100644 index 0000000..a97af9c Binary files /dev/null and b/test_policies/gail/expB/policy_seed1.pt differ diff --git a/test_policies/gail/expB/policy_seed2.pt b/test_policies/gail/expB/policy_seed2.pt new file mode 100644 index 0000000..62b1490 Binary files /dev/null and b/test_policies/gail/expB/policy_seed2.pt differ diff --git a/test_policies/gail/expB/policy_seed3.pt b/test_policies/gail/expB/policy_seed3.pt new file mode 100644 index 0000000..4291a26 Binary files /dev/null and b/test_policies/gail/expB/policy_seed3.pt differ diff --git a/test_policies/gail/expB/policy_seed4.pt b/test_policies/gail/expB/policy_seed4.pt new file mode 100644 index 0000000..a8b9423 Binary files /dev/null and b/test_policies/gail/expB/policy_seed4.pt differ diff --git a/test_policies/gail/expB/policy_seed5.pt b/test_policies/gail/expB/policy_seed5.pt new file mode 100644 index 0000000..c9fe325 Binary files /dev/null and b/test_policies/gail/expB/policy_seed5.pt differ diff --git a/test_policies/hail/expA/policy_seed1.pt b/test_policies/hail/expA/policy_seed1.pt new file mode 100644 index 0000000..2ec567e Binary files /dev/null and b/test_policies/hail/expA/policy_seed1.pt differ diff --git a/test_policies/hail/expA/policy_seed2.pt b/test_policies/hail/expA/policy_seed2.pt new file mode 100644 index 0000000..6135439 Binary files /dev/null and b/test_policies/hail/expA/policy_seed2.pt differ diff --git a/test_policies/hail/expA/policy_seed3.pt b/test_policies/hail/expA/policy_seed3.pt new file mode 100644 index 0000000..bd50d14 Binary files /dev/null and b/test_policies/hail/expA/policy_seed3.pt differ diff --git a/test_policies/hail/expA/policy_seed4.pt b/test_policies/hail/expA/policy_seed4.pt new file mode 100644 index 0000000..342c473 Binary files /dev/null and b/test_policies/hail/expA/policy_seed4.pt differ diff --git a/test_policies/hail/expA/policy_seed5.pt b/test_policies/hail/expA/policy_seed5.pt new file mode 100644 index 0000000..5cb425d Binary files /dev/null and b/test_policies/hail/expA/policy_seed5.pt differ diff --git a/test_policies/hail/expB/policy_seed1.pt b/test_policies/hail/expB/policy_seed1.pt new file mode 100644 index 0000000..dfb17c9 Binary files /dev/null and b/test_policies/hail/expB/policy_seed1.pt differ diff --git a/test_policies/hail/expB/policy_seed2.pt b/test_policies/hail/expB/policy_seed2.pt new file mode 100644 index 0000000..407583f Binary files /dev/null and b/test_policies/hail/expB/policy_seed2.pt differ diff --git a/test_policies/hail/expB/policy_seed3.pt b/test_policies/hail/expB/policy_seed3.pt new file mode 100644 index 0000000..92a72b1 Binary files /dev/null and b/test_policies/hail/expB/policy_seed3.pt differ diff --git a/test_policies/hail/expB/policy_seed4.pt b/test_policies/hail/expB/policy_seed4.pt new file mode 100644 index 0000000..3e3baab Binary files /dev/null and b/test_policies/hail/expB/policy_seed4.pt differ diff --git a/test_policies/hail/expB/policy_seed5.pt b/test_policies/hail/expB/policy_seed5.pt new file mode 100644 index 0000000..2727888 Binary files /dev/null and b/test_policies/hail/expB/policy_seed5.pt differ diff --git a/test_policies/shail/expA/policy_seed1.pt b/test_policies/shail/expA/policy_seed1.pt new file mode 100644 index 0000000..3819718 Binary files /dev/null and b/test_policies/shail/expA/policy_seed1.pt differ diff --git a/test_policies/shail/expA/policy_seed2.pt b/test_policies/shail/expA/policy_seed2.pt new file mode 100644 index 0000000..52753fd Binary files /dev/null and b/test_policies/shail/expA/policy_seed2.pt differ diff --git a/test_policies/shail/expA/policy_seed3.pt b/test_policies/shail/expA/policy_seed3.pt new file mode 100644 index 0000000..15a6c1d Binary files /dev/null and b/test_policies/shail/expA/policy_seed3.pt differ diff --git a/test_policies/shail/expA/policy_seed4.pt b/test_policies/shail/expA/policy_seed4.pt new file mode 100644 index 0000000..98cc3e7 Binary files /dev/null and b/test_policies/shail/expA/policy_seed4.pt differ diff --git a/test_policies/shail/expA/policy_seed5.pt b/test_policies/shail/expA/policy_seed5.pt new file mode 100644 index 0000000..f3e3bff Binary files /dev/null and b/test_policies/shail/expA/policy_seed5.pt differ diff --git a/test_policies/shail/expB/policy_seed1.pt b/test_policies/shail/expB/policy_seed1.pt new file mode 100644 index 0000000..e5a93dc Binary files /dev/null and b/test_policies/shail/expB/policy_seed1.pt differ diff --git a/test_policies/shail/expB/policy_seed2.pt b/test_policies/shail/expB/policy_seed2.pt new file mode 100644 index 0000000..c1ce024 Binary files /dev/null and b/test_policies/shail/expB/policy_seed2.pt differ diff --git a/test_policies/shail/expB/policy_seed3.pt b/test_policies/shail/expB/policy_seed3.pt new file mode 100644 index 0000000..ad03c95 Binary files /dev/null and b/test_policies/shail/expB/policy_seed3.pt differ diff --git a/test_policies/shail/expB/policy_seed4.pt b/test_policies/shail/expB/policy_seed4.pt new file mode 100644 index 0000000..3c309da Binary files /dev/null and b/test_policies/shail/expB/policy_seed4.pt differ diff --git a/test_policies/shail/expB/policy_seed5.pt b/test_policies/shail/expB/policy_seed5.pt new file mode 100644 index 0000000..7600e77 Binary files /dev/null and b/test_policies/shail/expB/policy_seed5.pt differ