Merge branch 'main' into idm_upgrade

.gitignore

@@ -1,3 +1,4 @@
+*.png
 *.pkl
 *.pt
 *.zip

bc-experiment.py

@@ -0,0 +1,212 @@
+# %%
+import os
+
+from tqdm import tqdm
+from src.core.sampling import rollout
+from src.core.gail import gail_ppo, Buffer
+from src.core.value import SetValue
+from src.core.policy import SetPolicy
+from src.core.discriminator import DeepsetDiscriminator
+import torch
+
+from intersim.envs import IntersimpleLidarFlatRandom
+from intersim.envs.intersimple import speed_reward
+import functools
+from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Setobs
+import numpy as np
+from torch.utils.tensorboard import SummaryWriter
+from ray import tune
+from datetime import datetime
+import json
+
+DIR = os.path.dirname(os.path.abspath(__file__))
+activations = [torch.nn.Tanh, torch.nn.LeakyReLU]
+
+obs_min = np.array([
+    [-1000, -1000, 0, -np.pi, -1e-1, 0.],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+]).reshape(-1)
+
+obs_max = np.array([
+    [1000, 1000, 20, np.pi, 1e-1, 0.],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+]).reshape(-1)
+
+def training_function(config):
+    np.random.seed(config['seed'])
+    torch.manual_seed(config['seed'])
+
+    # choose validation environment
+    if config['experiment'] == 'A': 
+        envs = [Setobs(TransformObservation(CollisionPenaltyWrapper(
+            IntersimpleLidarFlatRandom(
+                n_rays=5,
+                reward=functools.partial(
+                    speed_reward,
+                    collision_penalty=0
+                ),
+                check_collisions=True,
+                stop_on_collision=config['trainenv']['stop_on_collision'],
+            ), collision_distance=6, collision_penalty=100),
+            lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
+        )) for _ in range(60)]
+    elif config['experiment'] == 'B': 
+        envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper(
+            IntersimpleLidarFlatRandom(
+                n_rays=5,
+                reward=functools.partial(
+                    speed_reward,
+                    collision_penalty=0
+                ),
+                check_collisions=True,
+                stop_on_collision=config['trainenv']['stop_on_collision'],
+            ), collision_distance=6, collision_penalty=100),
+            lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
+        )) for _ in range(15)] for track in range(4)],[])
+    else:
+        raise NotImplementedError
+
+    env_fn = lambda i: envs[i]
+
+    # load expert data
+
+    if config['experiment'] == 'A': 
+        expert_data = torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt'))
+    elif config['experiment'] == 'B': 
+        expert_data = [
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')),
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track1.pt')),
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track2.pt')),
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track3.pt')),
+        ]
+        d0 = [d[0] for d in expert_data]
+        d1 = [d[1] for d in expert_data]
+        d2 = [d[2] for d in expert_data]
+        d3 = [d[3] for d in expert_data]
+        expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3))
+    
+    expert_data = Buffer(*expert_data)
+
+    # configure and train policy
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    policy = SetPolicy(expert_data.actions.shape[-1], 
+        n_hidden_layers=config['policy']['n_hidden_layers'],
+        hidden_layer_size=config['policy']['hidden_layer_size'],
+        activation=activations[config['policy']['activation']] ) # config net architecture
+    policy = policy.to(device)
+    
+    pi_opt = torch.optim.Adam(policy.parameters(), lr=config['policy']['learning_rate'])
+    pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=config['policy']['learning_rate_decay'])
+
+    expert_states = expert_data.states[~expert_data.dones].to(device)
+    expert_actions = expert_data.actions[~expert_data.dones].to(device)
+
+    for epoch in range(config['train_epochs']):
+        pi_opt.zero_grad()
+        loss = -policy.log_prob(policy(expert_states), expert_actions).mean()
+        loss.backward()
+        pi_opt.step()
+        pi_lr_scheduler.step()
+
+        if epoch % 25 == 0:
+            gen_states, gen_actions, gen_rewards, gen_dones, gen_collisions = rollout(env_fn, policy.cpu(), n_episodes=60, max_steps_per_episode=200)
+            gen_mean_episode_length = (~gen_dones).sum() / gen_states.shape[0]
+            gen_mean_reward_per_episode = gen_rewards[~gen_dones].sum() / gen_states.shape[0]
+            gen_collision_rate = (1. * gen_collisions.any(-1)).mean()
+
+            tune.report(
+                gen_mean_reward_per_episode=gen_mean_reward_per_episode.item(),
+                mean_episode_length=gen_mean_episode_length.item(),
+                gen_collision_rate=gen_collision_rate.item(),
+                loss=loss.item(),
+            )
+        
+        # save model checkpoints
+        ep = epoch + 1
+        if (ep % 50 == 0):
+            torch.save(policy.state_dict(), f'policy_epoch{ep}.pt')
+
+    # save model
+    torch.save(policy.state_dict(), 'policy_final.pt')
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--train', choices=['A', 'B'])
+    parser.add_argument('--epochs', type=int, default=1000)
+    parser.add_argument('--test', type=str, help='path to config file to run final training on')
+    parser.add_argument('--test_seeds', type=int, default=5)
+    parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over')
+    args = parser.parse_args()
+
+    assert (args.train is None) ^ (args.test is None), 'Must either train on an experiment or test with a config file'
+
+    # if no test config specified, train
+    if args.test is None:
+        print('Running Tuning for Experiment %s'%(args.train))
+        analysis = tune.run(
+            training_function,
+            config={
+                'experiment': args.train,
+                'trainenv': {
+                    'stop_on_collision': False, 
+                },
+                'policy': {
+                    'learning_rate': 3e-4, 
+                    'learning_rate_decay': 1.0, 
+                    'hidden_layer_size': tune.grid_search([20, 40]),
+                    'n_hidden_layers': tune.grid_search([2, 3]), 
+                    'activation':0,
+                },
+                'train_epochs': args.epochs,
+                'seed': 0,
+            }
+            # TODO resources_per_trial={'gpu': 1}
+        )
+        best_config = analysis.get_best_config(metric='gen_collision_rate', mode='min')
+        print('Best config: ', best_config)
+
+        # safe best_config
+        if not os.path.isdir(os.path.join(DIR, 'best_configs')):
+            os.mkdir(os.path.join(DIR, 'best_configs'))
+        
+        # save gail
+        with open(os.path.join(DIR, 'best_configs',f'bc_exp{args.train}.json'), 'w', encoding='utf-8') as f:
+            json.dump(best_config, f, ensure_ascii=False, indent=4)
+
+    # if config file specified, rerun it with appropriate number of seeds
+    else:
+        with open(args.test, 'rb') as f:
+            config = json.load(f)
+
+        print(f'Retraining {args.test} with {args.test_seeds} seeds on experiment {config["experiment"]}')
+
+        # rerun with appropriate number of seeds
+        rpt = {'cpu': int(args.test_cpus/args.test_seeds)} if (args.test_cpus is not None) else None
+        config['seed'] = tune.grid_search(list(range(1,args.test_seeds+1)))
+        analysis = tune.run(training_function, config=config, resources_per_trial=rpt)
+
+        # move final policies to appropriate directory
+        split_ = os.path.basename(args.test).split('_')
+        model = split_[0]
+        exper = split_[-1].split('.')[0]
+        savepath = os.path.join('test_policies',model,exper)
+        
+        if not os.path.isdir(savepath):
+            os.makedirs(savepath)
+        
+        import shutil
+        for i in range(args.test_seeds):
+            s = analysis._checkpoints[i]['config']['seed']
+            check_dir = analysis._checkpoints[i]['logdir']
+            shutil.copyfile(os.path.join(check_dir,'policy_final.pt'), 
+                os.path.join(savepath, f'policy_seed{s}.pt'))

best_configs/bc_expA.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 300,
+    "seed": 0
+}

best_configs/bc_expB.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 300,
+    "seed": 0
+}

best_configs/gail-trpo_expA.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "delta": 0.01,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/gail-trpo_expB.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "delta": 0.01,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/gail_expA.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/gail_expB.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/hail-etienne2_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 10,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/hail-etienne2_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 10,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/hail-etienne3_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/hail-etienne3_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/hail-etienne_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/hail-etienne_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/hail_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 90,
+    "seed": 0
+}

best_configs/hail_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 85,
+    "seed": 0
+}

best_configs/shail-etienne2_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 10,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/shail-etienne2_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 10,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/shail-etienne3_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/shail-etienne3_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 3,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/shail-etienne_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/shail-etienne_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

best_configs/shail_expA.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 90,
+    "seed": 0
+}

best_configs/shail_expB.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 85,
+    "seed": 0
+}

eval_experiments.py

@@ -0,0 +1,83 @@
+import os
+from src.eval_main import eval_main
+from src.evaluation.utils import load_and_average
+
+def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=False):
+    
+    policy_kwargs = {}
+    if method in ['expert', 'idm']:
+        env, env_kwargs ='NRasterizedRouteIncrementingAgent', {}
+    elif method in ['bc','gail']:
+        env='NormalizedContinuousEvalEnv' 
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
+    elif method in ['hail']:
+        env = 'NormalizedOptionsEvalEnv'
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
+    elif method in ['shail']:
+        env = 'NormalizedSafeOptionsEvalEnv'
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
+    else:
+        raise NotImplementedError
+
+    files = ['']
+
+    if folder is not None:
+        files = [os.path.join(folder, f) for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
+        print('%i folders found in %s folder' %(len(files), folder))
+
+    if not skip_running:
+        for policy_file in files:
+            # run metrics on that file
+            outbase = eval_main(locations=locations,
+                                method=method, 
+                                policy_file=policy_file, 
+                                policy_kwargs=policy_kwargs,
+                                env=env, 
+                                env_kwargs=env_kwargs)
+        outfolder = os.path.dirname(outbase)
+    else:
+        locstr = 'loc_'+'_'.join([f'r{ro}t{tr}' for (ro,tr) in locations])
+        if folder is None:
+            outfolder = os.path.join('out',method,locstr)
+        else:
+            path_items = folder.split('/')
+            outfolder = os.path.join('out', '/'.join(path_items[1:]), locstr)
+    
+    # load metrics from save_path
+    average_metrics = load_and_average(outfolder)
+    if method in ['expert', 'idm']:
+        latex_print(average_metrics, light=True)
+    else:
+        latex_print(average_metrics)
+
+def latex_print(am, light=False):
+    """
+    print latex line
+
+    am (Dict[str,tuple]): dict mapping metric_name to (mean, std)
+    """
+
+    print('success rate, distance travelled, RWSE_10, |DeltaV|, AccelJSD')
+    if light:
+        if 'rwse_10s' in am.keys():
+            print("%2.1f& %2.1f & %1.2f & %2.1f& "
+            "%0.3f \\\\" %( 100*am['success rate'][0], am['mean travel distance'][0], am['rwse_10s'][0],
+                            am['average absolute average velocity'][0],am['acceleration distribution divergence'][0] ))
+            return
+
+
+        print("%2.1f& %2.1f & $---$ & $---$ & "
+            "$---$ \\\\" %( 100*am['success rate'][0], am['mean travel distance'][0]))
+        return
+    
+    print("%2.1f \\scriptstyle\\pm %2.1f & %2.1f \\scriptstyle\\pm %2.1f & "
+        "%1.2f \\scriptstyle\\pm %1.2f & %2.1f \\scriptstyle\\pm %1.1f & "
+        "%0.3f \\scriptstyle\\pm %0.3f \\\\" %( 100*am['success rate'][0], 100*am['success rate'][1],
+               am['mean travel distance'][0] , am['mean travel distance'][1]  ,
+               am['rwse_10s'][0] , am['rwse_10s'][1]  ,
+               am['average absolute average velocity'][0] , am['average absolute average velocity'][1]  ,
+               am['acceleration distribution divergence'][0] , am['acceleration distribution divergence'][1]  ))
+
+if __name__=='__main__':
+    import fire 
+    fire.Fire(main)

evaluate_models.sh

@@ -1,42 +1,19 @@
-# eval_main inputs
+# can add --skip_running if you've run the runs before on the saved policies
-# locations: List[Tuple[int,int]]= [(0,0)],
-# method: str='expert', 
-# policy_file: str='', 
-# policy_kwargs: dict={},
-# env: str='NRasterizedRouteIncrementingAgent', 
-# env_kwargs: dict={},
-# seed: int=0
 
-# expert
+python -m eval_experiments 
-python -m src.eval_main
+python -m eval_experiments --locations='[(0,4)]' 
+python -m eval_experiments --method idm 
+python -m eval_experiments --method idm --locations='[(0,4)]' 
+python -m eval_experiments --method bc --folder='test_policies/bc/expA' 
+python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]' 
+python -m eval_experiments --method gail --folder='test_policies/gail/expA' 
+python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]' 
+python -m eval_experiments --method hail --folder='test_policies/hail/expA' --env_kwargs='{safe_actions_collision_method=None,abort_unsafe_collision_method=None}'
+python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]' --env_kwargs='{safe_actions_collision_method=None,abort_unsafe_collision_method=None}'
+python -m eval_experiments --method shail --folder='test_policies/shail/expA'
+python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]'
 
-# idm
+python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expA'
-python -m src.eval_main --method=idm
+python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expB' --locations='[(0,4)]'
-
+python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expA'
-# behavior cloning
+python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expB' --locations='[(0,4)]'
-python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=0
-python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=1
-python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=2
-python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=3
-python -m src.eval_main --method=bc --policy_file='checkpoints/bc-intersimple-setobs2.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=4
-python -m src.evaluation.utils load_and_average out/bc
-
-# GAIL
-python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=0
-python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=1
-python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=2
-python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=3
-python -m src.eval_main --method=gail --policy_file='checkpoints/gail-intersimple-setobs2-03-02-22.pt' --env='NormalizedContinuousEvalEnv' --env_kwargs='{stop_on_collision:True}' --seed=4
-python -m src.evaluation.utils load_and_average out/gail
-
-# options GAIL
-python -m src.eval_main --method=ogail --policy_file='checkpoints/gail-options-setobs2-Feb15_18-49-05.pt' --env='NormalizedOptionsEvalEnv' --env_kwargs='{stop_on_collision:True}'
-
-# options GAIL-PPO
-python -m src.eval_main --method=ogail-ppo --policy_file='checkpoints/gail-ppo-options-setobs2-Feb15_22-05-38.pt' --env='NormalizedOptionsEvalEnv' --env_kwargs='{stop_on_collision:True}'
-
-# SHAIL
-python -m src.eval_main --method=sgail --policy_file='checkpoints/sgail-options-setobs2.pt' --env='NormalizedSafeOptionsEvalEnv' --env_kwargs='{stop_on_collision:True,max_episode_steps:1000}'
-
-# SHAIL-PPO
-python -m src.eval_main --method=sgail-ppo --policy_file='checkpoints/sgail-ppo-options-setobs2-17-02-2022.pt' --env='NormalizedSafeOptionsEvalEnv' --env_kwargs='{stop_on_collision:True,max_episode_steps:1000}'

gail-experiment.py

@@ -0,0 +1,238 @@
+# %%
+import os
+
+import gym
+from src.core.gail import gail_ppo, Buffer
+from src.core.value import SetValue
+from src.core.policy import SetPolicy
+from src.core.discriminator import DeepsetDiscriminator
+import torch
+
+from intersim.envs import IntersimpleLidarFlatRandom
+from intersim.envs.intersimple import speed_reward
+import functools
+from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Setobs
+import numpy as np
+from torch.utils.tensorboard import SummaryWriter
+from ray import tune
+from datetime import datetime
+import json
+
+DIR = os.path.dirname(os.path.abspath(__file__))
+activations = [torch.nn.Tanh, torch.nn.LeakyReLU]
+
+obs_min = np.array([
+    [-1000, -1000, 0, -np.pi, -1e-1, 0.],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+    [0, -np.pi, -20, -20, -np.pi, -1e-1],
+]).reshape(-1)
+
+obs_max = np.array([
+    [1000, 1000, 20, np.pi, 1e-1, 0.],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+    [50, np.pi, 20, 20, np.pi, 1e-1],
+]).reshape(-1)
+
+def training_function(config):
+    np.random.seed(config['seed'])
+    torch.manual_seed(config['seed'])
+
+    if config['experiment'] == 'A':
+        envs = [Setobs(TransformObservation(CollisionPenaltyWrapper(
+            IntersimpleLidarFlatRandom(
+                n_rays=5,
+                reward=functools.partial(
+                    speed_reward,
+                    collision_penalty=0
+                ),
+                check_collisions=True,
+                stop_on_collision=config['trainenv']['stop_on_collision'],
+            ), collision_distance=6, collision_penalty=100),
+            lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
+        )) for _ in range(60)]
+
+    elif config['experiment'] == 'B': 
+        envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper(
+            IntersimpleLidarFlatRandom(
+                n_rays=5,
+                reward=functools.partial(
+                    speed_reward,
+                    collision_penalty=0
+                ),
+                check_collisions=True,
+                stop_on_collision=config['trainenv']['stop_on_collision'],
+                track=track,
+            ), collision_distance=6, collision_penalty=100),
+            lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
+        )) for _ in range(15)] for track in range(4)],[])
+    
+    else:
+        raise NotImplementedError
+
+    env_fn = lambda i: envs[i]
+
+    policy = SetPolicy(env_fn(0).action_space.shape[0], 
+        n_hidden_layers=config['policy']['n_hidden_layers'],
+        hidden_layer_size=config['policy']['hidden_layer_size'],
+        activation=activations[config['policy']['activation']] ) # config net architecture
+    pi_opt = torch.optim.Adam(policy.parameters(), lr=config['policy']['learning_rate'])
+    pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=config['policy']['learning_rate_decay'])
+
+    value = SetValue() # config net architecture
+    v_opt = torch.optim.Adam(value.parameters(), lr=config['value']['learning_rate'], weight_decay=config['value']['weight_decay'])
+
+    discriminator = DeepsetDiscriminator(
+        n_hidden_layers_element=config['discriminator']['n_hidden_layers_element'],
+        n_hidden_layers_global=config['discriminator']['n_hidden_layers_global'],
+        hidden_layer_size=config['discriminator']['hidden_layer_size'],
+        activation=activations[config['discriminator']['activation']],
+    )
+    disc_opt = torch.optim.Adam(discriminator.parameters(), lr=config['discriminator']['learning_rate'], weight_decay=config['discriminator']['weight_decay'])
+
+    if config['experiment'] == 'A': 
+        expert_data = torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt'))
+    elif config['experiment'] == 'B': 
+        expert_data = [
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')),
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track1.pt')),
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track2.pt')),
+            torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track3.pt')),
+        ]
+        d0 = [d[0] for d in expert_data]
+        d1 = [d[1] for d in expert_data]
+        d2 = [d[2] for d in expert_data]
+        d3 = [d[3] for d in expert_data]
+        expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3))
+    
+    expert_data = Buffer(*expert_data)
+
+    def callback(info):
+        tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'],
+                    disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'], 
+                    mean_episode_length=info['gen/mean_episode_length'],
+                    gen_collision_rate=info['gen/collision_rate'])
+        
+        # save model checkpoints
+        ep = info['epoch'] + 1
+        if (ep % 25 == 0):
+            torch.save(info['policy'].state_dict(), f'policy_epoch{ep}.pt')
+
+    value, policy = gail_ppo(
+        env_fn=env_fn,
+        expert_data=expert_data,
+        discriminator=discriminator,
+        disc_opt=disc_opt,
+        disc_iters=config['discriminator']['iterations_per_epoch'],
+        policy=policy,
+        value=value,
+        v_opt=v_opt,
+        v_iters=config['value']['iterations_per_epoch'],
+        epochs=config['train_epochs'],
+        rollout_episodes=60, 
+        rollout_steps=200,
+        gamma=0.99,
+        gae_lambda=0.9,
+        clip_ratio=config['policy']['clip_ratio'],
+        pi_opt=pi_opt,
+        pi_iters=config['policy']['iterations_per_epoch'],
+        logger=SummaryWriter(comment='gail-ppo-options-setobs2'),
+        callback=callback,
+        lr_schedulers=[pi_lr_scheduler],
+    )
+
+    # save model
+    torch.save(policy.state_dict(), 'policy_final.pt')
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--train', choices=['A', 'B'])
+    parser.add_argument('--epochs', type=int, default=200)
+    parser.add_argument('--test', type=str, help='path to config file to run final training on')
+    parser.add_argument('--test_seeds', type=int, default=5)
+    parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over')
+    args = parser.parse_args()
+
+    assert (args.train is None) ^ (args.test is None), 'Must either train on an experiment or test with a config file'
+
+    # if no test config specified, train
+    if args.test is None:
+        print('Running Tuning for Experiment %s'%(args.train))
+        analysis = tune.run(
+            training_function,
+            config={
+                'experiment': args.train,
+                'trainenv': {
+                    'stop_on_collision': False, 
+                },
+                'policy': {
+                    'learning_rate': 3e-4, 
+                    'learning_rate_decay': 1.0, 
+                    'clip_ratio': 0.2, 
+                    'iterations_per_epoch': 100,
+                    'hidden_layer_size': tune.grid_search([20, 40]),
+                    'n_hidden_layers': tune.grid_search([2, 3]), 
+                    'activation':0,
+                },
+                'value': {
+                    'learning_rate': 1e-4,
+                    'weight_decay': 1e-3, 
+                    'iterations_per_epoch': 1000, 
+                },
+                'discriminator': {
+                    'learning_rate': 1e-3, 
+                    'weight_decay': 1e-4, 
+                    'iterations_per_epoch': 100, 
+                    'n_hidden_layers_element': tune.grid_search([3,4]),
+                    'n_hidden_layers_global': tune.grid_search([1,2]),
+                    'hidden_layer_size': 10, 
+                    'activation': 0,
+                },
+                'train_epochs': args.epochs,
+                'seed': 0,
+            }
+        )
+        best_config = analysis.get_best_config(metric='gen_collision_rate', mode='min')
+        print('Best config: ', best_config)
+
+        # safe best_config
+        if not os.path.isdir(os.path.join(DIR, 'best_configs')):
+            os.mkdir(os.path.join(DIR, 'best_configs'))
+        
+        # save gail
+        with open(os.path.join(DIR, 'best_configs',f'gail_exp{args.train}.json'), 'w', encoding='utf-8') as f:
+            json.dump(best_config, f, ensure_ascii=False, indent=4)
+
+    # if config file specified, rerun it with appropriate number of seeds
+    else:
+        with open(args.test, 'rb') as f:
+            config = json.load(f)
+
+        print(f'Retraining {args.test} with {args.test_seeds} seeds on experiment {config["experiment"]}')
+
+        # rerun with appropriate number of seeds
+        rpt = {'cpu': int(args.test_cpus/args.test_seeds)} if (args.test_cpus is not None) else None
+        config['seed'] = tune.grid_search(list(range(1,args.test_seeds+1)))
+        analysis = tune.run(training_function, config=config, resources_per_trial=rpt)
+
+        # move final policies to appropriate directory
+        split_ = os.path.basename(args.test).split('_')
+        model = split_[0]
+        exper = split_[-1].split('.')[0]
+        savepath = os.path.join('test_policies',model,exper)
+        
+        if not os.path.isdir(savepath):
+            os.makedirs(savepath)
+        
+        import shutil
+        for i in range(args.test_seeds):
+            s = analysis._checkpoints[i]['config']['seed']
+            check_dir = analysis._checkpoints[i]['logdir']
+            shutil.copyfile(os.path.join(check_dir,'policy_final.pt'), 
+                os.path.join(savepath, f'policy_seed{s}.pt'))