Compare commits
1 Commits
idm_upgrad
...
smaller-co
| Author | SHA1 | Date | |
|---|---|---|---|
|
e1f2d58255 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,9 +1,7 @@
|
||||
*.png
|
||||
*.pkl
|
||||
*.pt
|
||||
*.zip
|
||||
**/ray/*
|
||||
**/runs/*
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
212
bc-experiment.py
212
bc-experiment.py
@@ -1,212 +0,0 @@
|
||||
# %%
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
from src.core.sampling import rollout
|
||||
from src.core.gail import gail_ppo, Buffer
|
||||
from src.core.value import SetValue
|
||||
from src.core.policy import SetPolicy
|
||||
from src.core.discriminator import DeepsetDiscriminator
|
||||
import torch
|
||||
|
||||
from intersim.envs import IntersimpleLidarFlatRandom
|
||||
from intersim.envs.intersimple import speed_reward
|
||||
import functools
|
||||
from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Setobs
|
||||
import numpy as np
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
from ray import tune
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
activations = [torch.nn.Tanh, torch.nn.LeakyReLU]
|
||||
|
||||
obs_min = np.array([
|
||||
[-1000, -1000, 0, -np.pi, -1e-1, 0.],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
]).reshape(-1)
|
||||
|
||||
obs_max = np.array([
|
||||
[1000, 1000, 20, np.pi, 1e-1, 0.],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
]).reshape(-1)
|
||||
|
||||
def training_function(config):
|
||||
np.random.seed(config['seed'])
|
||||
torch.manual_seed(config['seed'])
|
||||
|
||||
# choose validation environment
|
||||
if config['experiment'] == 'A':
|
||||
envs = [Setobs(TransformObservation(CollisionPenaltyWrapper(
|
||||
IntersimpleLidarFlatRandom(
|
||||
n_rays=5,
|
||||
reward=functools.partial(
|
||||
speed_reward,
|
||||
collision_penalty=0
|
||||
),
|
||||
check_collisions=True,
|
||||
stop_on_collision=config['trainenv']['stop_on_collision'],
|
||||
), collision_distance=6, collision_penalty=100),
|
||||
lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
|
||||
)) for _ in range(60)]
|
||||
elif config['experiment'] == 'B':
|
||||
envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper(
|
||||
IntersimpleLidarFlatRandom(
|
||||
n_rays=5,
|
||||
reward=functools.partial(
|
||||
speed_reward,
|
||||
collision_penalty=0
|
||||
),
|
||||
check_collisions=True,
|
||||
stop_on_collision=config['trainenv']['stop_on_collision'],
|
||||
), collision_distance=6, collision_penalty=100),
|
||||
lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
|
||||
)) for _ in range(15)] for track in range(4)],[])
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
env_fn = lambda i: envs[i]
|
||||
|
||||
# load expert data
|
||||
|
||||
if config['experiment'] == 'A':
|
||||
expert_data = torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt'))
|
||||
elif config['experiment'] == 'B':
|
||||
expert_data = [
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')),
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track1.pt')),
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track2.pt')),
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track3.pt')),
|
||||
]
|
||||
d0 = [d[0] for d in expert_data]
|
||||
d1 = [d[1] for d in expert_data]
|
||||
d2 = [d[2] for d in expert_data]
|
||||
d3 = [d[3] for d in expert_data]
|
||||
expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3))
|
||||
|
||||
expert_data = Buffer(*expert_data)
|
||||
|
||||
# configure and train policy
|
||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
policy = SetPolicy(expert_data.actions.shape[-1],
|
||||
n_hidden_layers=config['policy']['n_hidden_layers'],
|
||||
hidden_layer_size=config['policy']['hidden_layer_size'],
|
||||
activation=activations[config['policy']['activation']] ) # config net architecture
|
||||
policy = policy.to(device)
|
||||
|
||||
pi_opt = torch.optim.Adam(policy.parameters(), lr=config['policy']['learning_rate'])
|
||||
pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=config['policy']['learning_rate_decay'])
|
||||
|
||||
expert_states = expert_data.states[~expert_data.dones].to(device)
|
||||
expert_actions = expert_data.actions[~expert_data.dones].to(device)
|
||||
|
||||
for epoch in range(config['train_epochs']):
|
||||
pi_opt.zero_grad()
|
||||
loss = -policy.log_prob(policy(expert_states), expert_actions).mean()
|
||||
loss.backward()
|
||||
pi_opt.step()
|
||||
pi_lr_scheduler.step()
|
||||
|
||||
if epoch % 25 == 0:
|
||||
gen_states, gen_actions, gen_rewards, gen_dones, gen_collisions = rollout(env_fn, policy.cpu(), n_episodes=60, max_steps_per_episode=200)
|
||||
gen_mean_episode_length = (~gen_dones).sum() / gen_states.shape[0]
|
||||
gen_mean_reward_per_episode = gen_rewards[~gen_dones].sum() / gen_states.shape[0]
|
||||
gen_collision_rate = (1. * gen_collisions.any(-1)).mean()
|
||||
|
||||
tune.report(
|
||||
gen_mean_reward_per_episode=gen_mean_reward_per_episode.item(),
|
||||
mean_episode_length=gen_mean_episode_length.item(),
|
||||
gen_collision_rate=gen_collision_rate.item(),
|
||||
loss=loss.item(),
|
||||
)
|
||||
|
||||
# save model checkpoints
|
||||
ep = epoch + 1
|
||||
if (ep % 50 == 0):
|
||||
torch.save(policy.state_dict(), f'policy_epoch{ep}.pt')
|
||||
|
||||
# save model
|
||||
torch.save(policy.state_dict(), 'policy_final.pt')
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--train', choices=['A', 'B'])
|
||||
parser.add_argument('--epochs', type=int, default=1000)
|
||||
parser.add_argument('--test', type=str, help='path to config file to run final training on')
|
||||
parser.add_argument('--test_seeds', type=int, default=5)
|
||||
parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over')
|
||||
args = parser.parse_args()
|
||||
|
||||
assert (args.train is None) ^ (args.test is None), 'Must either train on an experiment or test with a config file'
|
||||
|
||||
# if no test config specified, train
|
||||
if args.test is None:
|
||||
print('Running Tuning for Experiment %s'%(args.train))
|
||||
analysis = tune.run(
|
||||
training_function,
|
||||
config={
|
||||
'experiment': args.train,
|
||||
'trainenv': {
|
||||
'stop_on_collision': False,
|
||||
},
|
||||
'policy': {
|
||||
'learning_rate': 3e-4,
|
||||
'learning_rate_decay': 1.0,
|
||||
'hidden_layer_size': tune.grid_search([20, 40]),
|
||||
'n_hidden_layers': tune.grid_search([2, 3]),
|
||||
'activation':0,
|
||||
},
|
||||
'train_epochs': args.epochs,
|
||||
'seed': 0,
|
||||
}
|
||||
# TODO resources_per_trial={'gpu': 1}
|
||||
)
|
||||
best_config = analysis.get_best_config(metric='gen_collision_rate', mode='min')
|
||||
print('Best config: ', best_config)
|
||||
|
||||
# safe best_config
|
||||
if not os.path.isdir(os.path.join(DIR, 'best_configs')):
|
||||
os.mkdir(os.path.join(DIR, 'best_configs'))
|
||||
|
||||
# save gail
|
||||
with open(os.path.join(DIR, 'best_configs',f'bc_exp{args.train}.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump(best_config, f, ensure_ascii=False, indent=4)
|
||||
|
||||
# if config file specified, rerun it with appropriate number of seeds
|
||||
else:
|
||||
with open(args.test, 'rb') as f:
|
||||
config = json.load(f)
|
||||
|
||||
print(f'Retraining {args.test} with {args.test_seeds} seeds on experiment {config["experiment"]}')
|
||||
|
||||
# rerun with appropriate number of seeds
|
||||
rpt = {'cpu': int(args.test_cpus/args.test_seeds)} if (args.test_cpus is not None) else None
|
||||
config['seed'] = tune.grid_search(list(range(1,args.test_seeds+1)))
|
||||
analysis = tune.run(training_function, config=config, resources_per_trial=rpt)
|
||||
|
||||
# move final policies to appropriate directory
|
||||
split_ = os.path.basename(args.test).split('_')
|
||||
model = split_[0]
|
||||
exper = split_[-1].split('.')[0]
|
||||
savepath = os.path.join('test_policies',model,exper)
|
||||
|
||||
if not os.path.isdir(savepath):
|
||||
os.makedirs(savepath)
|
||||
|
||||
import shutil
|
||||
for i in range(args.test_seeds):
|
||||
s = analysis._checkpoints[i]['config']['seed']
|
||||
check_dir = analysis._checkpoints[i]['logdir']
|
||||
shutil.copyfile(os.path.join(check_dir,'policy_final.pt'),
|
||||
os.path.join(savepath, f'policy_seed{s}.pt'))
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 300,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 300,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"delta": 0.01,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.0001,
|
||||
"weight_decay": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 1,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"delta": 0.01,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.0001,
|
||||
"weight_decay": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 1,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.0001,
|
||||
"weight_decay": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 1,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.0001,
|
||||
"weight_decay": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 1,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 10,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 10,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 20,
|
||||
"n_hidden_layers": 4,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 20,
|
||||
"n_hidden_layers": 4,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 1,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 90,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": null,
|
||||
"abort_unsafe_collision_method": null
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 20,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 85,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 10,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 10,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 3,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 20,
|
||||
"n_hidden_layers": 4,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 20,
|
||||
"n_hidden_layers": 4,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 3,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 100,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "A",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 40,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 1,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 90,
|
||||
"seed": 0
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"experiment": "B",
|
||||
"trainenv": {
|
||||
"stop_on_collision": false,
|
||||
"safe_actions_collision_method": "circle",
|
||||
"abort_unsafe_collision_method": "circle"
|
||||
},
|
||||
"policy": {
|
||||
"learning_rate": 0.0003,
|
||||
"learning_rate_decay": 1.0,
|
||||
"clip_ratio": 0.2,
|
||||
"iterations_per_epoch": 100,
|
||||
"hidden_layer_size": 20,
|
||||
"n_hidden_layers": 2,
|
||||
"activation": 0,
|
||||
"option": 0
|
||||
},
|
||||
"value": {
|
||||
"learning_rate": 0.001,
|
||||
"iterations_per_epoch": 1000
|
||||
},
|
||||
"discriminator": {
|
||||
"learning_rate": 0.001,
|
||||
"weight_decay": 0.0001,
|
||||
"iterations_per_epoch": 100,
|
||||
"n_hidden_layers_element": 4,
|
||||
"n_hidden_layers_global": 2,
|
||||
"hidden_layer_size": 10,
|
||||
"activation": 0
|
||||
},
|
||||
"train_epochs": 85,
|
||||
"seed": 0
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,83 +0,0 @@
|
||||
import os
|
||||
from src.eval_main import eval_main
|
||||
from src.evaluation.utils import load_and_average
|
||||
|
||||
def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=False):
|
||||
|
||||
policy_kwargs = {}
|
||||
if method in ['expert', 'idm']:
|
||||
env, env_kwargs ='NRasterizedRouteIncrementingAgent', {}
|
||||
elif method in ['bc','gail']:
|
||||
env='NormalizedContinuousEvalEnv'
|
||||
env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
|
||||
elif method in ['hail']:
|
||||
env = 'NormalizedOptionsEvalEnv'
|
||||
env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None}
|
||||
elif method in ['shail']:
|
||||
env = 'NormalizedSafeOptionsEvalEnv'
|
||||
env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
files = ['']
|
||||
|
||||
if folder is not None:
|
||||
files = [os.path.join(folder, f) for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
|
||||
print('%i folders found in %s folder' %(len(files), folder))
|
||||
|
||||
if not skip_running:
|
||||
for policy_file in files:
|
||||
# run metrics on that file
|
||||
outbase = eval_main(locations=locations,
|
||||
method=method,
|
||||
policy_file=policy_file,
|
||||
policy_kwargs=policy_kwargs,
|
||||
env=env,
|
||||
env_kwargs=env_kwargs)
|
||||
outfolder = os.path.dirname(outbase)
|
||||
else:
|
||||
locstr = 'loc_'+'_'.join([f'r{ro}t{tr}' for (ro,tr) in locations])
|
||||
if folder is None:
|
||||
outfolder = os.path.join('out',method,locstr)
|
||||
else:
|
||||
path_items = folder.split('/')
|
||||
outfolder = os.path.join('out', '/'.join(path_items[1:]), locstr)
|
||||
|
||||
# load metrics from save_path
|
||||
average_metrics = load_and_average(outfolder)
|
||||
if method in ['expert', 'idm']:
|
||||
latex_print(average_metrics, light=True)
|
||||
else:
|
||||
latex_print(average_metrics)
|
||||
|
||||
def latex_print(am, light=False):
|
||||
"""
|
||||
print latex line
|
||||
|
||||
am (Dict[str,tuple]): dict mapping metric_name to (mean, std)
|
||||
"""
|
||||
|
||||
print('success rate, distance travelled, RWSE_10, |DeltaV|, AccelJSD')
|
||||
if light:
|
||||
if 'rwse_10s' in am.keys():
|
||||
print("%2.1f& %2.1f & %1.2f & %2.1f& "
|
||||
"%0.3f \\\\" %( 100*am['success rate'][0], am['mean travel distance'][0], am['rwse_10s'][0],
|
||||
am['average absolute average velocity'][0],am['acceleration distribution divergence'][0] ))
|
||||
return
|
||||
|
||||
|
||||
print("%2.1f& %2.1f & $---$ & $---$ & "
|
||||
"$---$ \\\\" %( 100*am['success rate'][0], am['mean travel distance'][0]))
|
||||
return
|
||||
|
||||
print("%2.1f \\scriptstyle\\pm %2.1f & %2.1f \\scriptstyle\\pm %2.1f & "
|
||||
"%1.2f \\scriptstyle\\pm %1.2f & %2.1f \\scriptstyle\\pm %1.1f & "
|
||||
"%0.3f \\scriptstyle\\pm %0.3f \\\\" %( 100*am['success rate'][0], 100*am['success rate'][1],
|
||||
am['mean travel distance'][0] , am['mean travel distance'][1] ,
|
||||
am['rwse_10s'][0] , am['rwse_10s'][1] ,
|
||||
am['average absolute average velocity'][0] , am['average absolute average velocity'][1] ,
|
||||
am['acceleration distribution divergence'][0] , am['acceleration distribution divergence'][1] ))
|
||||
|
||||
if __name__=='__main__':
|
||||
import fire
|
||||
fire.Fire(main)
|
||||
@@ -1,19 +0,0 @@
|
||||
# can add --skip_running if you've run the runs before on the saved policies
|
||||
|
||||
python -m eval_experiments
|
||||
python -m eval_experiments --locations='[(0,4)]'
|
||||
python -m eval_experiments --method idm
|
||||
python -m eval_experiments --method idm --locations='[(0,4)]'
|
||||
python -m eval_experiments --method bc --folder='test_policies/bc/expA'
|
||||
python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]'
|
||||
python -m eval_experiments --method gail --folder='test_policies/gail/expA'
|
||||
python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]'
|
||||
python -m eval_experiments --method hail --folder='test_policies/hail/expA'
|
||||
python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]'
|
||||
python -m eval_experiments --method shail --folder='test_policies/shail/expA'
|
||||
python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]'
|
||||
|
||||
python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expA'
|
||||
python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expB' --locations='[(0,4)]'
|
||||
python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expA'
|
||||
python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expB' --locations='[(0,4)]'
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,238 +0,0 @@
|
||||
# %%
|
||||
import os
|
||||
|
||||
import gym
|
||||
from src.core.gail import gail_ppo, Buffer
|
||||
from src.core.value import SetValue
|
||||
from src.core.policy import SetPolicy
|
||||
from src.core.discriminator import DeepsetDiscriminator
|
||||
import torch
|
||||
|
||||
from intersim.envs import IntersimpleLidarFlatRandom
|
||||
from intersim.envs.intersimple import speed_reward
|
||||
import functools
|
||||
from src.util.wrappers import CollisionPenaltyWrapper, TransformObservation, Setobs
|
||||
import numpy as np
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
from ray import tune
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
activations = [torch.nn.Tanh, torch.nn.LeakyReLU]
|
||||
|
||||
obs_min = np.array([
|
||||
[-1000, -1000, 0, -np.pi, -1e-1, 0.],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
[0, -np.pi, -20, -20, -np.pi, -1e-1],
|
||||
]).reshape(-1)
|
||||
|
||||
obs_max = np.array([
|
||||
[1000, 1000, 20, np.pi, 1e-1, 0.],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
[50, np.pi, 20, 20, np.pi, 1e-1],
|
||||
]).reshape(-1)
|
||||
|
||||
def training_function(config):
|
||||
np.random.seed(config['seed'])
|
||||
torch.manual_seed(config['seed'])
|
||||
|
||||
if config['experiment'] == 'A':
|
||||
envs = [Setobs(TransformObservation(CollisionPenaltyWrapper(
|
||||
IntersimpleLidarFlatRandom(
|
||||
n_rays=5,
|
||||
reward=functools.partial(
|
||||
speed_reward,
|
||||
collision_penalty=0
|
||||
),
|
||||
check_collisions=True,
|
||||
stop_on_collision=config['trainenv']['stop_on_collision'],
|
||||
), collision_distance=6, collision_penalty=100),
|
||||
lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
|
||||
)) for _ in range(60)]
|
||||
|
||||
elif config['experiment'] == 'B':
|
||||
envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper(
|
||||
IntersimpleLidarFlatRandom(
|
||||
n_rays=5,
|
||||
reward=functools.partial(
|
||||
speed_reward,
|
||||
collision_penalty=0
|
||||
),
|
||||
check_collisions=True,
|
||||
stop_on_collision=config['trainenv']['stop_on_collision'],
|
||||
track=track,
|
||||
), collision_distance=6, collision_penalty=100),
|
||||
lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
|
||||
)) for _ in range(15)] for track in range(4)],[])
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
env_fn = lambda i: envs[i]
|
||||
|
||||
policy = SetPolicy(env_fn(0).action_space.shape[0],
|
||||
n_hidden_layers=config['policy']['n_hidden_layers'],
|
||||
hidden_layer_size=config['policy']['hidden_layer_size'],
|
||||
activation=activations[config['policy']['activation']] ) # config net architecture
|
||||
pi_opt = torch.optim.Adam(policy.parameters(), lr=config['policy']['learning_rate'])
|
||||
pi_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(pi_opt, gamma=config['policy']['learning_rate_decay'])
|
||||
|
||||
value = SetValue() # config net architecture
|
||||
v_opt = torch.optim.Adam(value.parameters(), lr=config['value']['learning_rate'], weight_decay=config['value']['weight_decay'])
|
||||
|
||||
discriminator = DeepsetDiscriminator(
|
||||
n_hidden_layers_element=config['discriminator']['n_hidden_layers_element'],
|
||||
n_hidden_layers_global=config['discriminator']['n_hidden_layers_global'],
|
||||
hidden_layer_size=config['discriminator']['hidden_layer_size'],
|
||||
activation=activations[config['discriminator']['activation']],
|
||||
)
|
||||
disc_opt = torch.optim.Adam(discriminator.parameters(), lr=config['discriminator']['learning_rate'], weight_decay=config['discriminator']['weight_decay'])
|
||||
|
||||
if config['experiment'] == 'A':
|
||||
expert_data = torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt'))
|
||||
elif config['experiment'] == 'B':
|
||||
expert_data = [
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track0.pt')),
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track1.pt')),
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track2.pt')),
|
||||
torch.load(os.path.join(DIR, 'intersimple-expert-data-setobs2-loc0-track3.pt')),
|
||||
]
|
||||
d0 = [d[0] for d in expert_data]
|
||||
d1 = [d[1] for d in expert_data]
|
||||
d2 = [d[2] for d in expert_data]
|
||||
d3 = [d[3] for d in expert_data]
|
||||
expert_data = (torch.cat(d0), torch.cat(d1), torch.cat(d2), torch.cat(d3))
|
||||
|
||||
expert_data = Buffer(*expert_data)
|
||||
|
||||
def callback(info):
|
||||
tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'],
|
||||
disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'],
|
||||
mean_episode_length=info['gen/mean_episode_length'],
|
||||
gen_collision_rate=info['gen/collision_rate'])
|
||||
|
||||
# save model checkpoints
|
||||
ep = info['epoch'] + 1
|
||||
if (ep % 25 == 0):
|
||||
torch.save(info['policy'].state_dict(), f'policy_epoch{ep}.pt')
|
||||
|
||||
value, policy = gail_ppo(
|
||||
env_fn=env_fn,
|
||||
expert_data=expert_data,
|
||||
discriminator=discriminator,
|
||||
disc_opt=disc_opt,
|
||||
disc_iters=config['discriminator']['iterations_per_epoch'],
|
||||
policy=policy,
|
||||
value=value,
|
||||
v_opt=v_opt,
|
||||
v_iters=config['value']['iterations_per_epoch'],
|
||||
epochs=config['train_epochs'],
|
||||
rollout_episodes=60,
|
||||
rollout_steps=200,
|
||||
gamma=0.99,
|
||||
gae_lambda=0.9,
|
||||
clip_ratio=config['policy']['clip_ratio'],
|
||||
pi_opt=pi_opt,
|
||||
pi_iters=config['policy']['iterations_per_epoch'],
|
||||
logger=SummaryWriter(comment='gail-ppo-options-setobs2'),
|
||||
callback=callback,
|
||||
lr_schedulers=[pi_lr_scheduler],
|
||||
)
|
||||
|
||||
# save model
|
||||
torch.save(policy.state_dict(), 'policy_final.pt')
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--train', choices=['A', 'B'])
|
||||
parser.add_argument('--epochs', type=int, default=200)
|
||||
parser.add_argument('--test', type=str, help='path to config file to run final training on')
|
||||
parser.add_argument('--test_seeds', type=int, default=5)
|
||||
parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over')
|
||||
args = parser.parse_args()
|
||||
|
||||
assert (args.train is None) ^ (args.test is None), 'Must either train on an experiment or test with a config file'
|
||||
|
||||
# if no test config specified, train
|
||||
if args.test is None:
|
||||
print('Running Tuning for Experiment %s'%(args.train))
|
||||
analysis = tune.run(
|
||||
training_function,
|
||||
config={
|
||||
'experiment': args.train,
|
||||
'trainenv': {
|
||||
'stop_on_collision': False,
|
||||
},
|
||||
'policy': {
|
||||
'learning_rate': 3e-4,
|
||||
'learning_rate_decay': 1.0,
|
||||
'clip_ratio': 0.2,
|
||||
'iterations_per_epoch': 100,
|
||||
'hidden_layer_size': tune.grid_search([20, 40]),
|
||||
'n_hidden_layers': tune.grid_search([2, 3]),
|
||||
'activation':0,
|
||||
},
|
||||
'value': {
|
||||
'learning_rate': 1e-4,
|
||||
'weight_decay': 1e-3,
|
||||
'iterations_per_epoch': 1000,
|
||||
},
|
||||
'discriminator': {
|
||||
'learning_rate': 1e-3,
|
||||
'weight_decay': 1e-4,
|
||||
'iterations_per_epoch': 100,
|
||||
'n_hidden_layers_element': tune.grid_search([3,4]),
|
||||
'n_hidden_layers_global': tune.grid_search([1,2]),
|
||||
'hidden_layer_size': 10,
|
||||
'activation': 0,
|
||||
},
|
||||
'train_epochs': args.epochs,
|
||||
'seed': 0,
|
||||
}
|
||||
)
|
||||
best_config = analysis.get_best_config(metric='gen_collision_rate', mode='min')
|
||||
print('Best config: ', best_config)
|
||||
|
||||
# safe best_config
|
||||
if not os.path.isdir(os.path.join(DIR, 'best_configs')):
|
||||
os.mkdir(os.path.join(DIR, 'best_configs'))
|
||||
|
||||
# save gail
|
||||
with open(os.path.join(DIR, 'best_configs',f'gail_exp{args.train}.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump(best_config, f, ensure_ascii=False, indent=4)
|
||||
|
||||
# if config file specified, rerun it with appropriate number of seeds
|
||||
else:
|
||||
with open(args.test, 'rb') as f:
|
||||
config = json.load(f)
|
||||
|
||||
print(f'Retraining {args.test} with {args.test_seeds} seeds on experiment {config["experiment"]}')
|
||||
|
||||
# rerun with appropriate number of seeds
|
||||
rpt = {'cpu': int(args.test_cpus/args.test_seeds)} if (args.test_cpus is not None) else None
|
||||
config['seed'] = tune.grid_search(list(range(1,args.test_seeds+1)))
|
||||
analysis = tune.run(training_function, config=config, resources_per_trial=rpt)
|
||||
|
||||
# move final policies to appropriate directory
|
||||
split_ = os.path.basename(args.test).split('_')
|
||||
model = split_[0]
|
||||
exper = split_[-1].split('.')[0]
|
||||
savepath = os.path.join('test_policies',model,exper)
|
||||
|
||||
if not os.path.isdir(savepath):
|
||||
os.makedirs(savepath)
|
||||
|
||||
import shutil
|
||||
for i in range(args.test_seeds):
|
||||
s = analysis._checkpoints[i]['config']['seed']
|
||||
check_dir = analysis._checkpoints[i]['logdir']
|
||||
shutil.copyfile(os.path.join(check_dir,'policy_final.pt'),
|
||||
os.path.join(savepath, f'policy_seed{s}.pt'))
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user