diff --git a/bc-experiment.py b/bc-experiment.py index 9188bb5..0d7dc5b 100644 --- a/bc-experiment.py +++ b/bc-experiment.py @@ -44,18 +44,36 @@ def training_function(config): np.random.seed(config['seed']) torch.manual_seed(config['seed']) - envs = [Setobs(TransformObservation(CollisionPenaltyWrapper( - IntersimpleLidarFlatRandom( - n_rays=5, - reward=functools.partial( - speed_reward, - collision_penalty=0 - ), - check_collisions=True, - stop_on_collision=config['trainenv']['stop_on_collision'], - ), collision_distance=6, collision_penalty=100), - lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) - )) for _ in range(60)] + # choose validation environment + if config['experiment'] == 'A': + envs = [Setobs(TransformObservation(CollisionPenaltyWrapper( + IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], + ), collision_distance=6, collision_penalty=100), + lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) + )) for _ in range(60)] + elif config['experiment'] == 'B': + envs = sum([[Setobs(TransformObservation(CollisionPenaltyWrapper( + IntersimpleLidarFlatRandom( + n_rays=5, + reward=functools.partial( + speed_reward, + collision_penalty=0 + ), + check_collisions=True, + stop_on_collision=config['trainenv']['stop_on_collision'], + ), collision_distance=6, collision_penalty=100), + lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10) + )) for _ in range(15)] for track in range(4)],[]) + else: + raise NotImplementedError + env_fn = lambda i: envs[i] # load expert data @@ -99,7 +117,7 @@ def training_function(config): pi_opt.step() pi_lr_scheduler.step() - if epoch % 100 == 0: + if epoch % 10 == 0: gen_states, gen_actions, gen_rewards, gen_dones, gen_collisions = rollout(env_fn, policy.cpu(), n_episodes=60, max_steps_per_episode=200) gen_mean_episode_length = (~gen_dones).sum() / gen_states.shape[0] gen_mean_reward_per_episode = gen_rewards[~gen_dones].sum() / gen_states.shape[0] @@ -114,7 +132,7 @@ def training_function(config): # save model checkpoints ep = epoch + 1 - if (ep % 25 == 0): + if (ep % 50 == 0): torch.save(policy.state_dict(), f'policy_epoch{ep}.pt') # save model @@ -124,7 +142,7 @@ if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('--train', choices=['A', 'B']) - parser.add_argument('--epochs', type=int, default=200) + parser.add_argument('--epochs', type=int, default=1000) parser.add_argument('--test', type=str, help='path to config file to run final training on') parser.add_argument('--test_seeds', type=int, default=5) parser.add_argument('--test_cpus', type=int, help='number of cpus available to split test seed training over') diff --git a/best_configs/bc_expA.json b/best_configs/bc_expA.json new file mode 100644 index 0000000..8581472 --- /dev/null +++ b/best_configs/bc_expA.json @@ -0,0 +1,15 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "train_epochs": 1000, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/bc_expB.json b/best_configs/bc_expB.json new file mode 100644 index 0000000..7b55224 --- /dev/null +++ b/best_configs/bc_expB.json @@ -0,0 +1,15 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "train_epochs": 1000, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail-trpo_expA.json b/best_configs/gail-trpo_expA.json new file mode 100644 index 0000000..ff708f1 --- /dev/null +++ b/best_configs/gail-trpo_expA.json @@ -0,0 +1,31 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "delta": 0.01, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail-trpo_expB.json b/best_configs/gail-trpo_expB.json new file mode 100644 index 0000000..ae0fa67 --- /dev/null +++ b/best_configs/gail-trpo_expB.json @@ -0,0 +1,31 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "delta": 0.01, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail_expA.json b/best_configs/gail_expA.json new file mode 100644 index 0000000..e254f61 --- /dev/null +++ b/best_configs/gail_expA.json @@ -0,0 +1,31 @@ +{ + "experiment": "A", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/best_configs/gail_expB.json b/best_configs/gail_expB.json new file mode 100644 index 0000000..b05e446 --- /dev/null +++ b/best_configs/gail_expB.json @@ -0,0 +1,31 @@ +{ + "experiment": "B", + "trainenv": { + "stop_on_collision": false + }, + "policy": { + "learning_rate": 0.0003, + "learning_rate_decay": 1.0, + "clip_ratio": 0.2, + "iterations_per_epoch": 100, + "hidden_layer_size": 40, + "n_hidden_layers": 2, + "activation": 0 + }, + "value": { + "learning_rate": 0.0001, + "weight_decay": 0.001, + "iterations_per_epoch": 1000 + }, + "discriminator": { + "learning_rate": 0.001, + "weight_decay": 0.0001, + "iterations_per_epoch": 100, + "n_hidden_layers_element": 4, + "n_hidden_layers_global": 1, + "hidden_layer_size": 10, + "activation": 0 + }, + "train_epochs": 100, + "seed": 0 +} \ No newline at end of file diff --git a/gail-experiment.py b/gail-experiment.py index 379e397..45084cd 100644 --- a/gail-experiment.py +++ b/gail-experiment.py @@ -187,8 +187,8 @@ if __name__ == '__main__': }, 'discriminator': { 'learning_rate': 1e-3, - 'weight_decay': 1e-5, - 'iterations_per_epoch': 500, + 'weight_decay': 1e-4, # 1e-5 Fixme + 'iterations_per_epoch': 100, # 500 Fixme 'n_hidden_layers_element': tune.grid_search([3,4]), 'n_hidden_layers_global': tune.grid_search([1,2]), 'hidden_layer_size': 10, diff --git a/gail-trpo-experiment.py b/gail-trpo-experiment.py index a2b8b1f..ff96929 100644 --- a/gail-trpo-experiment.py +++ b/gail-trpo-experiment.py @@ -184,8 +184,8 @@ if __name__ == '__main__': }, 'discriminator': { 'learning_rate': 1e-3, - 'weight_decay': 1e-5, - 'iterations_per_epoch': 500, + 'weight_decay': 1e-4, # 1e-5 Fixme + 'iterations_per_epoch': 100, # 500 Fixme 'n_hidden_layers_element': tune.grid_search([3,4]), 'n_hidden_layers_global': tune.grid_search([1,2]), 'hidden_layer_size': 10,