huangfu/InteractionImitation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

making option list indexable to visualize in tensorboard. noticing run from last night had much better performance under more long options, unclear if due to choice or environment factors, like episodes lasting longer. making stoponcollision and seed parameters. fixing ability to see reward metrics under ray/tune/, problem was they were being returned as tensors, not floats

Browse Source
This commit is contained in:
Arec Jamgochian
2022-02-25 16:36:14 -08:00
parent d1f23e6d25
commit 59083ebce3
2 changed files with 69 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
sgail-ppo-options-setobs2.py
View File

@@ -6,8 +6,8 @@ from src.safe_options.options import gail_ppo, Buffer
from src.core.value import SetValue
from src.safe_options.policy import SetMaskedDiscretePolicy
from src.core.discriminator import DeepsetDiscriminator
import torch.optim
import torch.nn
import torch
from intersim.envs import IntersimpleLidarFlatRandom
from intersim.envs.intersimple import speed_reward
import functools
@@ -18,25 +18,35 @@ from torch.utils.tensorboard import SummaryWriter
from ray import tune
DIR = os.path.dirname(os.path.abspath(__file__))
option_list = [[(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5]],
[(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20]],
[(vel, time) for vel in [0, 1, 2, 4, 6, 8, 10] for time in [5, 10, 20]], # was the best in training with single hidden layer, but very slow
[(vel, time) for vel in [0, 1, 2, 5, 7.5, 10] for time in [5, 20, 40]],
[(vel, time) for vel in [0, 2, 5, 10] for time in [5, 10, 20]],
[(vel, time) for vel in [0, 3, 10] for time in [5, 20, 40]]
]
obs_min = np.array([
[-1000, -1000, 0, -np.pi, -1e-1, 0.],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
]).reshape(-1)
obs_max = np.array([
[1000, 1000, 20, np.pi, 1e-1, 0.],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
]).reshape(-1)
def training_function(config):
obs_min = np.array([
[-1000, -1000, 0, -np.pi, -1e-1, 0.],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
[0, -np.pi, -20, -20, -np.pi, -1e-1],
]).reshape(-1)
obs_max = np.array([
[1000, 1000, 20, np.pi, 1e-1, 0.],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
[50, np.pi, 20, 20, np.pi, 1e-1],
]).reshape(-1)
np.random.seed(config['seed'])
torch.manual_seed(config['seed'])
envs = sum([[SafeOptionsEnv(Setobs(
TransformObservation(CollisionPenaltyWrapper(IntersimpleLidarFlatRandom(
@@ -45,9 +55,9 @@ def training_function(config):
speed_reward,
collision_penalty=0
),
stop_on_collision=False, track=track,
stop_on_collision=config['stop_on_collision'], track=track,
), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10))
), options=config['policy']['option'], safe_actions_collision_method='circle',
), options=option_list[config['policy']['option']], safe_actions_collision_method='circle',
abort_unsafe_collision_method='circle') for _ in range(20)] for track in range(4)],[])
env_fn = lambda i: envs[i]
@@ -79,7 +89,9 @@ def training_function(config):
expert_data = Buffer(*expert_data)
def callback(info):
tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'])
tune.report(gen_mean_reward_per_episode=info['gen/mean_reward_per_episode'],
disc_mean_reward_per_episode=info['disc/mean_reward_per_episode'],
mean_episode_length=info['gen/mean_episode_length'])
value, policy = gail_ppo(
env_fn=env_fn,
@@ -91,9 +103,9 @@ def training_function(config):
value=value,
v_opt=v_opt,
v_iters=config['value']['iterations_per_epoch'],
epochs=200,
rollout_episodes=60,
rollout_steps=60,
epochs=2, # 200 FIXME
rollout_episodes=6, #60, FIXME
rollout_steps=6, #60, FIXME
gamma=0.99,
gae_lambda=0.9,
clip_ratio=config['policy']['clip_ratio'],
@@ -104,33 +116,33 @@ def training_function(config):
lr_schedulers=[pi_lr_scheduler],
)
# save value, policy
analysis = tune.run(
training_function,
config={
'stop_on_collision': tune.grid_search([True, False]),
'policy': {
'learning_rate': tune.grid_search([3e-4]),
'learning_rate_decay': tune.grid_search([1.0]),
'clip_ratio': tune.grid_search([0.2]),
'iterations_per_epoch': tune.grid_search([100]),
'hidden_layer_size': tune.grid_search([10, 20, 30]),
'n_hidden_layers': tune.grid_search([1, 2, 3]),
'learning_rate': 3e-4, # tune.grid_search([3e-4]),
'learning_rate_decay': 1.0, #tune.grid_search([1.0]),
'clip_ratio': 0.2, #tune.grid_search([0.2]),
'iterations_per_epoch': 100, #tune.grid_search([100]),
'hidden_layer_size': tune.grid_search([10, 20, 40]),
'n_hidden_layers': 1, #tune.grid_search([1, 2, 3]), #FIXME
'activation':tune.grid_search([torch.nn.LeakyReLU, torch.nn.Tanh]),
'option': tune.grid_search([[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5)],
[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5),
(0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10)],
[(0, 5), (1, 5), (2, 5), (4, 5), (6, 5), (8, 5), (10, 5),
(0, 10), (1, 10), (2, 10), (4, 10), (6, 10), (8, 10), (10, 10),
(0, 20), (1, 20), (2, 20), (4, 20), (6, 20), (8, 20), (20, 10)]])
'option': tune.grid_search(list(range(len(option_list))))
},
'value': {
'learning_rate': tune.grid_search([1e-3]),
'iterations_per_epoch': tune.grid_search([1000]),
'learning_rate': 1e-3, # tune.grid_search([1e-3]),
'iterations_per_epoch': 1000, #tune.grid_search([1000]),
},
'discriminator': {
'learning_rate': tune.grid_search([1e-3]),
'weight_decay': tune.grid_search([1e-4]),
'iterations_per_epoch': tune.grid_search([100]),
}
'learning_rate': 1e-3, #tune.grid_search([1e-3]),
'weight_decay': 1e-4, #tune.grid_search([1e-4]),
'iterations_per_epoch': 100, #tune.grid_search([100]),
},
'seed': 0,
}
)