huangfu/InteractionImitation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

making option list indexable to visualize in tensorboard. noticing run from last night had much better performance under more long options, unclear if due to choice or environment factors, like episodes lasting longer. making stoponcollision and seed parameters. fixing ability to see reward metrics under ray/tune/, problem was they were being returned as tensors, not floats

Browse Source
This commit is contained in:
Arec Jamgochian
2022-02-25 16:36:14 -08:00
parent d1f23e6d25
commit 59083ebce3
2 changed files with 69 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/safe_options/options.py
View File

@@ -52,7 +52,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value
generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions)
logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch)
gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0]
logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length , epoch)
gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0]
logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch)
logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch)
@@ -63,7 +64,8 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value
else:
generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions))
logger.add_scalar('disc/final_loss', loss, epoch)
logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch)
disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0]
logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode , epoch)
#assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape
generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1)
@@ -76,7 +78,9 @@ def gail(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, value
'epoch': epoch,
'value': value,
'policy': policy,
'gen/mean_reward_per_episode': gen_mean_reward_per_episode,
'gen/mean_episode_length': gen_mean_episode_length.item(),
'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(),
'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(),
})
return value, policy
@@ -93,8 +97,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v
generator_data = OptionsRollout(HLBuffer(*hl_data), Buffer(*ll_data))
generator_data.ll.actions += 0.1 * torch.randn_like(generator_data.ll.actions)
logger.add_scalar('gen/mean_episode_length', (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0], epoch)
gen_mean_episode_length = (~generator_data.ll.dones).sum() / generator_data.ll.states.shape[0]
logger.add_scalar('gen/mean_episode_length', gen_mean_episode_length, epoch)
gen_mean_reward_per_episode = generator_data.hl.rewards[~generator_data.hl.dones].sum() / generator_data.hl.states.shape[0]
logger.add_scalar('gen/mean_reward_per_episode', gen_mean_reward_per_episode, epoch)
logger.add_scalar('gen/unsafe_probability_mass', policy.unsafe_probability_mass(policy(generator_data.hl.states[~generator_data.hl.dones], generator_data.hl.safe_actions[~generator_data.hl.dones])).mean(), epoch)
@@ -105,7 +109,8 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v
else:
generator_data.ll.rewards = -F.logsigmoid(discriminator(generator_data.ll.states, generator_data.ll.actions))
logger.add_scalar('disc/final_loss', loss, epoch)
logger.add_scalar('disc/mean_reward_per_episode', generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0], epoch)
disc_mean_reward_per_episode = generator_data.ll.rewards[~generator_data.ll.dones].sum() / generator_data.ll.states.shape[0]
logger.add_scalar('disc/mean_reward_per_episode', disc_mean_reward_per_episode, epoch)
#assert generator_data.ll.rewards.shape == generator_data.ll.dones.shape
generator_data.hl.rewards = torch.where(~generator_data.ll.dones, generator_data.ll.rewards, torch.tensor(0.)).sum(-1)
@@ -118,7 +123,9 @@ def gail_ppo(env_fn, expert_data, discriminator, disc_opt, disc_iters, policy, v
'epoch': epoch,
'value': value,
'policy': policy,
'gen/mean_reward_per_episode': gen_mean_reward_per_episode,
'gen/mean_episode_length': gen_mean_episode_length.item(),
'gen/mean_reward_per_episode': gen_mean_reward_per_episode.item(),
'disc/mean_reward_per_episode': disc_mean_reward_per_episode.item(),
})
for lr_scheduler in lr_schedulers: