diff --git a/eval_experiments.py b/eval_experiments.py index 00e42d0..37af327 100644 --- a/eval_experiments.py +++ b/eval_experiments.py @@ -9,13 +9,13 @@ def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=F env, env_kwargs ='NRasterizedRouteIncrementingAgent', {} elif method in ['bc','gail']: env='NormalizedContinuousEvalEnv' - env_kwargs={stop_on_collision:True, max_episode_steps:1000} + env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000} elif method in ['hail']: env = 'NormalizedOptionsEvalEnv' - env_kwargs={stop_on_collision:True, max_episode_steps:1000} + env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000} elif method in ['shail']: env = 'NormalizedSafeOptionsEvalEnv' - env_kwargs={stop_on_collision:True, max_episode_steps:1000} + env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000} else: raise NotImplementedError @@ -39,8 +39,6 @@ def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=F locstr = 'loc_'+'_'.join([f'r{ro}t{tr}' for (ro,tr) in locations]) outfolder = os.path.join('out',method,locstr) - import pdb - pdb.set_trace() # load metrics from save_path average_metrics = load_and_average(outfolder) if method in ['expert', 'idm']: diff --git a/evaluate_models.sh b/evaluate_models.sh index 6461cf2..2f32c5a 100755 --- a/evaluate_models.sh +++ b/evaluate_models.sh @@ -1,14 +1,14 @@ # can add --skip_running if you've run before python -m eval_experiments -python -m eval_experiments --locations='[(0,4)]' -python -m eval_experiments --method idm -python -m eval_experiments --method idm --locations='[(0,4)]' -python -m eval_experiments --method bc --folder='test_policies/bc/expA' -python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]' -python -m eval_experiments --method gail --folder='test_policies/gail/expA' -python -m eval_experiments --method gail --folder='test_policies/gail/expB'--locations='[(0,4)]' -python -m eval_experiments --method hail -python -m eval_experiments --method hail --locations='[(0,4)]' -python -m eval_experiments --method shail -python -m eval_experiments --method shail --locations='[(0,4)]' \ No newline at end of file +python -m eval_experiments --locations='[(0,4)]' +python -m eval_experiments --method idm +python -m eval_experiments --method idm --locations='[(0,4)]' +python -m eval_experiments --method bc --folder='test_policies/bc/expA' +python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]' +python -m eval_experiments --method gail --folder='test_policies/gail/expA' +python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]' +python -m eval_experiments --method hail --folder='test_policies/hail/expA' +python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]' +python -m eval_experiments --method shail --folder='test_policies/shail/expA' +python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]' \ No newline at end of file diff --git a/test_policies/bc/expA/policy_seed1.pt b/test_policies/bc/expA/policy_seed1.pt new file mode 100644 index 0000000..a23bb84 Binary files /dev/null and b/test_policies/bc/expA/policy_seed1.pt differ diff --git a/test_policies/bc/expA/policy_seed2.pt b/test_policies/bc/expA/policy_seed2.pt new file mode 100644 index 0000000..dadcdbc Binary files /dev/null and b/test_policies/bc/expA/policy_seed2.pt differ diff --git a/test_policies/bc/expA/policy_seed3.pt b/test_policies/bc/expA/policy_seed3.pt new file mode 100644 index 0000000..0fc6651 Binary files /dev/null and b/test_policies/bc/expA/policy_seed3.pt differ diff --git a/test_policies/bc/expA/policy_seed4.pt b/test_policies/bc/expA/policy_seed4.pt new file mode 100644 index 0000000..a00f1bb Binary files /dev/null and b/test_policies/bc/expA/policy_seed4.pt differ diff --git a/test_policies/bc/expA/policy_seed5.pt b/test_policies/bc/expA/policy_seed5.pt new file mode 100644 index 0000000..443e3e3 Binary files /dev/null and b/test_policies/bc/expA/policy_seed5.pt differ diff --git a/test_policies/bc/expB/policy_seed1.pt b/test_policies/bc/expB/policy_seed1.pt new file mode 100644 index 0000000..6e44cf8 Binary files /dev/null and b/test_policies/bc/expB/policy_seed1.pt differ diff --git a/test_policies/bc/expB/policy_seed2.pt b/test_policies/bc/expB/policy_seed2.pt new file mode 100644 index 0000000..cc49704 Binary files /dev/null and b/test_policies/bc/expB/policy_seed2.pt differ diff --git a/test_policies/bc/expB/policy_seed3.pt b/test_policies/bc/expB/policy_seed3.pt new file mode 100644 index 0000000..715b217 Binary files /dev/null and b/test_policies/bc/expB/policy_seed3.pt differ diff --git a/test_policies/bc/expB/policy_seed4.pt b/test_policies/bc/expB/policy_seed4.pt new file mode 100644 index 0000000..84ee90b Binary files /dev/null and b/test_policies/bc/expB/policy_seed4.pt differ diff --git a/test_policies/bc/expB/policy_seed5.pt b/test_policies/bc/expB/policy_seed5.pt new file mode 100644 index 0000000..ce52202 Binary files /dev/null and b/test_policies/bc/expB/policy_seed5.pt differ diff --git a/test_policies/gail/expA/policy_seed1.pt b/test_policies/gail/expA/policy_seed1.pt new file mode 100644 index 0000000..0530c10 Binary files /dev/null and b/test_policies/gail/expA/policy_seed1.pt differ diff --git a/test_policies/gail/expA/policy_seed2.pt b/test_policies/gail/expA/policy_seed2.pt new file mode 100644 index 0000000..9d85579 Binary files /dev/null and b/test_policies/gail/expA/policy_seed2.pt differ diff --git a/test_policies/gail/expA/policy_seed3.pt b/test_policies/gail/expA/policy_seed3.pt new file mode 100644 index 0000000..6d48834 Binary files /dev/null and b/test_policies/gail/expA/policy_seed3.pt differ diff --git a/test_policies/gail/expA/policy_seed4.pt b/test_policies/gail/expA/policy_seed4.pt new file mode 100644 index 0000000..709f468 Binary files /dev/null and b/test_policies/gail/expA/policy_seed4.pt differ diff --git a/test_policies/gail/expA/policy_seed5.pt b/test_policies/gail/expA/policy_seed5.pt new file mode 100644 index 0000000..a524ac9 Binary files /dev/null and b/test_policies/gail/expA/policy_seed5.pt differ diff --git a/test_policies/gail/expB/policy_seed1.pt b/test_policies/gail/expB/policy_seed1.pt new file mode 100644 index 0000000..a97af9c Binary files /dev/null and b/test_policies/gail/expB/policy_seed1.pt differ diff --git a/test_policies/gail/expB/policy_seed2.pt b/test_policies/gail/expB/policy_seed2.pt new file mode 100644 index 0000000..62b1490 Binary files /dev/null and b/test_policies/gail/expB/policy_seed2.pt differ diff --git a/test_policies/gail/expB/policy_seed3.pt b/test_policies/gail/expB/policy_seed3.pt new file mode 100644 index 0000000..4291a26 Binary files /dev/null and b/test_policies/gail/expB/policy_seed3.pt differ diff --git a/test_policies/gail/expB/policy_seed4.pt b/test_policies/gail/expB/policy_seed4.pt new file mode 100644 index 0000000..a8b9423 Binary files /dev/null and b/test_policies/gail/expB/policy_seed4.pt differ diff --git a/test_policies/gail/expB/policy_seed5.pt b/test_policies/gail/expB/policy_seed5.pt new file mode 100644 index 0000000..c9fe325 Binary files /dev/null and b/test_policies/gail/expB/policy_seed5.pt differ diff --git a/test_policies/hail/expA/policy_seed1.pt b/test_policies/hail/expA/policy_seed1.pt index f737503..2ec567e 100644 Binary files a/test_policies/hail/expA/policy_seed1.pt and b/test_policies/hail/expA/policy_seed1.pt differ diff --git a/test_policies/hail/expA/policy_seed2.pt b/test_policies/hail/expA/policy_seed2.pt index 24efe77..6135439 100644 Binary files a/test_policies/hail/expA/policy_seed2.pt and b/test_policies/hail/expA/policy_seed2.pt differ diff --git a/test_policies/hail/expA/policy_seed3.pt b/test_policies/hail/expA/policy_seed3.pt index 8b4304a..bd50d14 100644 Binary files a/test_policies/hail/expA/policy_seed3.pt and b/test_policies/hail/expA/policy_seed3.pt differ diff --git a/test_policies/hail/expA/policy_seed4.pt b/test_policies/hail/expA/policy_seed4.pt index 3fe9fb8..342c473 100644 Binary files a/test_policies/hail/expA/policy_seed4.pt and b/test_policies/hail/expA/policy_seed4.pt differ diff --git a/test_policies/hail/expA/policy_seed5.pt b/test_policies/hail/expA/policy_seed5.pt index 816651f..5cb425d 100644 Binary files a/test_policies/hail/expA/policy_seed5.pt and b/test_policies/hail/expA/policy_seed5.pt differ diff --git a/test_policies/hail/expB/policy_seed1.pt b/test_policies/hail/expB/policy_seed1.pt index 0e66e7b..dfb17c9 100644 Binary files a/test_policies/hail/expB/policy_seed1.pt and b/test_policies/hail/expB/policy_seed1.pt differ diff --git a/test_policies/hail/expB/policy_seed2.pt b/test_policies/hail/expB/policy_seed2.pt index fd2256e..407583f 100644 Binary files a/test_policies/hail/expB/policy_seed2.pt and b/test_policies/hail/expB/policy_seed2.pt differ diff --git a/test_policies/hail/expB/policy_seed3.pt b/test_policies/hail/expB/policy_seed3.pt index c8404e8..92a72b1 100644 Binary files a/test_policies/hail/expB/policy_seed3.pt and b/test_policies/hail/expB/policy_seed3.pt differ diff --git a/test_policies/hail/expB/policy_seed4.pt b/test_policies/hail/expB/policy_seed4.pt index ba59da4..3e3baab 100644 Binary files a/test_policies/hail/expB/policy_seed4.pt and b/test_policies/hail/expB/policy_seed4.pt differ diff --git a/test_policies/hail/expB/policy_seed5.pt b/test_policies/hail/expB/policy_seed5.pt index b4988fc..2727888 100644 Binary files a/test_policies/hail/expB/policy_seed5.pt and b/test_policies/hail/expB/policy_seed5.pt differ diff --git a/test_policies/shail/expA/policy_seed1.pt b/test_policies/shail/expA/policy_seed1.pt index 13104ca..3819718 100644 Binary files a/test_policies/shail/expA/policy_seed1.pt and b/test_policies/shail/expA/policy_seed1.pt differ diff --git a/test_policies/shail/expA/policy_seed2.pt b/test_policies/shail/expA/policy_seed2.pt index 55cc32f..52753fd 100644 Binary files a/test_policies/shail/expA/policy_seed2.pt and b/test_policies/shail/expA/policy_seed2.pt differ diff --git a/test_policies/shail/expA/policy_seed3.pt b/test_policies/shail/expA/policy_seed3.pt index 8ef0c48..15a6c1d 100644 Binary files a/test_policies/shail/expA/policy_seed3.pt and b/test_policies/shail/expA/policy_seed3.pt differ diff --git a/test_policies/shail/expA/policy_seed4.pt b/test_policies/shail/expA/policy_seed4.pt index bf309e7..98cc3e7 100644 Binary files a/test_policies/shail/expA/policy_seed4.pt and b/test_policies/shail/expA/policy_seed4.pt differ diff --git a/test_policies/shail/expA/policy_seed5.pt b/test_policies/shail/expA/policy_seed5.pt index 9c0af5d..f3e3bff 100644 Binary files a/test_policies/shail/expA/policy_seed5.pt and b/test_policies/shail/expA/policy_seed5.pt differ diff --git a/test_policies/shail/expB/policy_seed1.pt b/test_policies/shail/expB/policy_seed1.pt index 4bf1612..e5a93dc 100644 Binary files a/test_policies/shail/expB/policy_seed1.pt and b/test_policies/shail/expB/policy_seed1.pt differ diff --git a/test_policies/shail/expB/policy_seed2.pt b/test_policies/shail/expB/policy_seed2.pt index 9b2abe7..c1ce024 100644 Binary files a/test_policies/shail/expB/policy_seed2.pt and b/test_policies/shail/expB/policy_seed2.pt differ diff --git a/test_policies/shail/expB/policy_seed3.pt b/test_policies/shail/expB/policy_seed3.pt index 51c6732..ad03c95 100644 Binary files a/test_policies/shail/expB/policy_seed3.pt and b/test_policies/shail/expB/policy_seed3.pt differ diff --git a/test_policies/shail/expB/policy_seed4.pt b/test_policies/shail/expB/policy_seed4.pt index 55bb8bc..3c309da 100644 Binary files a/test_policies/shail/expB/policy_seed4.pt and b/test_policies/shail/expB/policy_seed4.pt differ diff --git a/test_policies/shail/expB/policy_seed5.pt b/test_policies/shail/expB/policy_seed5.pt index fbbde28..7600e77 100644 Binary files a/test_policies/shail/expB/policy_seed5.pt and b/test_policies/shail/expB/policy_seed5.pt differ