From fc04f8e9ee25219d3fd2e677daea2b8ce2042afc Mon Sep 17 00:00:00 2001 From: Arec Jamgochian Date: Thu, 3 Mar 2022 12:12:12 -0800 Subject: [PATCH] adding final commands for tuning, training final models, testing them --- evaluate_models.sh | 23 +++++++++++------------ train_models.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) create mode 100755 train_models.sh diff --git a/evaluate_models.sh b/evaluate_models.sh index 8ec9a15..3824ffe 100755 --- a/evaluate_models.sh +++ b/evaluate_models.sh @@ -1,19 +1,18 @@ -# can add --skip_running if you've run the runs before on the saved policies +# can add --skip_running if you've already run the saved policies through the test environments and have appropriate +# metrics in the out folder. Doing so will generate average metrics quickly. +# Experiment A python -m eval_experiments -python -m eval_experiments --locations='[(0,4)]' python -m eval_experiments --method idm -python -m eval_experiments --method idm --locations='[(0,4)]' python -m eval_experiments --method bc --folder='test_policies/bc/expA' -python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]' python -m eval_experiments --method gail --folder='test_policies/gail/expA' -python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]' python -m eval_experiments --method hail --folder='test_policies/hail/expA' -python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]' -python -m eval_experiments --method shail --folder='test_policies/shail/expA' -python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]' +python -m eval_experiments --method shail --folder='test_policies/shail/expA' -python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expA' -python -m eval_experiments --method hail --folder='test_policies/hail-etienne/expB' --locations='[(0,4)]' -python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expA' -python -m eval_experiments --method shail --folder='test_policies/shail-etienne/expB' --locations='[(0,4)]' \ No newline at end of file +# Experiment B +python -m eval_experiments --locations='[(0,4)]' +python -m eval_experiments --method idm --locations='[(0,4)]' --skip_running +python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]' --skip_running +python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]' --skip_running +python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]' --skip_running +python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]' --skip_running \ No newline at end of file diff --git a/train_models.sh b/train_models.sh new file mode 100755 index 0000000..dd0590a --- /dev/null +++ b/train_models.sh @@ -0,0 +1,28 @@ +# Hyperparameter tuning runs (will run search defined in files) +# Can add --epochs kwarg to change number of training epochs +# shail-experiment.py will generate the hail config aswell + +# Experiment A +python bc-experiment.py --train A +python gail-experiment.py --train A +python shail-experiment.py --train A + +# Experiment B +python bc-experiment.py --train B +python gail-experiment.py --train B +python shail-experiment.py --train B + +# Retrain best models with 5 seeds +# Can add --test_seeds and --test_cpus kwargs to change number of seeds, allocate more memory + +# Experiment A +python bc-experiment.py --test best_configs/bc_expA.json +python gail-experiment.py --test best_configs/gail_expA.json +python shail-experiment.py --train best_configs/hail_expA.json +python shail-experiment.py --train best_configs/shail_expA.json + +# Experiment B +python bc-experiment.py --test best_configs/bc_expB.json +python gail-experiment.py --test best_configs/gail_expB.json +python shail-experiment.py --train best_configs/hail_expB.json +python shail-experiment.py --train best_configs/shail_expB.json \ No newline at end of file