committing learned policies and summary files

README.md

@@ -1,10 +1,22 @@
 # InteractionImitation
-Imitation Learning with the INTERACTION Dataset
+Imitation Learning with the [Interaction Dataset](https://interaction-dataset.com/) via the [InteractionSimulator](https://github.com/sisl/InteractionSimulator) gym environments.
+
+Code for "[SHAIL: Safety-Aware Hierarchical Adversarial Imitation Learning for Autonomous Driving in Urban Environments](https://arxiv.org/abs/2204.01922)".
+If you find this repository useful, please cite the paper:
+
+```
+@article{jamgochian2022shail,
+  author = {Arec Jamgochian and Etienne Buehrle and Johannes Fischer and Mykel J. Kochenderfer},
+  title = {{SHAIL}: Safety-Aware Hierarchical Adversarial Imitation Learning for Autonomous Driving in Urban Environments},
+  journal = {arXiv:2204.01922 [cs]},
+  year = {2022}
+}
+```
 
 ## Getting started
-Clone InteractionSimulator and pip install the module.
+Clone the `InteractionSimulator` with the `shail` tag and pip install the module.
 ```
-git clone https://github.com/sisl/InteractionSimulator.git
+git clone --branch shail https://github.com/sisl/InteractionSimulator.git
 cd InteractionSimulator
 pip install -e .
 cd ..

bc-experiment.py

@@ -55,6 +55,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
         )) for _ in range(60)]
@@ -68,6 +69,8 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
+                track=track,
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
         )) for _ in range(15)] for track in range(4)],[])
@@ -159,6 +162,7 @@ if __name__ == '__main__':
                 'experiment': args.train,
                 'trainenv': {
                     'stop_on_collision': False, 
+                    'use_idm':True,
                 },
                 'policy': {
                     'learning_rate': 3e-4, 

best_configs/bc_expA.json

@@ -1,7 +1,8 @@
 {
     "experiment": "A",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/bc_expB.json

@@ -1,7 +1,8 @@
 {
     "experiment": "B",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/gail_expA.json

@@ -1,7 +1,8 @@
 {
     "experiment": "A",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/gail_expB.json

@@ -1,7 +1,8 @@
 {
     "experiment": "B",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/hail_expA.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": null,
-        "abort_unsafe_collision_method": null
+        "abort_unsafe_collision_method": null,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/hail_expB.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": null,
-        "abort_unsafe_collision_method": null
+        "abort_unsafe_collision_method": null,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/shail_expA.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": "circle",
-        "abort_unsafe_collision_method": "circle"
+        "abort_unsafe_collision_method": "circle",
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/shail_expB.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": "circle",
-        "abort_unsafe_collision_method": "circle"
+        "abort_unsafe_collision_method": "circle",
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

eval_experiments.py

@@ -12,16 +12,16 @@ def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=F
     policy_kwargs = {}
 
     if method in ['expert', 'idm']:
-        env, env_kwargs ='NRasterizedRouteIncrementingAgent', {}
+        env, env_kwargs ='NRasterizedRouteIncrementingAgent', {'use_idm':True}
     elif method in ['bc','gail']:
         env='NormalizedContinuousEvalEnv' 
-        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'use_idm':True}
     elif method in ['hail']:
         env = 'NormalizedSafeOptionsEvalEnv'
-        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None}
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None, 'use_idm':True}
     elif method in ['shail']:
         env = 'NormalizedSafeOptionsEvalEnv'
-        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'use_idm':True}
     else:
         raise NotImplementedError
 

gail-experiment.py

@@ -53,6 +53,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
         )) for _ in range(60)]
@@ -67,6 +68,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
                 track=track,
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
@@ -169,7 +171,8 @@ if __name__ == '__main__':
             config={
                 'experiment': args.train,
                 'trainenv': {
-                    'stop_on_collision': False, 
+                    'stop_on_collision': False,
+                    'use_idm': True, 
                 },
                 'policy': {
                     'learning_rate': 3e-4, 

shail-experiment.py

@@ -58,6 +58,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
             ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10))
             ), options=option_list[config['policy']['option']],
             safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], 
@@ -73,7 +74,9 @@ def training_function(config):
                     collision_penalty=0
                 ),
                 check_collisions=True,
-                stop_on_collision=config['trainenv']['stop_on_collision'], track=track,
+                stop_on_collision=config['trainenv']['stop_on_collision'], 
+                use_idm=config['trainenv']['use_idm'], 
+                track=track,
             ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10))
             ), options=option_list[config['policy']['option']],
             safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], 
@@ -180,6 +183,7 @@ if __name__ == '__main__':
                     'stop_on_collision': False, 
                     'safe_actions_collision_method': 'circle', 
                     'abort_unsafe_collision_method': 'circle',
+                    'use_idm':True,
                 },
                 'policy': {
                     'learning_rate': 3e-4, 

src/__init__.py

@@ -1,2 +0,0 @@
-from src.data.expert_data import generate_expert_data, load_expert_data
-from src.data.data_utils import InteractionDatasetSingleAgent

test_policies/gail/expA/config.json

@@ -1,31 +1,32 @@
 {
-    "experiment": "A",
-    "trainenv": {
-        "stop_on_collision": false
-    },
-    "policy": {
-        "learning_rate": 0.0003,
-        "learning_rate_decay": 1.0,
-        "clip_ratio": 0.2,
-        "iterations_per_epoch": 100,
-        "hidden_layer_size": 40,
-        "n_hidden_layers": 2,
-        "activation": 0
-    },
-    "value": {
-        "learning_rate": 0.0001,
-        "weight_decay": 0.001,
-        "iterations_per_epoch": 1000
-    },
-    "discriminator": {
-        "learning_rate": 0.001,
-        "weight_decay": 0.0001,
-        "iterations_per_epoch": 100,
-        "n_hidden_layers_element": 4,
-        "n_hidden_layers_global": 1,
-        "hidden_layer_size": 10,
-        "activation": 0
-    },
-    "train_epochs": 100,
-    "seed": 0
+  "discriminator": {
+    "activation": 0,
+    "hidden_layer_size": 10,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.001,
+    "n_hidden_layers_element": 4,
+    "n_hidden_layers_global": 1,
+    "weight_decay": 0.0001
+  },
+  "experiment": "A",
+  "policy": {
+    "activation": 0,
+    "clip_ratio": 0.2,
+    "hidden_layer_size": 40,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.0003,
+    "learning_rate_decay": 1.0,
+    "n_hidden_layers": 2
+  },
+  "seed": 5,
+  "train_epochs": 100,
+  "trainenv": {
+    "stop_on_collision": false,
+    "use_idm": true
+  },
+  "value": {
+    "iterations_per_epoch": 1000,
+    "learning_rate": 0.0001,
+    "weight_decay": 0.001
+  }
 }

test_policies/gail/expB/config.json

@@ -1,31 +1,32 @@
 {
-    "experiment": "B",
-    "trainenv": {
-        "stop_on_collision": false
-    },
-    "policy": {
-        "learning_rate": 0.0003,
-        "learning_rate_decay": 1.0,
-        "clip_ratio": 0.2,
-        "iterations_per_epoch": 100,
-        "hidden_layer_size": 40,
-        "n_hidden_layers": 2,
-        "activation": 0
-    },
-    "value": {
-        "learning_rate": 0.0001,
-        "weight_decay": 0.001,
-        "iterations_per_epoch": 1000
-    },
-    "discriminator": {
-        "learning_rate": 0.001,
-        "weight_decay": 0.0001,
-        "iterations_per_epoch": 100,
-        "n_hidden_layers_element": 4,
-        "n_hidden_layers_global": 1,
-        "hidden_layer_size": 10,
-        "activation": 0
-    },
-    "train_epochs": 100,
-    "seed": 0
+  "discriminator": {
+    "activation": 0,
+    "hidden_layer_size": 10,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.001,
+    "n_hidden_layers_element": 4,
+    "n_hidden_layers_global": 1,
+    "weight_decay": 0.0001
+  },
+  "experiment": "B",
+  "policy": {
+    "activation": 0,
+    "clip_ratio": 0.2,
+    "hidden_layer_size": 40,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.0003,
+    "learning_rate_decay": 1.0,
+    "n_hidden_layers": 2
+  },
+  "seed": 4,
+  "train_epochs": 100,
+  "trainenv": {
+    "stop_on_collision": false,
+    "use_idm": true
+  },
+  "value": {
+    "iterations_per_epoch": 1000,
+    "learning_rate": 0.0001,
+    "weight_decay": 0.001
+  }
 }

test_policies/shail/expA/config.json

@@ -1,33 +1,34 @@
 {
-    "experiment": "A",
-    "trainenv": {
-        "stop_on_collision": false,
-        "safe_actions_collision_method": "circle",
-        "abort_unsafe_collision_method": "circle"
-    },
-    "policy": {
-        "learning_rate": 0.0003,
-        "learning_rate_decay": 1.0,
-        "clip_ratio": 0.2,
-        "iterations_per_epoch": 100,
-        "hidden_layer_size": 40,
-        "n_hidden_layers": 2,
-        "activation": 0,
-        "option": 0
-    },
-    "value": {
-        "learning_rate": 0.001,
-        "iterations_per_epoch": 1000
-    },
-    "discriminator": {
-        "learning_rate": 0.001,
-        "weight_decay": 0.0001,
-        "iterations_per_epoch": 100,
-        "n_hidden_layers_element": 4,
-        "n_hidden_layers_global": 1,
-        "hidden_layer_size": 10,
-        "activation": 0
-    },
-    "train_epochs": 90,
-    "seed": 0
+  "discriminator": {
+    "activation": 0,
+    "hidden_layer_size": 10,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.001,
+    "n_hidden_layers_element": 4,
+    "n_hidden_layers_global": 1,
+    "weight_decay": 0.0001
+  },
+  "experiment": "A",
+  "policy": {
+    "activation": 0,
+    "clip_ratio": 0.2,
+    "hidden_layer_size": 40,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.0003,
+    "learning_rate_decay": 1.0,
+    "n_hidden_layers": 2,
+    "option": 0
+  },
+  "seed": 3,
+  "train_epochs": 90,
+  "trainenv": {
+    "abort_unsafe_collision_method": "circle",
+    "safe_actions_collision_method": "circle",
+    "stop_on_collision": false,
+    "use_idm": true
+  },
+  "value": {
+    "iterations_per_epoch": 1000,
+    "learning_rate": 0.001
+  }
 }

test_policies/shail/expB/config.json

@@ -1,33 +1,34 @@
 {
-    "experiment": "B",
-    "trainenv": {
-        "stop_on_collision": false,
-        "safe_actions_collision_method": "circle",
-        "abort_unsafe_collision_method": "circle"
-    },
-    "policy": {
-        "learning_rate": 0.0003,
-        "learning_rate_decay": 1.0,
-        "clip_ratio": 0.2,
-        "iterations_per_epoch": 100,
-        "hidden_layer_size": 20,
-        "n_hidden_layers": 2,
-        "activation": 0,
-        "option": 0
-    },
-    "value": {
-        "learning_rate": 0.001,
-        "iterations_per_epoch": 1000
-    },
-    "discriminator": {
-        "learning_rate": 0.001,
-        "weight_decay": 0.0001,
-        "iterations_per_epoch": 100,
-        "n_hidden_layers_element": 4,
-        "n_hidden_layers_global": 2,
-        "hidden_layer_size": 10,
-        "activation": 0
-    },
-    "train_epochs": 85,
-    "seed": 0
+  "discriminator": {
+    "activation": 0,
+    "hidden_layer_size": 10,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.001,
+    "n_hidden_layers_element": 4,
+    "n_hidden_layers_global": 2,
+    "weight_decay": 0.0001
+  },
+  "experiment": "B",
+  "policy": {
+    "activation": 0,
+    "clip_ratio": 0.2,
+    "hidden_layer_size": 20,
+    "iterations_per_epoch": 100,
+    "learning_rate": 0.0003,
+    "learning_rate_decay": 1.0,
+    "n_hidden_layers": 2,
+    "option": 0
+  },
+  "seed": 3,
+  "train_epochs": 85,
+  "trainenv": {
+    "abort_unsafe_collision_method": "circle",
+    "safe_actions_collision_method": "circle",
+    "stop_on_collision": false,
+    "use_idm": true
+  },
+  "value": {
+    "iterations_per_epoch": 1000,
+    "learning_rate": 0.001
+  }
 }

train_models.sh

@@ -18,11 +18,11 @@ python shail-experiment.py --train B
 # Experiment A
 python bc-experiment.py --test best_configs/bc_expA.json
 python gail-experiment.py --test best_configs/gail_expA.json
-python shail-experiment.py --train best_configs/hail_expA.json
-python shail-experiment.py --train best_configs/shail_expA.json
+python shail-experiment.py --test best_configs/hail_expA.json
+python shail-experiment.py --test best_configs/shail_expA.json
 
 # Experiment B
 python bc-experiment.py --test best_configs/bc_expB.json
 python gail-experiment.py --test best_configs/gail_expB.json
-python shail-experiment.py --train best_configs/hail_expB.json
-python shail-experiment.py --train best_configs/shail_expB.json
+python shail-experiment.py --test best_configs/hail_expB.json
+python shail-experiment.py --test best_configs/shail_expB.json