Merge pull request #6 from sisl/dev-idm-videos

Optionally save videos of evaluated policies

README.md

@@ -1,10 +1,22 @@
 # InteractionImitation
-Imitation Learning with the INTERACTION Dataset
+Imitation Learning with the [Interaction Dataset](https://interaction-dataset.com/) via the [InteractionSimulator](https://github.com/sisl/InteractionSimulator) gym environments.
+
+Code for "[SHAIL: Safety-Aware Hierarchical Adversarial Imitation Learning for Autonomous Driving in Urban Environments](https://arxiv.org/abs/2204.01922)".
+If you find this repository useful, please cite the paper:
+
+```
+@article{jamgochian2022shail,
+  author = {Arec Jamgochian and Etienne Buehrle and Johannes Fischer and Mykel J. Kochenderfer},
+  title = {{SHAIL}: Safety-Aware Hierarchical Adversarial Imitation Learning for Autonomous Driving in Urban Environments},
+  journal = {arXiv:2204.01922 [cs]},
+  year = {2022}
+}
+```
 
 ## Getting started
-Clone InteractionSimulator and pip install the module.
+Clone the `InteractionSimulator` with the `shail` tag and pip install the module.
 ```
-git clone https://github.com/sisl/InteractionSimulator.git
+git clone --branch shail https://github.com/sisl/InteractionSimulator.git
 cd InteractionSimulator
 pip install -e .
 cd ..

bc-experiment.py

@@ -55,6 +55,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
         )) for _ in range(60)]
@@ -68,6 +69,8 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
+                track=track,
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
         )) for _ in range(15)] for track in range(4)],[])
@@ -159,6 +162,7 @@ if __name__ == '__main__':
                 'experiment': args.train,
                 'trainenv': {
                     'stop_on_collision': False, 
+                    'use_idm':True,
                 },
                 'policy': {
                     'learning_rate': 3e-4, 

best_configs/bc_expA.json

@@ -1,7 +1,8 @@
 {
     "experiment": "A",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/bc_expB.json

@@ -1,7 +1,8 @@
 {
     "experiment": "B",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/gail_expA.json

@@ -1,7 +1,8 @@
 {
     "experiment": "A",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/gail_expB.json

@@ -1,7 +1,8 @@
 {
     "experiment": "B",
     "trainenv": {
-        "stop_on_collision": false
+        "stop_on_collision": false,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/hail_expA.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": null,
-        "abort_unsafe_collision_method": null
+        "abort_unsafe_collision_method": null,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/hail_expB.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": null,
-        "abort_unsafe_collision_method": null
+        "abort_unsafe_collision_method": null,
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/shail_expA.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": "circle",
-        "abort_unsafe_collision_method": "circle"
+        "abort_unsafe_collision_method": "circle",
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

best_configs/shail_expB.json

@@ -3,7 +3,8 @@
     "trainenv": {
         "stop_on_collision": false,
         "safe_actions_collision_method": "circle",
-        "abort_unsafe_collision_method": "circle"
+        "abort_unsafe_collision_method": "circle",
+        "use_idm": true
     },
     "policy": {
         "learning_rate": 0.0003,

cp-videos.sh

@@ -0,0 +1,10 @@
+# cp-videos videos/ videos/icra23/
+
+agents=( 5 27 39 43 47 53 63 81 83 87 93 96 105 113 124 127 130 134 )
+
+for a in "${agents[@]}"
+do
+    cp "$1/expert_agent/loc0/track0/agent${a}_ani.mp4" "$2/t${a}expert.mp4"
+    cp "$1/idm/loc0/track0/agent${a}_ani.mp4" "$2/t${a}idm.mp4"
+    cp "$1/shail/loc0/track0/agent${a}_ani.mp4" "$2/t${a}shail.mp4"
+done

eval_experiments.py

@@ -11,17 +11,19 @@ def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=F
     exclude_keys_from_policy_kwargs = {'learning_rate', 'learning_rate_decay', 'clip_ratio', 'iterations_per_epoch', 'option'}
     policy_kwargs = {}
 
-    if method in ['expert', 'expert_agent', 'idm']:
+    if method in ['expert', 'expert_agent']:
         env, env_kwargs ='NRasterizedRouteIncrementingAgent', {}
+    elif method in ['idm']:
+        env, env_kwargs ='NRasterizedRouteIncrementingAgent', {'use_idm':True}
     elif method in ['bc','gail']:
         env='NormalizedContinuousEvalEnv' 
-        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'use_idm':True}
     elif method in ['hail']:
         env = 'NormalizedSafeOptionsEvalEnv'
-        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None}
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None, 'use_idm':True}
     elif method in ['shail']:
         env = 'NormalizedSafeOptionsEvalEnv'
-        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
+        env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'use_idm':True}
     else:
         raise NotImplementedError
 

evaluate_models.sh

@@ -11,8 +11,8 @@ python -m eval_experiments --method shail --folder='test_policies/shail/expA'
 
 # Experiment B
 python -m eval_experiments --locations='[(0,4)]' 
-python -m eval_experiments --method idm --locations='[(0,4)]' --skip_running
+python -m eval_experiments --method idm --locations='[(0,4)]'
-python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]' --skip_running
+python -m eval_experiments --method bc --folder='test_policies/bc/expB' --locations='[(0,4)]'
-python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]' --skip_running
+python -m eval_experiments --method gail --folder='test_policies/gail/expB' --locations='[(0,4)]'
-python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]' --skip_running
+python -m eval_experiments --method hail --folder='test_policies/hail/expB' --locations='[(0,4)]'
-python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]' --skip_running
+python -m eval_experiments --method shail --folder='test_policies/shail/expB' --locations='[(0,4)]'

gail-experiment.py

@@ -53,6 +53,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
         )) for _ in range(60)]
@@ -67,6 +68,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
                 track=track,
             ), collision_distance=6, collision_penalty=100),
             lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10)
@@ -169,7 +171,8 @@ if __name__ == '__main__':
             config={
                 'experiment': args.train,
                 'trainenv': {
-                    'stop_on_collision': False, 
+                    'stop_on_collision': False,
+                    'use_idm': True, 
                 },
                 'policy': {
                     'learning_rate': 3e-4, 

shail-experiment.py

@@ -58,6 +58,7 @@ def training_function(config):
                 ),
                 check_collisions=True,
                 stop_on_collision=config['trainenv']['stop_on_collision'],
+                use_idm=config['trainenv']['use_idm'],
             ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10))
             ), options=option_list[config['policy']['option']],
             safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], 
@@ -73,7 +74,9 @@ def training_function(config):
                     collision_penalty=0
                 ),
                 check_collisions=True,
-                stop_on_collision=config['trainenv']['stop_on_collision'], track=track,
+                stop_on_collision=config['trainenv']['stop_on_collision'], 
+                use_idm=config['trainenv']['use_idm'], 
+                track=track,
             ), collision_distance=6, collision_penalty=100), lambda obs: (obs - obs_min) / (obs_max - obs_min + 1e-10))
             ), options=option_list[config['policy']['option']],
             safe_actions_collision_method=config['trainenv']['safe_actions_collision_method'], 
@@ -180,6 +183,7 @@ if __name__ == '__main__':
                     'stop_on_collision': False, 
                     'safe_actions_collision_method': 'circle', 
                     'abort_unsafe_collision_method': 'circle',
+                    'use_idm':True,
                 },
                 'policy': {
                     'learning_rate': 3e-4, 

src/__init__.py

@@ -1 +0,0 @@
-

src/baselines/rule_policies.py

@@ -172,6 +172,7 @@ class IDMRulePolicy(BaseAlgorithm):
 
             # Update environment interaction graph with leader
             self._env._env._graph._neighbor_dict={agent:[leader]}
+            self._env._update_graph = True
 
             delta_v = v_ego - v[leader, 0]
             d_des = self.d_min + self.tau * v_ego + v_ego * delta_v / (2* (self.a_max*self.b_pref)**0.5 )

train_models.sh

@@ -18,11 +18,11 @@ python shail-experiment.py --train B
 # Experiment A
 python bc-experiment.py --test best_configs/bc_expA.json
 python gail-experiment.py --test best_configs/gail_expA.json
-python shail-experiment.py --train best_configs/hail_expA.json
+python shail-experiment.py --test best_configs/hail_expA.json
-python shail-experiment.py --train best_configs/shail_expA.json
+python shail-experiment.py --test best_configs/shail_expA.json
 
 # Experiment B
 python bc-experiment.py --test best_configs/bc_expB.json
 python gail-experiment.py --test best_configs/gail_expB.json
-python shail-experiment.py --train best_configs/hail_expB.json
+python shail-experiment.py --test best_configs/hail_expB.json
-python shail-experiment.py --train best_configs/shail_expB.json
+python shail-experiment.py --test best_configs/shail_expB.json