Copy over config files

bc-experiment.py

@@ -162,9 +162,9 @@ if __name__ == '__main__':
                 },
                 'policy': {
                     'learning_rate': 3e-4, 
-                    'learning_rate_decay': 1.0, 
-                    'hidden_layer_size': tune.grid_search([20, 40]),
-                    'n_hidden_layers': tune.grid_search([2, 3]), 
+                    'learning_rate_decay': tune.grid_search([0.001, 1.0]), 
+                    'hidden_layer_size': tune.grid_search([10, 20, 40, 80]),
+                    'n_hidden_layers': tune.grid_search([2, 3, 4]), 
                     'activation':0,
                 },
                 'train_epochs': args.epochs,

eval_experiments.py

@@ -1,17 +1,23 @@
 import os
 from src.eval_main import eval_main
 from src.evaluation.utils import load_and_average
+import torch
+import json
+
+activations = [torch.nn.Tanh, torch.nn.LeakyReLU]
 
 def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=False):
     
+    exclude_keys_from_policy_kwargs = {'learning_rate', 'learning_rate_decay', 'clip_ratio', 'iterations_per_epoch', 'option'}
     policy_kwargs = {}
+
     if method in ['expert', 'idm']:
         env, env_kwargs ='NRasterizedRouteIncrementingAgent', {}
     elif method in ['bc','gail']:
         env='NormalizedContinuousEvalEnv' 
         env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000}
     elif method in ['hail']:
-        env = 'NormalizedOptionsEvalEnv'
+        env = 'NormalizedSafeOptionsEvalEnv'
         env_kwargs={'stop_on_collision':True, 'max_episode_steps':1000, 'safe_actions_collision_method': None, 'abort_unsafe_collision_method': None}
     elif method in ['shail']:
         env = 'NormalizedSafeOptionsEvalEnv'
@@ -23,7 +29,18 @@ def main(method:str='expert', folder:str=None, locations=[(0,0)], skip_running=F
 
     if folder is not None:
         files = [os.path.join(folder, f) for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
-        print('%i folders found in %s folder' %(len(files), folder))
+        files = [f for f in files if f.endswith('.pt')]
+        with open(os.path.join(folder, 'config.json'), 'rb') as f:
+            config = json.load(f)
+        print('%i policy files found in %s folder' %(len(files), folder))
+        print('found policy config', config['policy'])
+
+    policy_config = {k: v for k, v in config['policy'].items() if k not in exclude_keys_from_policy_kwargs}
+    policy_config['activation'] = activations[policy_config['activation']]
+    print('final policy config', policy_config)
+
+    policy_kwargs.update(policy_config)
+    print('final policy kwargs', policy_kwargs)
 
     if not skip_running:
         for policy_file in files:

shail-experiment.py

@@ -247,6 +247,13 @@ if __name__ == '__main__':
             os.makedirs(savepath)
         
         import shutil
+
+        # save config
+        shutil.copyfile(
+            args.test,
+            os.path.join(savepath, 'config.json')
+        )
+
         for i in range(args.test_seeds):
             s = analysis._checkpoints[i]['config']['seed']
             check_dir = analysis._checkpoints[i]['logdir']

src/baselines/rule_policies.py

@@ -14,9 +14,9 @@ class PControllerPolicy(BaseAlgorithm):
         self._env = env
         self.target_v = 8.94 # m/s
         self.attn_weight = 20
-
+    
     # BaseAlgorithm abstract methods
-    def _setup_model(self):
+    def _setup_model(self): 
         return None
     def learn(self, *args, **kwargs):
         return self
@@ -26,7 +26,7 @@ class PControllerPolicy(BaseAlgorithm):
         Generate action, state from observation
 
         (But actually generate next action from underlying environment state)
-
+        
         Args:
             observation (np.ndarray): instantaneous observation from environment
 
@@ -36,16 +36,16 @@ class PControllerPolicy(BaseAlgorithm):
         """
         agent = self._env._agent
         ego_state = self._env._env.projected_state[agent].numpy() # (5,) tensor
-
-
+        
+        
         # relative_state = np.delete(self._env._env.relative_state[agent].numpy(), agent, axis=0) #(nv-1, 6) tensor
-
+        
         # calculate front and left distances from ego
 
         # calculate relative speed in direction of position difference vector
-
+        
         # calculate angle alpha and distance d of vehicle i from ego heading
-
+        
         # attn[i] ~= exp( -(alpha[i])^2 - .01 * d[i] - .1 * vrel[i]
 
 
@@ -60,14 +60,14 @@ class IDMRulePolicy(BaseAlgorithm):
 
     The front car is chosen as the closer of:
         - closest car within a 45 degree half angle cone of the ego's heading
-        - ''' after propagating the environment forward by `t_future' seconds with
+        - ''' after propagating the environment forward by `t_future' seconds with 
             current headings and velocities
-
+    
     """
 
-    def __init__(self, env: Intersimple,
-        target_speed:float= 8.94,
-        t_future:List[float]=[0., 1., 2., 3.],
+    def __init__(self, env: Intersimple, 
+        target_speed:float= 8.94, 
+        t_future:List[float]=[0., 1., 2., 3.], 
         half_angle:float=60.):
         """
         Initialize policy with pointer to environment it will run on and target speed
@@ -82,7 +82,6 @@ class IDMRulePolicy(BaseAlgorithm):
         self._env = env
         self.t_future = t_future
         self.half_angle = half_angle
-        self.max_heading_diff = 120
 
         # Default IDM parameters
         assert target_speed>0, 'negative target speed'
@@ -96,18 +95,18 @@ class IDMRulePolicy(BaseAlgorithm):
         np.seterr(invalid='ignore')
 
     # BaseAlgorithm abstract methods
-    def _setup_model(self):
+    def _setup_model(self): 
         return None
     def learn(self, *args, **kwargs):
         return self
-
-    def predict(self, observation:np.ndarray,
+    
+    def predict(self, observation:np.ndarray, 
         *args, **kwargs) -> Tuple[np.ndarray, None]:
         """
         Predict action, state from observation
 
         (But actually generate next action from underlying environment state)
-
+        
         Args:
             observation (np.ndarray): instantaneous observation from environment
 
@@ -139,11 +138,11 @@ class IDMRulePolicy(BaseAlgorithm):
             if t > 0:
                 xy2 = xy + t * v * np.vstack((np.cos(psi[:,0]), np.sin(psi[:,0]))).T
                 d2, r2, i2 = self.get_ego_dr(agent, xy2, v, psi)
-
+                
                 # choose closer vehicle (now vs imagined)
                 if d2 < d:
                     d, r, i = d2, r2, i2
-
+        
         # Update environment interaction graph with i
         if i:
             self._env._env._graph._neighbor_dict={agent:[i]}
@@ -156,19 +155,19 @@ class IDMRulePolicy(BaseAlgorithm):
 
         assert (d_des>= self.d_min)
         action = self.a_max*(1 - (s/self.s_max)**4 - (d_des/d)**2)
-
+        
         # normalize action to range if env is a NormalizedActionSpace
         if isinstance(self._env, NormalizedActionSpace):
             action = self._env._normalize(action)
 
         assert action.shape==(1,)
         return action
-
-    def get_ego_dr(self, agent:int, xy: np.ndarray,
+    
+    def get_ego_dr(self, agent:int, xy: np.ndarray, 
         v: np.ndarray, psi: np.ndarray) -> Tuple[float, float, Optional[int]]:
         """
         Return distance and relative speed of closest car within half angle from heading
-
+        
         Args:
             agent (int): agent index
             xy (np.ndarray): (nv, 2) x and y positions
@@ -178,7 +177,7 @@ class IDMRulePolicy(BaseAlgorithm):
         Returns:
             d (float): distance to closest vehicle in cone
             r (float): relative speed between the two vehicles
-            i (Optional[int]): index of closest vehicle, or None
+            i (Optional[int]): index of closest vehicle, or None 
         """
         nv, nxy = xy.shape
         nv2, nvel = v.shape
@@ -193,12 +192,8 @@ class IDMRulePolicy(BaseAlgorithm):
         dl = (dxys*np.hstack((-np.sin(psi), np.cos(psi)))).sum(-1) # (nv, )
         alpha = to_circle(np.arctan2(dl, df))
 
-        heading_diff = to_circle(psi - psi[agent]).flatten()
-
-        val_idx = np.arange(nv)[
-            (np.abs(alpha) < self.half_angle*np.pi/180) & (np.arange(nv) != agent) & (np.abs(heading_diff) < self.max_heading_diff*np.pi/180)
-            ]
-
+        val_idx = np.arange(nv)[(np.abs(alpha) < self.half_angle*np.pi/180) & (np.arange(nv) != agent)]
+        
         if len(val_idx)==0:
             i = None
             d = float('inf')
@@ -207,10 +202,10 @@ class IDMRulePolicy(BaseAlgorithm):
             idx = np.argmin(ds[val_idx]) # closest car which meets requirements
             i = int(val_idx[idx])
             d = ds[i]
-            r = v[i,0]-v[agent,0]
-
+            r = v[i,0]-v[agent,0] 
+            
         return d, r, i
-
+    
 def to_circle(x: np.ndarray) -> np.ndarray:
     """
     Casts x (in rad) to [-pi, pi)

src/eval_main.py

@@ -41,33 +41,33 @@ def load_policy(method:str,
     if method == 'idm':
         policy = IDMRulePolicy(env, **policy_kwargs)
     elif method == 'bc':
-        policy = SetPolicy(env.action_space.shape[-1])
+        policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'gail-trpo':
-        policy = SetPolicy(env.action_space.shape[-1])
+        policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs)
         policy(torch.zeros(env.observation_space.shape))
         policy = ReparamPolicy(policy)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'gail':
-        policy = SetPolicy(env.action_space.shape[-1])
+        policy = SetPolicy(env.action_space.shape[-1], **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'rail':
         raise NotImplementedError
     elif method == 'hail-trpo':
-        policy = SetDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy(torch.zeros(env.observation_space.shape))
         policy = ReparamPolicy(policy)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'hail':
-        policy = SetDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'shail-trpo':
-        policy = SetMaskedDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy(
             torch.zeros(env.observation_space['observation'].shape),
             torch.zeros(env.observation_space['safe_actions'].shape)
@@ -76,7 +76,7 @@ def load_policy(method:str,
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     elif method == 'shail':
-        policy = SetMaskedDiscretePolicy(env.action_space.n)
+        policy = SetMaskedDiscretePolicy(env.action_space.n, **policy_kwargs)
         policy.load_state_dict(torch.load(policy_file, map_location=ml))
         policy.eval()
     else:

test_policies/bc/expA/config.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 300,
+    "seed": 0
+}

test_policies/bc/expB/config.json

@@ -0,0 +1,15 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "train_epochs": 300,
+    "seed": 0
+}

test_policies/gail/expA/config.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/gail/expB/config.json

@@ -0,0 +1,31 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0
+    },
+    "value": {
+        "learning_rate": 0.0001,
+        "weight_decay": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/hail-etienne/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/hail-etienne/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/hail/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 90,
+    "seed": 0
+}

test_policies/hail/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": null,
+        "abort_unsafe_collision_method": null
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 85,
+    "seed": 0
+}

test_policies/shail-etienne/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/shail-etienne/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 4,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 3,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 100,
+    "seed": 0
+}

test_policies/shail/expA/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "A",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 40,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 1,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 90,
+    "seed": 0
+}

test_policies/shail/expB/config.json

@@ -0,0 +1,33 @@
+{
+    "experiment": "B",
+    "trainenv": {
+        "stop_on_collision": false,
+        "safe_actions_collision_method": "circle",
+        "abort_unsafe_collision_method": "circle"
+    },
+    "policy": {
+        "learning_rate": 0.0003,
+        "learning_rate_decay": 1.0,
+        "clip_ratio": 0.2,
+        "iterations_per_epoch": 100,
+        "hidden_layer_size": 20,
+        "n_hidden_layers": 2,
+        "activation": 0,
+        "option": 0
+    },
+    "value": {
+        "learning_rate": 0.001,
+        "iterations_per_epoch": 1000
+    },
+    "discriminator": {
+        "learning_rate": 0.001,
+        "weight_decay": 0.0001,
+        "iterations_per_epoch": 100,
+        "n_hidden_layers_element": 4,
+        "n_hidden_layers_global": 2,
+        "hidden_layer_size": 10,
+        "activation": 0
+    },
+    "train_epochs": 85,
+    "seed": 0
+}