Implement ValueDICE and some restructuring

2021-08-04 19:34:49 +02:00
parent cba42c6e4d
commit 5c40de66fa
8 changed files with 442 additions and 86 deletions
--- a/config/value_dice.json5
+++ b/config/value_dice.json5
@@ -0,0 +1,84 @@
+{
+    policy_net: {
+        ego_encoder: {
+            input_dim: 5, // number of state vars
+            hidden_n: 0,
+            hidden_dim: 5,
+            output_dim: 5
+        },
+        deepsets: {
+            input_dim: 6, // number of relative state vars for others
+            phi: {
+                hidden_n: 2,
+                hidden_dim: 20,
+            },
+            latent_dim: 20,
+            rho: {
+                hidden_n: 2,
+                hidden_dim: 10,
+            },
+            output_dim: 10
+        },
+        path_encoder: {
+            input_dim: 40, // 2 * path length for (x,y) coordinates
+            hidden_n: 0,
+            hidden_dim: 20,
+            output_dim: 10,
+        },
+        head: {
+            input_dim: 0, // computed in policy constructor
+            hidden_n: 3,
+            hidden_dim: 50,
+            output_dim: 1, // number of outputs e.g. number of actions, or just one
+            final_activation: 'sigmoid',
+        },
+    },
+    value_net: {
+        ego_encoder: {
+            input_dim: 5, // number of state vars
+            hidden_n: 0,
+            hidden_dim: 5,
+            output_dim: 5
+        },
+        deepsets: {
+            input_dim: 6, // number of relative state vars for others
+            phi: {
+                hidden_n: 2,
+                hidden_dim: 20,
+            },
+            latent_dim: 20,
+            rho: {
+                hidden_n: 2,
+                hidden_dim: 10,
+            },
+            output_dim: 10
+        },
+        path_encoder: {
+            input_dim: 40, // 2 * path length for (x,y) coordinates
+            hidden_n: 0,
+            hidden_dim: 20,
+            output_dim: 10,
+        },
+        action_dim: 1, // number of actions
+        head: {
+            input_dim: 0, // computed in policy constructor
+            hidden_n: 3,
+            hidden_dim: 50,
+            output_dim: 1, // number of outputs e.g. number of actions, or just one
+            final_activation: 'sigmoid',
+        },
+    },
+    policy_optim: {
+            optimizer: 'adam',
+            lr: 1e-3,
+            weight_decay: 0.1,
+    },
+    value_optim: {
+            optimizer: 'adam',
+            lr: 1e-3,
+            weight_decay: 0.1,
+    },
+    train_epochs: 8,
+    train_batch_size: 32,
+    discount: 0.95,
+}