From 367f72ec46f1ef0d4ace80648ab8095e86847126 Mon Sep 17 00:00:00 2001 From: Johannes Fischer Date: Fri, 30 Jul 2021 18:19:15 +0200 Subject: [PATCH 1/7] Imrove print output --- experiments/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/experiment.py b/experiments/experiment.py index bed727e..7170e21 100644 --- a/experiments/experiment.py +++ b/experiments/experiment.py @@ -142,7 +142,7 @@ if __name__ == '__main__': analysis = Analysis(outdir, default_metric="cv_loss", default_mode="min") config = analysis.get_best_config() filepath = analysis.get_best_logdir() - print(filepath) + print("Best ray experiment:", filepath) main(None, filestr=opj(filepath, 'exp'), **kwargs) else: raise Exception('No valid config found') From 99f7df2e7c0e1841b4685084e8fe353b4ea03a96 Mon Sep 17 00:00:00 2001 From: Johannes Fischer Date: Fri, 30 Jul 2021 18:19:29 +0200 Subject: [PATCH 2/7] Add comment --- src/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics.py b/src/metrics.py index a114afe..28662e1 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -18,7 +18,7 @@ def metrics(filestr: str, test_dataset, policy): # a) simulation files that were saved under the trained policy with prefix 'policy' # b) applying the policy to observations in the test dataset - # load trajectory + # load simulated trajectory states = torch.load(filestr + '_sim_states.pt').detach() lengths = torch.load(filestr + '_sim_lengths.pt').detach() widths = torch.load(filestr + '_sim_widths.pt').detach() From 281f7773c41de0351fcece95cd81f2937be579db Mon Sep 17 00:00:00 2001 From: Johannes Fischer Date: Mon, 2 Aug 2021 11:04:03 +0200 Subject: [PATCH 3/7] Implement kl divergence methods and tests --- src/metrics.py | 43 +++++++++++++++++++++++++++++++++++++++++-- tests/test_metrics.py | 18 ++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 tests/test_metrics.py diff --git a/src/metrics.py b/src/metrics.py index 28662e1..1722f86 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -79,7 +79,7 @@ def average_velocity(states): arg_v = nanmean(vehicle_avg_v) return arg_v -def divergence(p, q, type='kl'): +def divergence(p, q, type='kl', n_components=0): """ Calculate a divergence between p and q Args: @@ -88,7 +88,46 @@ def divergence(p, q, type='kl'): Returns: d (float): approximate divergence """ - pass + if type == 'kl': + if n_components == 0: + pm = torch.mean(p) + qm = torch.mean(q) + pv = torch.var(p) + qv = torch.var(q) + d = kl_normal(pm, pv, qm, qv).item() + return d + else: + from sklearn.mixture import GaussianMixture + p = p.unsqueeze(-1) + q = q.unsqueeze(-1) + p_gmm = GaussianMixture(n_components=n_components).fit(p) + q_gmm = GaussianMixture(n_components=n_components).fit(q) + px = p_gmm.score_samples(p) + qx = q_gmm.score_samples(p) + d = np.mean(px - qx).item() + return d + else: + raise NotImplementedError("Please implement divergence for type '{}'".format(type)) + + +def kl_normal(pm, pv, qm, qv): + """ + Computes the elem-wise KL divergence between two normal distributions KL(p || q) and + sum over the last dimension + + Args: + pm: tensor: (batch, dim): p mean + pv: tensor: (batch, dim): p variance + qm: tensor: (batch, dim): q mean + qv: tensor: (batch, dim): q variance + + Return: + kl: tensor: (batch,): kl between each sample + """ + element_wise = 0.5 * (torch.log(qv) - torch.log(pv) + pv / qv + (pm - qm).pow(2) / qv - 1) + kl = element_wise.sum(-1) + return kl + def nanmean(v, *args, inplace=False, **kwargs): """ diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..2653b84 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,18 @@ +import torch +import numpy as np + +from src import metrics +from src.metrics import divergence + +def test_kl_divergence(): + p = torch.randn(1000000) + q = 1.0 + 2.0 * torch.randn(1000000) + + d1 = divergence(p, q, type='kl', n_components=0) + assert isinstance(d1, float) + d2 = divergence(p, q, type='kl', n_components=1) + assert isinstance(d2, float) + assert np.isclose(d1, d2, atol=1e-5) + d3 = divergence(p, q, type='kl', n_components=3) + assert isinstance(d3, float) + From 6177b1f7e1382c86957bf386df26b745bac71141 Mon Sep 17 00:00:00 2001 From: Johannes Fischer Date: Mon, 2 Aug 2021 11:12:24 +0200 Subject: [PATCH 4/7] Add kl_cat --- src/metrics.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/metrics.py b/src/metrics.py index c9f838c..175d574 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -127,6 +127,23 @@ def kl_normal(pm, pv, qm, qv): return kl +def kl_cat(q, log_q, log_p): + """ + Computes the KL divergence between two categorical distributions + + Args: + q: tensor: (batch, dim): Categorical distribution parameters + log_q: tensor: (batch, dim): Log of q + log_p: tensor: (batch, dim): Log of p + + Return: + kl: tensor: (batch,) kl between each sample + """ + element_wise = (q * (log_q - log_p)) + kl = element_wise.sum(-1) + return kl + + def nanmean(v, *args, inplace=False, **kwargs): """ Calculate mean over not nan entries From 9dd655bc75df70e9b6130c3bd83fac15fa8dc22d Mon Sep 17 00:00:00 2001 From: Johannes Fischer Date: Mon, 2 Aug 2021 17:38:52 +0200 Subject: [PATCH 5/7] test out kd divergence estimate based on CV-KDE (very slow) --- tests/test_metrics.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 2653b84..b219bf2 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -3,16 +3,56 @@ import numpy as np from src import metrics from src.metrics import divergence +from sklearn.model_selection import GridSearchCV +from sklearn.neighbors import KernelDensity def test_kl_divergence(): p = torch.randn(1000000) q = 1.0 + 2.0 * torch.randn(1000000) d1 = divergence(p, q, type='kl', n_components=0) + print(d1) assert isinstance(d1, float) d2 = divergence(p, q, type='kl', n_components=1) + print(d2) assert isinstance(d2, float) assert np.isclose(d1, d2, atol=1e-5) d3 = divergence(p, q, type='kl', n_components=3) + print(d3) assert isinstance(d3, float) + +def evaluate_histogram(x, hist, bin_edges): + idx = np.digitize(x, bin_edges) + mask = np.logical_and(np.less(0, idx), np.less(idx, len(bin_edges))) + r = np.zeros_like(x) + r[mask] = hist[idx[mask] - 1] + + +if __name__ == '__main__': + # p = torch.randn(10) + # q = 1.0 + 2.0 * torch.randn(10) + + # p = p.unsqueeze(-1) + # q = q.unsqueeze(-1) + # p_hist, p_edges = np.histogram(p.unsqueeze(-1), bins='auto', density=True) + # q_hist, q_edges = np.histogram(q.unsqueeze(-1), bins='auto', density=True) + # # px = p_hist[np.digitize(p, p_edges) - 1] + + # # qx = q_hist[np.digitize(p, q_edges) - 1] + # px = evaluate_histogram(p, p_hist, p_edges) + # qx = evaluate_histogram(q, p_hist, p_edges) + + # use grid search cross-validation to optimize the bandwidth + params = {'bandwidth': np.logspace(-1, 1, 3)} + grid = GridSearchCV(KernelDensity(), params) + grid.fit(p) + p_kde = grid.best_estimator_ + grid = GridSearchCV(KernelDensity(), params) + grid.fit(q) + q_kde = grid.best_estimator_ + px = p_kde.score_samples(p) + qx = q_kde.score_samples(p) + d = np.mean(px - qx).item() + print(d) + test_kl_divergence() \ No newline at end of file From f468b3b7a4463b025bb8028da0114e6e03a53c3a Mon Sep 17 00:00:00 2001 From: Johannes Fischer Date: Mon, 2 Aug 2021 19:14:40 +0200 Subject: [PATCH 6/7] Implement histogram based kl divergence computation --- src/metrics.py | 40 +++++++++++++++++++++++++-- tests/test_metrics.py | 64 +++++++++++++++++++++++-------------------- 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index 175d574..fc28f7f 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -87,7 +87,24 @@ def divergence(p, q, type='kl', n_components=0): d (float): approximate divergence """ if type == 'kl': - if n_components == 0: + if n_components < 0: + # Use histogram binning to discretize sampled distributions + p_hist, p_edges = np.histogram(p.unsqueeze(-1), bins='auto', density=True) + q_hist, q_edges = np.histogram(q.unsqueeze(-1), bins='auto', density=True) + px = evaluate_histogram(p, p_hist, p_edges) + qx = evaluate_histogram(p, q_hist, q_edges) + p_supp = ~np.isclose(px, 0.0) + q_supp = ~np.isclose(qx, 0.0) + if np.any(np.logical_and(p_supp, ~q_supp)): + # if not support(p) subset support(q) + return np.nan + elif ~np.any(p_supp): + # if p is zero everywhere + return 0. + d = np.mean(np.log(px[p_supp] / qx[p_supp])) + return d + elif n_components == 0: + # Assume p and q to be Gaussian pm = torch.mean(p) qm = torch.mean(q) pv = torch.var(p) @@ -161,4 +178,23 @@ def nanmean(v, *args, inplace=False, **kwargs): is_nan = torch.isnan(v) v[is_nan] = 0 result = v.sum(*args, **kwargs) / (~is_nan).float().sum(*args, **kwargs) - return result \ No newline at end of file + return result + + +def evaluate_histogram(x, hist, bin_edges): + """ + Evaluate a histogram + Args: + x (array) : points at which to evaluate the histogram + hist (array): histogram values in terms of number of occurrences or probability + bin_edges (array): edges of histogram bins + e.g. from hist, bin_edges = np.histogram(p, bins='auto', density=True) + Return: + r: tensor: (batch,) kl between each sample + """ + idx = np.digitize(x, bin_edges) + mask = np.logical_and(np.less(0, idx), np.less(idx, len(bin_edges))) + r = np.zeros_like(x) + r[mask] = hist[idx[mask] - 1] + return r + diff --git a/tests/test_metrics.py b/tests/test_metrics.py index b219bf2..bef990d 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -2,7 +2,7 @@ import torch import numpy as np from src import metrics -from src.metrics import divergence +from src.metrics import divergence, evaluate_histogram from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KernelDensity @@ -20,39 +20,43 @@ def test_kl_divergence(): d3 = divergence(p, q, type='kl', n_components=3) print(d3) assert isinstance(d3, float) + d4 = divergence(p, q, type='kl', n_components=-1) + assert isinstance(d4, float) + d5 = divergence(q, p, type='kl', n_components=-1) + assert isinstance(d5, float) + p = 1000000 * torch.randn(1000) + q = torch.randn(1000) + d6 = divergence(p, q, type='kl', n_components=-1) + assert np.isnan(d6) -def evaluate_histogram(x, hist, bin_edges): - idx = np.digitize(x, bin_edges) - mask = np.logical_and(np.less(0, idx), np.less(idx, len(bin_edges))) - r = np.zeros_like(x) - r[mask] = hist[idx[mask] - 1] +def test_evaluate_histogram(): + N = 10000 + p = torch.randn(N) + q = 1.0 + 2.0 * torch.randn(2*N) + + p_hist, p_edges = np.histogram(p.unsqueeze(-1), bins='auto', density=True) + q_hist, q_edges = np.histogram(q.unsqueeze(-1), bins='auto', density=True) + px = evaluate_histogram(p, p_hist, p_edges) + assert px.shape == p.shape + qx = evaluate_histogram(q, p_hist, p_edges) + assert qx.shape == q.shape + px = evaluate_histogram(p, q_hist, q_edges) + assert px.shape == p.shape + qx = evaluate_histogram(q, q_hist, q_edges) + assert qx.shape == q.shape if __name__ == '__main__': - # p = torch.randn(10) - # q = 1.0 + 2.0 * torch.randn(10) + p = torch.randn(10) + q = 1.0 + 2.0 * torch.randn(10) - # p = p.unsqueeze(-1) - # q = q.unsqueeze(-1) - # p_hist, p_edges = np.histogram(p.unsqueeze(-1), bins='auto', density=True) - # q_hist, q_edges = np.histogram(q.unsqueeze(-1), bins='auto', density=True) - # # px = p_hist[np.digitize(p, p_edges) - 1] + p = p.unsqueeze(-1) + q = q.unsqueeze(-1) + p_hist, p_edges = np.histogram(p.unsqueeze(-1), bins='auto', density=True) + q_hist, q_edges = np.histogram(q.unsqueeze(-1), bins='auto', density=True) + # px = p_hist[np.digitize(p, p_edges) - 1] - # # qx = q_hist[np.digitize(p, q_edges) - 1] - # px = evaluate_histogram(p, p_hist, p_edges) - # qx = evaluate_histogram(q, p_hist, p_edges) - - # use grid search cross-validation to optimize the bandwidth - params = {'bandwidth': np.logspace(-1, 1, 3)} - grid = GridSearchCV(KernelDensity(), params) - grid.fit(p) - p_kde = grid.best_estimator_ - grid = GridSearchCV(KernelDensity(), params) - grid.fit(q) - q_kde = grid.best_estimator_ - px = p_kde.score_samples(p) - qx = q_kde.score_samples(p) - d = np.mean(px - qx).item() - print(d) - test_kl_divergence() \ No newline at end of file + # qx = q_hist[np.digitize(p, q_edges) - 1] + px = evaluate_histogram(p, p_hist, p_edges) + qx = evaluate_histogram(q, p_hist, p_edges) From b869597717c5f6b8b8ab4065ca9d662c25e8c846 Mon Sep 17 00:00:00 2001 From: Johannes Fischer Date: Mon, 2 Aug 2021 19:15:02 +0200 Subject: [PATCH 7/7] extend comment on divergence --- src/metrics.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/metrics.py b/src/metrics.py index fc28f7f..5e4997f 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -83,6 +83,11 @@ def divergence(p, q, type='kl', n_components=0): Args: p (torch.tensor): (n) samples from p q (torch.tensor): (m) samples from q + type (str): divergence to use + n_components (int): method to use to compute kl divergence + n_components < 0: approximate samples with histogram density + n_components == 0: approximate samples by Gaussian distributions and compute analytically + n_components > 0: approximate samples as Gaussian mixture models with n_components components Returns: d (float): approximate divergence """