From f37c9d1ea12a34fb2c6b1ea4773b2927cc68b7f1 Mon Sep 17 00:00:00 2001 From: Steve Chien Date: Wed, 13 Feb 2019 14:41:52 -0800 Subject: [PATCH] Use mean loss within each microbatch. PiperOrigin-RevId: 233832864 --- privacy/optimizers/dp_optimizer.py | 4 ++-- privacy/optimizers/dp_optimizer_test.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/privacy/optimizers/dp_optimizer.py b/privacy/optimizers/dp_optimizer.py index 40016d6..01283b6 100644 --- a/privacy/optimizers/dp_optimizer.py +++ b/privacy/optimizers/dp_optimizer.py @@ -71,7 +71,8 @@ def make_optimizer_class(cls): def process_microbatch(i, sample_state): """Process one microbatch (record) with privacy helper.""" grads, _ = zip(*super(cls, self).compute_gradients( - tf.gather(microbatches_losses, [i]), var_list, gate_gradients, + tf.reduce_mean(tf.gather(microbatches_losses, + [i])), var_list, gate_gradients, aggregation_method, colocate_gradients_with_ops, grad_loss)) grads_list = list(grads) sample_state = self._dp_average_query.accumulate_record( @@ -155,4 +156,3 @@ DPAdagradGaussianOptimizer = make_gaussian_optimizer_class( DPAdamGaussianOptimizer = make_gaussian_optimizer_class(tf.train.AdamOptimizer) DPGradientDescentGaussianOptimizer = make_gaussian_optimizer_class( tf.train.GradientDescentOptimizer) - diff --git a/privacy/optimizers/dp_optimizer_test.py b/privacy/optimizers/dp_optimizer_test.py index cb3c235..fcc9bd0 100644 --- a/privacy/optimizers/dp_optimizer_test.py +++ b/privacy/optimizers/dp_optimizer_test.py @@ -37,16 +37,16 @@ class DPOptimizerTest(tf.test.TestCase, parameterized.TestCase): # Parameters for testing: optimizer, num_microbatches, expected answer. @parameterized.named_parameters( ('DPGradientDescent 1', dp_optimizer.DPGradientDescentOptimizer, 1, - [-10.0, -10.0]), + [-2.5, -2.5]), ('DPGradientDescent 2', dp_optimizer.DPGradientDescentOptimizer, 2, - [-5.0, -5.0]), + [-2.5, -2.5]), ('DPGradientDescent 4', dp_optimizer.DPGradientDescentOptimizer, 4, [-2.5, -2.5]), - ('DPAdagrad 1', dp_optimizer.DPAdagradOptimizer, 1, [-10.0, -10.0]), - ('DPAdagrad 2', dp_optimizer.DPAdagradOptimizer, 2, [-5.0, -5.0]), + ('DPAdagrad 1', dp_optimizer.DPAdagradOptimizer, 1, [-2.5, -2.5]), + ('DPAdagrad 2', dp_optimizer.DPAdagradOptimizer, 2, [-2.5, -2.5]), ('DPAdagrad 4', dp_optimizer.DPAdagradOptimizer, 4, [-2.5, -2.5]), - ('DPAdam 1', dp_optimizer.DPAdamOptimizer, 1, [-10.0, -10.0]), - ('DPAdam 2', dp_optimizer.DPAdamOptimizer, 2, [-5.0, -5.0]), + ('DPAdam 1', dp_optimizer.DPAdamOptimizer, 1, [-2.5, -2.5]), + ('DPAdam 2', dp_optimizer.DPAdamOptimizer, 2, [-2.5, -2.5]), ('DPAdam 4', dp_optimizer.DPAdamOptimizer, 4, [-2.5, -2.5])) def testBaseline(self, cls, num_microbatches, expected_answer): with self.cached_session() as sess: