1. Adding a CLI script for computing privacy loss for DP-SGD.

2. Fixing typos in the MNIST tutorial.

PiperOrigin-RevId: 230608908

privacy/analysis/compute_dp_sgd_privacy.py

@@ -0,0 +1,92 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Command-line script for computing privacy of a model trained with DP-SGD.
+
+The script applies the RDP accountant to estimate privacy budget of an iterated
+Sampled Gaussian Mechanism. The mechanism's parameters are controlled by flags.
+
+Example:
+  compute_dp_sgd_privacy
+    --N=60000 \
+    --batch_size=256 \
+    --noise_multiplier=1.12 \
+    --epochs=60 \
+    --delta=1e-5
+
+The output states that DP-SGD with these parameters satisfies (2.92, 1e-5)-DP.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import math
+
+from absl import app
+from absl import flags
+
+from privacy.analysis.rdp_accountant import compute_rdp
+from privacy.analysis.rdp_accountant import get_privacy_spent
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_integer('N', None, 'Total number of examples')
+flags.DEFINE_integer('batch_size', None, 'Batch size')
+flags.DEFINE_float('noise_multiplier', None, 'Noise multiplier for DP-SGD')
+flags.DEFINE_float('epochs', None, 'Number of epochs (may be fractional)')
+flags.DEFINE_float('delta', 1e-6, 'Target delta')
+
+flags.mark_flag_as_required('N')
+flags.mark_flag_as_required('batch_size')
+flags.mark_flag_as_required('noise_multiplier')
+flags.mark_flag_as_required('epochs')
+
+
+def apply_dp_sgd_analysis(q, sigma, steps, orders, delta):
+  """Compute and print results of DP-SGD analysis."""
+
+  rdp = compute_rdp(q, sigma, steps, orders)
+
+  eps, _, opt_order = get_privacy_spent(orders, rdp, target_delta=delta)
+
+  print('DP-SGD with sampling rate = {:.3g}% and noise_multiplier = {} iterated'
+        ' over {} steps satisfies'.format(100 * q, sigma, steps), end=' ')
+  print('differential privacy with eps = {:.3g} and delta = {}.'.format(
+      eps, delta))
+  print('The optimal RDP order is {}.'.format(opt_order))
+
+  if opt_order == max(orders) or opt_order == min(orders):
+    print('The privacy estimate is likely to be improved by expanding '
+          'the set of orders.')
+
+
+def main(argv):
+  del argv  # argv is not used.
+
+  q = FLAGS.batch_size / FLAGS.N  # q - the sampling ratio.
+
+  if q > 1:
+    raise app.UsageError('N must be larger than the batch size.')
+
+  orders = ([1.25, 1.5, 1.75, 2., 2.25, 2.5, 3., 3.5, 4., 4.5] + range(5, 64) +
+            [128, 256, 512])
+
+  steps = int(math.ceil(FLAGS.epochs * FLAGS.N / FLAGS.batch_size))
+
+  apply_dp_sgd_analysis(q, FLAGS.noise_multiplier, steps, orders, FLAGS.delta)
+
+
+if __name__ == '__main__':
+  app.run(main)

tutorials/README.md

@@ -70,6 +70,19 @@ Test accuracy after 60 epochs is: 0.966
 For delta=1e-5, the current epsilon is: 2.92
 ```
 
+## Using Command-Line Interface for Privacy Budgeting
+
+Before launching a (possibly quite lengthy) training procedure, it is possible
+to compute, quickly and accurately, privacy loss at any point of the training.
+To do so, run the script `privacy/analysis/compute_dp_sgd_privacy.py`, which
+does not have any TensorFlow dependencies. For example, executing
+```
+compute_dp_sgd_privacy.py --N=60000 --batch_size=256 --noise_multiplier=1.12 --epochs=60 --delta=1e-5
+```
+allows us to conclude, in a matter of seconds, that DP-SGD run with default
+parameters satisfies differential privacy with eps = 2.92 and delta = 1e-05.
+
+
 ## Select Parameters
 
 The table below has a few sample parameters illustrating various accuracy/privacy

tutorials/mnist_dpsgd_tutorial.py

@@ -25,7 +25,7 @@ from privacy.analysis.rdp_accountant import compute_rdp
 from privacy.analysis.rdp_accountant import get_privacy_spent
 from privacy.optimizers import dp_optimizer
 
-tf.flags.DEFINE_boolean('dpsgd', True, 'If True, train with DP-SGD. If False,'
+tf.flags.DEFINE_boolean('dpsgd', True, 'If True, train with DP-SGD. If False, '
                         'train with vanilla SGD.')
 tf.flags.DEFINE_float('learning_rate', 0.08, 'Learning rate for training')
 tf.flags.DEFINE_float('noise_multiplier', 1.12,
@@ -33,8 +33,8 @@ tf.flags.DEFINE_float('noise_multiplier', 1.12,
 tf.flags.DEFINE_float('l2_norm_clip', 1.0, 'Clipping norm')
 tf.flags.DEFINE_integer('batch_size', 256, 'Batch size')
 tf.flags.DEFINE_integer('epochs', 60, 'Number of epochs')
-tf.flags.DEFINE_integer('microbatches', 256,
-                        'Number of microbatches (must evenly divide batch_size')
+tf.flags.DEFINE_integer('microbatches', 256, 'Number of microbatches '
+                        '(must evenly divide batch_size)')
 tf.flags.DEFINE_string('model_dir', None, 'Model directory')
 
 FLAGS = tf.flags.FLAGS