Lira2021: 4x multigpu

PiperOrigin-RevId: 689814531

research/mi_lira_2021/README.md

@@ -8,14 +8,19 @@ by Nicholas Carlini, Steve Chien, Milad Nasr, Shuang Song, Andreas Terzis, and F
 
 ### INSTALLING
 
-You will need to install fairly standard dependencies
+You will need to install fairly standard dependencies and python 3.11 minimum.
 
-`pip install scipy, sklearn, numpy, matplotlib`
+```
+pip install scipy scikit-learn numpy matplotlib tensorflow tensorflow_datasets
 
-and also some machine learning framework to train models. We train our models
+# This needs to be separate
-with JAX + ObJAX so you will need to follow build instructions for that
+pip install objax
-https://github.com/google/objax
+
-https://objax.readthedocs.io/en/latest/installation_setup.html
+RELEASE_URL="https://storage.googleapis.com/jax-releases/jax_cuda_releases.html"
+JAX_VERSION=`python3 -c 'import jax; print(jax.__version__)'`
+pip uninstall -y jaxlib
+pip install -f $RELEASE_URL jax[cuda]==$JAX_VERSION
+```
 
 ### RUNNING THE CODE
 

research/mi_lira_2021/scripts/train_demo.sh

@@ -11,20 +11,41 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+mkdir -p logs
 
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 0 --logdir exp/cifar10 &> logs/log_0
+SECONDS=0
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 1 --logdir exp/cifar10 &> logs/log_1
+
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 2 --logdir exp/cifar10 &> logs/log_2
+echo '======== 1 of 16 ========'
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 3 --logdir exp/cifar10 &> logs/log_3
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 0 --logdir exp/cifar10 2>&1 | tee logs/log_0
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 &> logs/log_4
+echo '======== 2 of 16 ========'
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 &> logs/log_5
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 1 --logdir exp/cifar10 2>&1 | tee logs/log_1
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 &> logs/log_6
+echo '======== 3 of 16 ========'
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 &> logs/log_7
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 2 --logdir exp/cifar10 2>&1 | tee logs/log_2
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 8 --logdir exp/cifar10 &> logs/log_8
+echo '======== 4 of 16 ========'
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 9 --logdir exp/cifar10 &> logs/log_9
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 3 --logdir exp/cifar10 2>&1 | tee logs/log_3
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 10 --logdir exp/cifar10 &> logs/log_10
+echo '======== 5 of 16 ========'
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 11 --logdir exp/cifar10 &> logs/log_11
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 2>&1 | tee logs/log_4
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 &> logs/log_12
+echo '======== 6 of 16 ========'
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 &> logs/log_13
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 2>&1 | tee logs/log_5
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 &> logs/log_14
+echo '======== 7 of 16 ========'
-CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 &> logs/log_15
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 2>&1 | tee logs/log_6
+echo '======== 8 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 2>&1 | tee logs/log_7
+echo '======== 9 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 8 --logdir exp/cifar10 2>&1 | tee logs/log_8
+echo '======== 10 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 9 --logdir exp/cifar10 2>&1 | tee logs/log_9
+echo '======== 11 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 10 --logdir exp/cifar10 2>&1 | tee logs/log_10
+echo '======== 12 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 11 --logdir exp/cifar10 2>&1 | tee logs/log_11
+echo '======== 13 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 2>&1 | tee logs/log_12
+echo '======== 14 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 2>&1 | tee logs/log_13
+echo '======== 15 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 2>&1 | tee logs/log_14
+echo '======== 16 of 16 ========'
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 2>&1 | tee logs/log_15
+
+echo "COMPLETE: Took ${SECONDS} seconds"

research/mi_lira_2021/scripts/train_demo_multigpu.sh

@@ -11,22 +11,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+mkdir -p logs
 
 CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 0 --logdir exp/cifar10 &> logs/log_0 &
 CUDA_VISIBLE_DEVICES='1' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 1 --logdir exp/cifar10 &> logs/log_1 &
 CUDA_VISIBLE_DEVICES='2' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 2 --logdir exp/cifar10 &> logs/log_2 &
 CUDA_VISIBLE_DEVICES='3' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 3 --logdir exp/cifar10 &> logs/log_3 &
-CUDA_VISIBLE_DEVICES='4' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 &> logs/log_4 &
+wait;
-CUDA_VISIBLE_DEVICES='5' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 &> logs/log_5 &
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 &> logs/log_4 &
-CUDA_VISIBLE_DEVICES='6' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 &> logs/log_6 &
+CUDA_VISIBLE_DEVICES='1' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 &> logs/log_5 &
-CUDA_VISIBLE_DEVICES='7' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 &> logs/log_7 &
+CUDA_VISIBLE_DEVICES='2' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 &> logs/log_6 &
+CUDA_VISIBLE_DEVICES='3' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 &> logs/log_7 &
 wait;
 CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 8 --logdir exp/cifar10 &> logs/log_8 &
 CUDA_VISIBLE_DEVICES='1' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 9 --logdir exp/cifar10 &> logs/log_9 &
 CUDA_VISIBLE_DEVICES='2' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 10 --logdir exp/cifar10 &> logs/log_10 &
 CUDA_VISIBLE_DEVICES='3' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 11 --logdir exp/cifar10 &> logs/log_11 &
-CUDA_VISIBLE_DEVICES='4' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 &> logs/log_12 &
+wait;
-CUDA_VISIBLE_DEVICES='5' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 &> logs/log_13 &
+CUDA_VISIBLE_DEVICES='0' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 &> logs/log_12 &
-CUDA_VISIBLE_DEVICES='6' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 &> logs/log_14 &
+CUDA_VISIBLE_DEVICES='1' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 &> logs/log_13 &
-CUDA_VISIBLE_DEVICES='7' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 &> logs/log_15 &
+CUDA_VISIBLE_DEVICES='2' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 &> logs/log_14 &
+CUDA_VISIBLE_DEVICES='3' python3 -u train.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 &> logs/log_15 &
 wait;

research/mi_lira_2021/train.py

@@ -66,7 +66,9 @@ class TrainLoop(objax.Module):
         for k, v in kv.items():
             if jn.isnan(v):
                 raise ValueError('NaN, try reducing learning rate', k)
-            if summary is not None:
+            if summary is not None and v.ndim == 1:
+                summary.scalar(k, float(v[0]))
+            elif summary is not None:
                 summary.scalar(k, float(v))
 
     def train(self, num_train_epochs: int, train_size: int, train: DataSet, test: DataSet, logdir: str, save_steps=100, patience=None):

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/README.md

@@ -10,7 +10,8 @@ memorization is present and thus the less privacy-preserving the model is.
 
 The privacy vulnerability (or memorization potential) is measured via the area
 under the ROC-curve (`auc`) or via max{|fpr - tpr|} (`advantage`) of the attack
-classifier. These measures are very closely related.
+classifier. These measures are very closely related. We can also obtain a lower
+bound for the differential privacy epsilon.
 
 The tests provided by the library are "black box". That is, only the outputs of
 the model are used (e.g., losses, logits, predictions). Neither model internals
@@ -38,9 +39,8 @@ from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_s
 # loss_test  shape: (n_test, )
 
 attacks_result = mia.run_attacks(
-    AttackInputData(
+    AttackInputData(loss_train=loss_train, loss_test=loss_test)
-        loss_train = loss_train,
+)
-        loss_test = loss_test))
 ```
 
 This example calls `run_attacks` with the default options to run a host of
@@ -57,9 +57,11 @@ Then, we can view the attack results by:
 ```python
 print(attacks_result.summary())
 # Example output:
-# -> Best-performing attacks over all slices
+# Best-performing attacks over all slices
-#      THRESHOLD_ATTACK (with 50000 training and 10000 test examples) achieved an AUC of 0.59 on slice Entire dataset
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved an AUC of 0.72 on slice CORRECTLY_CLASSIFIED=False
-#      THRESHOLD_ATTACK (with 50000 training and 10000 test examples) achieved an advantage of 0.20 on slice Entire dataset
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved an advantage of 0.34 on slice CORRECTLY_CLASSIFIED=False
+#   LOGISTIC_REGRESSION (with 5000 training and 1000 test examples) achieved a positive predictive value of 1.00 on slice CLASS=0
+#   THRESHOLD_ATTACK (with 50000 training and 10000 test examples) achieved top-5 epsilon lower bounds of 4.6254, 4.6121, 4.5986, 4.5850, 4.5711 on slice Entire dataset
 ```
 
 ### Other codelabs
@@ -100,16 +102,17 @@ First, similar as before, we specify the input for the attack as an
 # loss_test  shape: (n_test, )
 
 attack_input = AttackInputData(
-    logits_train = logits_train,
+    logits_train=logits_train,
-    logits_test = logits_test,
+    logits_test=logits_test,
-    loss_train = loss_train,
+    loss_train=loss_train,
-    loss_test = loss_test,
+    loss_test=loss_test,
-    labels_train = labels_train,
+    labels_train=labels_train,
-    labels_test = labels_test)
+    labels_test=labels_test,
+)
 ```
 
 Instead of `logits`, you can also specify `probs_train` and `probs_test` as the
-predicted probabilty vectors of each example.
+predicted probability vectors of each example.
 
 Then, we specify some details of the attack. The first part includes the
 specifications of the slicing of the data. For example, we may want to evaluate
@@ -118,10 +121,11 @@ the model's classification. These can be specified by a `SlicingSpec` object.
 
 ```python
 slicing_spec = SlicingSpec(
-    entire_dataset = True,
+    entire_dataset=True,
-    by_class = True,
+    by_class=True,
-    by_percentiles = False,
+    by_percentiles=False,
-    by_classification_correctness = True)
+    by_classification_correctness=True,
+)
 ```
 
 The second part specifies the classifiers for the attacker to use. Currently,
@@ -129,56 +133,64 @@ our API supports five classifiers, including `AttackType.THRESHOLD_ATTACK` for
 simple threshold attack, `AttackType.LOGISTIC_REGRESSION`,
 `AttackType.MULTI_LAYERED_PERCEPTRON`, `AttackType.RANDOM_FOREST`, and
 `AttackType.K_NEAREST_NEIGHBORS` which use the corresponding machine learning
-models. For some model, different classifiers can yield pertty different
+models. For some model, different classifiers can yield pretty different
-results. We can put multiple classifers in a list:
+results. We can put multiple classifiers in a list:
 
 ```python
 attack_types = [
     AttackType.THRESHOLD_ATTACK,
-    AttackType.LOGISTIC_REGRESSION
+    AttackType.LOGISTIC_REGRESSION,
 ]
 ```
 
 Now, we can call the `run_attacks` methods with all specifications:
 
 ```python
-attacks_result = mia.run_attacks(attack_input=attack_input,
+attacks_result = mia.run_attacks(
+    attack_input=attack_input,
     slicing_spec=slicing_spec,
-                                 attack_types=attack_types)
+    attack_types=attack_types,
+)
 ```
 
 This returns an object of type `AttackResults`. We can, for example, use the
-following code to see the attack results specificed per-slice, as we have
+following code to see the attack results specified per-slice, as we have request
-request attacks by class and by model's classification correctness.
+attacks by class and by model's classification correctness.
 
 ```python
 print(attacks_result.summary(by_slices = True))
 # Example output:
-# ->  Best-performing attacks over all slices
+# Best-performing attacks over all slices
-#       THRESHOLD_ATTACK achieved an AUC of 0.75 on slice CORRECTLY_CLASSIFIED=False
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved an AUC of 0.72 on slice CORRECTLY_CLASSIFIED=False
-#       THRESHOLD_ATTACK achieved an advantage of 0.38 on slice CORRECTLY_CLASSIFIED=False
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved an advantage of 0.34 on slice CORRECTLY_CLASSIFIED=False
-#
+#   LOGISTIC_REGRESSION (with 5000 training and 1000 test examples) achieved a positive predictive value of 1.00 on slice CLASS=0
+#   THRESHOLD_ATTACK (with 50000 training and 10000 test examples) achieved top-5 epsilon lower bounds of 4.6254, 4.6121, 4.5986, 4.5850, 4.5711 on slice Entire dataset
+
 # Best-performing attacks over slice: "Entire dataset"
-#       LOGISTIC_REGRESSION achieved an AUC of 0.61
+#   LOGISTIC_REGRESSION (with 50000 training and 10000 test examples) achieved an AUC of 0.58
-#       THRESHOLD_ATTACK achieved an advantage of 0.22
+#   LOGISTIC_REGRESSION (with 50000 training and 10000 test examples) achieved an advantage of 0.17
-#
+#   THRESHOLD_ATTACK (with 50000 training and 10000 test examples) achieved a positive predictive value of 0.86
+#   THRESHOLD_ATTACK (with 50000 training and 10000 test examples) achieved top-5 epsilon lower bounds of 4.6254, 4.6121, 4.5986, 4.5850, 4.5711
+
 # Best-performing attacks over slice: "CLASS=0"
-#       LOGISTIC_REGRESSION achieved an AUC of 0.62
+#   LOGISTIC_REGRESSION (with 5000 training and 1000 test examples) achieved an AUC of 0.63
-#       LOGISTIC_REGRESSION achieved an advantage of 0.24
+#   LOGISTIC_REGRESSION (with 5000 training and 1000 test examples) achieved an advantage of 0.19
-#
+#   LOGISTIC_REGRESSION (with 5000 training and 1000 test examples) achieved a positive predictive value of 1.00
-#     Best-performing attacks over slice: "CLASS=1"
+#   THRESHOLD_ATTACK (with 5000 training and 1000 test examples) achieved top-5 epsilon lower bounds of 4.1920, 4.1645, 4.1364, 4.1074, 4.0775
-#       LOGISTIC_REGRESSION achieved an AUC of 0.61
+
-#       LOGISTIC_REGRESSION achieved an advantage of 0.19
-#
 # ...
-#
+
 # Best-performing attacks over slice: "CORRECTLY_CLASSIFIED=True"
-#       LOGISTIC_REGRESSION achieved an AUC of 0.53
+#   LOGISTIC_REGRESSION (with 42959 training and 6844 test examples) achieved an AUC of 0.51
-#       THRESHOLD_ATTACK achieved an advantage of 0.05
+#   LOGISTIC_REGRESSION (with 42959 training and 6844 test examples) achieved an advantage of 0.05
-#
+#   LOGISTIC_REGRESSION (with 42959 training and 6844 test examples) achieved a positive predictive value of 0.94
+#   THRESHOLD_ATTACK (with 42959 training and 6844 test examples) achieved top-5 epsilon lower bounds of 0.9495, 0.6358, 0.5630, 0.4536, 0.4341
+
 # Best-performing attacks over slice: "CORRECTLY_CLASSIFIED=False"
-#       THRESHOLD_ATTACK achieved an AUC of 0.75
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved an AUC of 0.72
-#       THRESHOLD_ATTACK achieved an advantage of 0.38
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved an advantage of 0.34
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved a positive predictive value of 0.97
+#   LOGISTIC_REGRESSION (with 7041 training and 3156 test examples) achieved top-5 epsilon lower bounds of 3.8844, 3.8678, 3.8510, 3.8339, 3.8165
 ```
 
 #### Viewing and plotting the attack results
@@ -186,23 +198,30 @@ print(attacks_result.summary(by_slices = True))
 We have seen an example of using `summary()` to view the attack results as text.
 We also provide some other ways for inspecting the attack results.
 
-To get the attack that achieves the maximum attacker advantage or AUC, we can do
+To get the attack that achieves the maximum attacker advantage, AUC, or epsilon
+lower bound, we can do
 
 ```python
 max_auc_attacker = attacks_result.get_result_with_max_auc()
 max_advantage_attacker = attacks_result.get_result_with_max_attacker_advantage()
+max_epsilon_attacker = attacks_result.get_result_with_max_epsilon()
 ```
 
 Then, for individual attack, such as `max_auc_attacker`, we can check its type,
-attacker advantage and AUC by
+attacker advantage, AUC, and epsilon lower bound by
 
 ```python
-print("Attack type with max AUC: %s, AUC of %.2f, Attacker advantage of %.2f" %
+print(
-      (max_auc_attacker.attack_type,
+    "Attack type with max AUC: %s, AUC of %.2f, Attacker advantage of %.2f, Epsilon lower bound of %s"
+    % (
+        max_auc_attacker.attack_type,
         max_auc_attacker.roc_curve.get_auc(),
-       max_auc_attacker.roc_curve.get_attacker_advantage()))
+        max_auc_attacker.roc_curve.get_attacker_advantage(),
+        max_auc_attacker.get_epsilon_lower_bound()
+    )
+)
 # Example output:
-# -> Attack type with max AUC: THRESHOLD_ATTACK, AUC of 0.75, Attacker advantage of 0.38
+# Attack type with max AUC: LOGISTIC_REGRESSION, AUC of 0.72, Attacker advantage of 0.34, Epsilon lower bound of [3.88435257 3.86781797 3.85100545 3.83390548 3.81650809]
 ```
 
 We can also plot its ROC curve by
@@ -217,7 +236,7 @@ which would give a figure like the one below
 ![roc_fig](https://github.com/tensorflow/privacy/blob/master/tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/codelab_roc_fig.png?raw=true)
 
 Additionally, we provide functionality to convert the attack results into Pandas
-data frame:
+dataframe:
 
 ```python
 import pandas as pd
@@ -225,16 +244,16 @@ import pandas as pd
 pd.set_option("display.max_rows", 8, "display.max_columns", None)
 print(attacks_result.calculate_pd_dataframe())
 # Example output:
-#           slice feature slice value attack type  Attacker advantage       AUC
+#            slice feature slice value  train size  test size          attack type  Attacker advantage  Positive predictive value       AUC  Epsilon lower bound_1  Epsilon lower bound_2  Epsilon lower bound_3  Epsilon lower bound_4  Epsilon lower bound_5
-# 0        entire_dataset               threshold            0.216440  0.600630
+# 0         Entire dataset                   50000      10000     THRESHOLD_ATTACK            0.172520                   0.862614  0.581630               4.625393               4.612104               4.598635               4.584982               4.571140
-# 1        entire_dataset                      lr            0.212073  0.612989
+# 1         Entire dataset                   50000      10000  LOGISTIC_REGRESSION            0.173060                   0.862081  0.583981               4.531399               4.513775               4.511974               4.498905               4.492165
-# 2                 class           0   threshold            0.226000  0.611669
+# 2                  class           0        5000       1000     THRESHOLD_ATTACK            0.162000                   0.877551  0.580728               4.191954               4.164547               4.136368               4.107372               4.077511
-# 3                 class           0          lr            0.239452  0.624076
+# 3                  class           0        5000       1000  LOGISTIC_REGRESSION            0.193800                   1.000000  0.627758               3.289194               3.220285               3.146292               3.118849               3.066407
-# ..                  ...         ...         ...                 ...       ...
+# ...
-# 22  correctly_classfied        True   threshold            0.054907  0.471290
+# 22  correctly_classified        True       42959       6844     THRESHOLD_ATTACK            0.043953                   0.862643  0.474713               0.949550               0.635773               0.563032               0.453640               0.434125
-# 23  correctly_classfied        True          lr            0.046986  0.525194
+# 23  correctly_classified        True       42959       6844  LOGISTIC_REGRESSION            0.048963                   0.943218  0.505334               0.597257               0.596095               0.594016               0.592702               0.590765
-# 24  correctly_classfied       False   threshold            0.379465  0.748138
+# 24  correctly_classified       False        7041       3156     THRESHOLD_ATTACK            0.326865                   0.941176  0.707597               3.818741               3.805451               3.791982               3.778329               3.764488
-# 25  correctly_classfied       False          lr            0.370713  0.737148
+# 25  correctly_classified       False        7041       3156  LOGISTIC_REGRESSION            0.336655                   0.972222  0.717386               3.884353               3.867818               3.851005               3.833905               3.816508
 ```
 
 #### Advanced Membership Inference Attacks

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/codelabs/codelab.ipynb

@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "1eiwVljWpzM7"
       },
       "source": [
@@ -14,9 +13,6 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-    "cellView": "both",
-    "colab": {},
-    "colab_type": "code",
         "id": "4rmwPgXeptiS"
       },
       "outputs": [],
@@ -37,7 +33,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "YM2gRaJMqvMi"
       },
       "source": [
@@ -47,24 +42,22 @@
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "-B5ZvlSqqLaR"
       },
       "source": [
-    "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
+        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-    "  <td>\n",
+        "  \u003ctd\u003e\n",
-    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/privacy/blob/master/tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/codelabs/codelab.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
+        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/privacy/blob/master/tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/codelabs/codelab.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-    "  </td>\n",
+        "  \u003c/td\u003e\n",
-    "  <td>\n",
+        "  \u003ctd\u003e\n",
-    "    <a target=\"_blank\" href=\"https://github.com/tensorflow/privacy/blob/master/tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/codelabs/codelab.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
+        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/privacy/blob/master/tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/codelabs/codelab.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-    "  </td>\n",
+        "  \u003c/td\u003e\n",
-    "</table>"
+        "\u003c/table\u003e"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "9rMuytY7Nn8P"
       },
       "source": [
@@ -75,30 +68,35 @@
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "FUWqArj_q8vs"
       },
       "source": [
         "## Setup\n",
-    "First, set this notebook's runtime to use a GPU, under Runtime > Change runtime type > Hardware accelerator. Then, begin importing the necessary libraries."
+        "First, set this notebook's runtime to use a GPU, under Runtime \u003e Change runtime type \u003e Hardware accelerator. Then, begin importing the necessary libraries."
       ]
     },
     {
       "cell_type": "code",
-   "execution_count": null,
+      "execution_count": 1,
       "metadata": {
-    "cellView": "form",
+        "executionInfo": {
-    "colab": {},
+          "elapsed": 4130,
-    "colab_type": "code",
+          "status": "ok",
+          "timestamp": 1729790860657,
+          "user": {
+            "displayName": "",
+            "userId": ""
+          },
+          "user_tz": 420
+        },
         "id": "Lr1pwHcbralz"
       },
       "outputs": [],
       "source": [
-    "#@title Import statements.\n",
+        "# @title Import statements.\n",
+        "from typing import Text, Tuple\n",
         "import numpy as np\n",
-    "from typing import Tuple, Text\n",
         "from scipy import special\n",
-    "\n",
         "import tensorflow as tf\n",
         "import tensorflow_datasets as tfds\n",
         "\n",
@@ -106,6 +104,7 @@
         "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
         "from warnings import simplefilter\n",
         "from sklearn.exceptions import ConvergenceWarning\n",
+        "\n",
         "simplefilter(action=\"ignore\", category=ConvergenceWarning)\n",
         "simplefilter(action=\"ignore\", category=FutureWarning)"
       ]
@@ -113,7 +112,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "ucw81ar6ru-6"
       },
       "source": [
@@ -125,8 +123,6 @@
       "execution_count": null,
       "metadata": {
         "cellView": "both",
-    "colab": {},
-    "colab_type": "code",
         "id": "zcqAmiGH90kl"
       },
       "outputs": [],
@@ -139,7 +135,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "pBbcG86th_sW"
       },
       "source": [
@@ -150,14 +145,12 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-    "cellView": "form",
+        "collapsed": true,
-    "colab": {},
-    "colab_type": "code",
         "id": "vCyOWyyhXLib"
       },
       "outputs": [],
       "source": [
-    "#@markdown Train a simple model on CIFAR10 with Keras.\n",
+        "# @markdown Train a simple model on CIFAR10 with Keras.\n",
         "\n",
         "dataset = 'cifar10'\n",
         "num_classes = 10\n",
@@ -169,10 +162,12 @@
         "epochs = 100  # Privacy risks are especially visible with lots of epochs.\n",
         "\n",
         "\n",
-    "def small_cnn(input_shape: Tuple[int],\n",
+        "def small_cnn(\n",
+        "    input_shape: Tuple[int],\n",
         "    num_classes: int,\n",
         "    num_conv: int,\n",
-    "              activation: Text = 'relu') -> tf.keras.models.Sequential:\n",
+        "    activation: Text = 'relu',\n",
+        ") -\u003e tf.keras.models.Sequential:\n",
         "  \"\"\"Setup a small CNN for image classification.\n",
         "\n",
         "  Args:\n",
@@ -200,12 +195,14 @@
         "\n",
         "print('Loading the dataset.')\n",
         "train_ds = tfds.as_numpy(\n",
-    "    tfds.load(dataset, split=tfds.Split.TRAIN, batch_size=-1))\n",
+        "    tfds.load(dataset, split=tfds.Split.TRAIN, batch_size=-1)\n",
+        ")\n",
         "test_ds = tfds.as_numpy(\n",
-    "    tfds.load(dataset, split=tfds.Split.TEST, batch_size=-1))\n",
+        "    tfds.load(dataset, split=tfds.Split.TEST, batch_size=-1)\n",
-    "x_train = train_ds['image'].astype('float32') / 255.\n",
+        ")\n",
+        "x_train = train_ds['image'].astype('float32') / 255.0\n",
         "y_train_indices = train_ds['label'][:, np.newaxis]\n",
-    "x_test = test_ds['image'].astype('float32') / 255.\n",
+        "x_test = test_ds['image'].astype('float32') / 255.0\n",
         "y_test_indices = test_ds['label'][:, np.newaxis]\n",
         "\n",
         "# Convert class vectors to binary class matrices.\n",
@@ -215,7 +212,8 @@
         "input_shape = x_train.shape[1:]\n",
         "\n",
         "model = small_cnn(\n",
-    "    input_shape, num_classes, num_conv=num_conv, activation=activation)\n",
+        "    input_shape, num_classes, num_conv=num_conv, activation=activation\n",
+        ")\n",
         "\n",
         "print('learning rate %f', lr)\n",
         "\n",
@@ -230,14 +228,14 @@
         "    batch_size=batch_size,\n",
         "    epochs=epochs,\n",
         "    validation_data=(x_test, y_test),\n",
-    "    shuffle=True)\n",
+        "    shuffle=True,\n",
+        ")\n",
         "print('Finished training.')"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "ee-zjGGGV1DC"
       },
       "source": [
@@ -251,8 +249,6 @@
       "execution_count": null,
       "metadata": {
         "cellView": "both",
-    "colab": {},
-    "colab_type": "code",
         "id": "um9r0tSiPx4u"
       },
       "outputs": [],
@@ -270,14 +266,17 @@
         "cce = tf.keras.backend.categorical_crossentropy\n",
         "constant = tf.keras.backend.constant\n",
         "\n",
-    "loss_train = cce(constant(y_train), constant(prob_train), from_logits=False).numpy()\n",
+        "loss_train = cce(\n",
-    "loss_test = cce(constant(y_test), constant(prob_test), from_logits=False).numpy()"
+        "    constant(y_train), constant(prob_train), from_logits=False\n",
+        ").numpy()\n",
+        "loss_test = cce(\n",
+        "    constant(y_test), constant(prob_test), from_logits=False\n",
+        ").numpy()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "QETxVOHLiHP4"
       },
       "source": [
@@ -292,53 +291,48 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-    "colab": {},
-    "colab_type": "code",
         "id": "B8NIwhVwQT7I"
       },
       "outputs": [],
       "source": [
         "from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackInputData\n",
-    "from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SlicingSpec\n",
         "from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackType\n",
-    "\n",
+        "from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SlicingSpec\n",
         "import tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.plotting as plotting\n",
         "\n",
         "labels_train = np.argmax(y_train, axis=1)\n",
         "labels_test = np.argmax(y_test, axis=1)\n",
         "\n",
-    "input = AttackInputData(\n",
+        "attack_input = AttackInputData(\n",
-    "  logits_train = logits_train,\n",
+        "    logits_train=logits_train,\n",
-    "  logits_test = logits_test,\n",
+        "    logits_test=logits_test,\n",
-    "  loss_train = loss_train,\n",
+        "    loss_train=loss_train,\n",
-    "  loss_test = loss_test,\n",
+        "    loss_test=loss_test,\n",
-    "  labels_train = labels_train,\n",
+        "    labels_train=labels_train,\n",
-    "  labels_test = labels_test\n",
+        "    labels_test=labels_test,\n",
         ")\n",
         "\n",
         "# Run several attacks for different data slices\n",
-    "attacks_result = mia.run_attacks(input,\n",
+        "attacks_result = mia.run_attacks(\n",
-    "                                 SlicingSpec(\n",
+        "    attack_input=attack_input,\n",
-    "                                     entire_dataset = True,\n",
+        "    slicing_spec=SlicingSpec(\n",
-    "                                     by_class = True,\n",
+        "        entire_dataset=True, by_class=True, by_classification_correctness=True\n",
-    "                                     by_classification_correctness = True\n",
         "    ),\n",
-    "                                 attack_types = [\n",
+        "    attack_types=[AttackType.THRESHOLD_ATTACK, AttackType.LOGISTIC_REGRESSION],\n",
-    "                                     AttackType.THRESHOLD_ATTACK,\n",
+        ")\n",
-    "                                     AttackType.LOGISTIC_REGRESSION])\n",
         "\n",
         "# Plot the ROC curve of the best classifier\n",
         "fig = plotting.plot_roc_curve(\n",
-    "    attacks_result.get_result_with_max_auc().roc_curve)\n",
+        "    attacks_result.get_result_with_max_auc().roc_curve\n",
+        ")\n",
         "\n",
         "# Print a user-friendly summary of the attacks\n",
-    "print(attacks_result.summary(by_slices = True))"
+        "print(attacks_result.summary(by_slices=True))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-    "colab_type": "text",
         "id": "E9zwsPGFujVq"
       },
       "source": [
@@ -353,9 +347,8 @@
   ],
   "metadata": {
     "colab": {
-   "collapsed_sections": [],
       "last_runtime": {
-    "build_target": "//learning/deepmind/public/tools/ml_python:ml_notebook",
+        "build_target": "//learning/grp/tools/ml_python/gpu:ml_notebook",
         "kind": "private"
       },
       "name": "Membership inference codelab",
@@ -389,5 +382,5 @@
     }
   },
   "nbformat": 4,
- "nbformat_minor": 1
+  "nbformat_minor": 0
 }