diff --git a/tensorflow_privacy/privacy/membership_inference_attack/codelabs/codelab_privacy_risk_score.ipynb b/tensorflow_privacy/privacy/membership_inference_attack/codelabs/codelab_privacy_risk_score.ipynb deleted file mode 100644 index c271f41..0000000 --- a/tensorflow_privacy/privacy/membership_inference_attack/codelabs/codelab_privacy_risk_score.ipynb +++ /dev/null @@ -1,816 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1eiwVljWpzM7" - }, - "source": [ - "Copyright 2020 The TensorFlow Authors.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "both", - "colab": {}, - "colab_type": "code", - "id": "4rmwPgXeptiS" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YM2gRaJMqvMi" - }, - "source": [ - "# Assess privacy risks with TensorFlow Privacy Membership Inference Attacks" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-B5ZvlSqqLaR" - }, - "source": [ - "\n", - " \n", - " \n", - "
\n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9rMuytY7Nn8P" - }, - "source": [ - "##Overview\n", - "In this codelab we'll train a simple image classification model on the CIFAR10 dataset, and then use the \"membership inference attack\" against this model to assess if the attacker is able to \"guess\" whether a particular sample was present in the training set." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FUWqArj_q8vs" - }, - "source": [ - "## Setup\n", - "First, set this notebook's runtime to use a GPU, under Runtime > Change runtime type > Hardware accelerator. Then, begin importing the necessary libraries." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "Lr1pwHcbralz" - }, - "outputs": [], - "source": [ - "#@title Import statements.\n", - "import numpy as np\n", - "from typing import Tuple, Text\n", - "from scipy import special\n", - "\n", - "import tensorflow as tf\n", - "import tensorflow_datasets as tfds\n", - "\n", - "# Set verbosity.\n", - "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n", - "from warnings import simplefilter\n", - "from sklearn.exceptions import ConvergenceWarning\n", - "simplefilter(action=\"ignore\", category=ConvergenceWarning)\n", - "simplefilter(action=\"ignore\", category=FutureWarning)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ucw81ar6ru-6" - }, - "source": [ - "### Install TensorFlow Privacy." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "cellView": "both", - "colab": {}, - "colab_type": "code", - "id": "zcqAmiGH90kl" - }, - "outputs": [], - "source": [ - "!pip3 install git+https://github.com/tensorflow/privacy\n", - "\n", - "from tensorflow_privacy.privacy.membership_inference_attack import membership_inference_attack as mia" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pBbcG86th_sW" - }, - "source": [ - "## Train a model" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "vCyOWyyhXLib" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading the dataset.\n", - "learning rate %f 0.02\n", - "Model: \"sequential\"\n", - "_________________________________________________________________\n", - "Layer (type) Output Shape Param # \n", - "=================================================================\n", - "conv2d (Conv2D) (None, 30, 30, 32) 896 \n", - "_________________________________________________________________\n", - "max_pooling2d (MaxPooling2D) (None, 15, 15, 32) 0 \n", - "_________________________________________________________________\n", - "conv2d_1 (Conv2D) (None, 13, 13, 32) 9248 \n", - "_________________________________________________________________\n", - "max_pooling2d_1 (MaxPooling2 (None, 6, 6, 32) 0 \n", - "_________________________________________________________________\n", - "conv2d_2 (Conv2D) (None, 4, 4, 32) 9248 \n", - "_________________________________________________________________\n", - "max_pooling2d_2 (MaxPooling2 (None, 2, 2, 32) 0 \n", - "_________________________________________________________________\n", - "flatten (Flatten) (None, 128) 0 \n", - "_________________________________________________________________\n", - "dense (Dense) (None, 64) 8256 \n", - "_________________________________________________________________\n", - "dense_1 (Dense) (None, 10) 650 \n", - "=================================================================\n", - "Total params: 28,298\n", - "Trainable params: 28,298\n", - "Non-trainable params: 0\n", - "_________________________________________________________________\n", - "Epoch 1/100\n", - "200/200 [==============================] - 2s 8ms/step - loss: 2.0185 - accuracy: 0.2515 - val_loss: 1.8635 - val_accuracy: 0.3168\n", - "Epoch 2/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 1.6232 - accuracy: 0.4059 - val_loss: 1.4847 - val_accuracy: 0.4549\n", - "Epoch 3/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 1.4421 - accuracy: 0.4752 - val_loss: 1.3781 - val_accuracy: 0.5041\n", - "Epoch 4/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 1.3402 - accuracy: 0.5152 - val_loss: 1.2500 - val_accuracy: 0.5520\n", - "Epoch 5/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 1.2316 - accuracy: 0.5614 - val_loss: 1.2739 - val_accuracy: 0.5524\n", - "Epoch 6/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 1.1568 - accuracy: 0.5899 - val_loss: 1.2040 - val_accuracy: 0.5748\n", - "Epoch 7/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 1.1007 - accuracy: 0.6094 - val_loss: 1.1218 - val_accuracy: 0.6042\n", - "Epoch 8/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 1.0437 - accuracy: 0.6313 - val_loss: 1.0968 - val_accuracy: 0.6192\n", - "Epoch 9/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.9965 - accuracy: 0.6489 - val_loss: 1.0501 - val_accuracy: 0.6338\n", - "Epoch 10/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.9673 - accuracy: 0.6589 - val_loss: 1.0594 - val_accuracy: 0.6322\n", - "Epoch 11/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.9388 - accuracy: 0.6711 - val_loss: 1.0302 - val_accuracy: 0.6445\n", - "Epoch 12/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.9104 - accuracy: 0.6800 - val_loss: 0.9907 - val_accuracy: 0.6553\n", - "Epoch 13/100\n", - "200/200 [==============================] - 1s 6ms/step - loss: 0.8827 - accuracy: 0.6896 - val_loss: 0.9999 - val_accuracy: 0.6509\n", - "Epoch 14/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.8453 - accuracy: 0.7023 - val_loss: 0.9708 - val_accuracy: 0.6674\n", - "Epoch 15/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.8407 - accuracy: 0.7067 - val_loss: 0.9434 - val_accuracy: 0.6739\n", - "Epoch 16/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.8152 - accuracy: 0.7136 - val_loss: 0.9440 - val_accuracy: 0.6786\n", - "Epoch 17/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.7988 - accuracy: 0.7184 - val_loss: 0.9670 - val_accuracy: 0.6710\n", - "Epoch 18/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.7763 - accuracy: 0.7270 - val_loss: 0.9224 - val_accuracy: 0.6854\n", - "Epoch 19/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.7650 - accuracy: 0.7307 - val_loss: 0.9305 - val_accuracy: 0.6832\n", - "Epoch 20/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.7508 - accuracy: 0.7354 - val_loss: 0.9674 - val_accuracy: 0.6707\n", - "Epoch 21/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.7306 - accuracy: 0.7410 - val_loss: 0.9122 - val_accuracy: 0.6917\n", - "Epoch 22/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.7142 - accuracy: 0.7498 - val_loss: 0.9287 - val_accuracy: 0.6868\n", - "Epoch 23/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.7071 - accuracy: 0.7514 - val_loss: 0.9046 - val_accuracy: 0.6934\n", - "Epoch 24/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6923 - accuracy: 0.7564 - val_loss: 0.9136 - val_accuracy: 0.6908\n", - "Epoch 25/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6791 - accuracy: 0.7603 - val_loss: 0.9856 - val_accuracy: 0.6702\n", - "Epoch 26/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6711 - accuracy: 0.7637 - val_loss: 0.9372 - val_accuracy: 0.6865\n", - "Epoch 27/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6556 - accuracy: 0.7672 - val_loss: 0.9847 - val_accuracy: 0.6768\n", - "Epoch 28/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6497 - accuracy: 0.7714 - val_loss: 0.9554 - val_accuracy: 0.6881\n", - "Epoch 29/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6311 - accuracy: 0.7765 - val_loss: 0.9962 - val_accuracy: 0.6801\n", - "Epoch 30/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6291 - accuracy: 0.7773 - val_loss: 0.9268 - val_accuracy: 0.6926\n", - "Epoch 31/100\n", - "200/200 [==============================] - 1s 6ms/step - loss: 0.6175 - accuracy: 0.7802 - val_loss: 0.9507 - val_accuracy: 0.6904\n", - "Epoch 32/100\n", - "200/200 [==============================] - 1s 6ms/step - loss: 0.6107 - accuracy: 0.7830 - val_loss: 0.9776 - val_accuracy: 0.6799\n", - "Epoch 33/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.6049 - accuracy: 0.7877 - val_loss: 0.9712 - val_accuracy: 0.6897\n", - "Epoch 34/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5963 - accuracy: 0.7884 - val_loss: 0.9548 - val_accuracy: 0.6889\n", - "Epoch 35/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5959 - accuracy: 0.7881 - val_loss: 0.9729 - val_accuracy: 0.6865\n", - "Epoch 36/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5801 - accuracy: 0.7955 - val_loss: 0.9659 - val_accuracy: 0.6949\n", - "Epoch 37/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5745 - accuracy: 0.7981 - val_loss: 0.9663 - val_accuracy: 0.6908\n", - "Epoch 38/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5651 - accuracy: 0.7993 - val_loss: 0.9689 - val_accuracy: 0.6931\n", - "Epoch 39/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5608 - accuracy: 0.8014 - val_loss: 0.9899 - val_accuracy: 0.6894\n", - "Epoch 40/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5507 - accuracy: 0.8049 - val_loss: 0.9990 - val_accuracy: 0.6888\n", - "Epoch 41/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5409 - accuracy: 0.8066 - val_loss: 0.9860 - val_accuracy: 0.6904\n", - "Epoch 42/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5494 - accuracy: 0.8040 - val_loss: 0.9937 - val_accuracy: 0.6916\n", - "Epoch 43/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5226 - accuracy: 0.8146 - val_loss: 0.9943 - val_accuracy: 0.6888\n", - "Epoch 44/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5214 - accuracy: 0.8148 - val_loss: 1.0146 - val_accuracy: 0.6826\n", - "Epoch 45/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5288 - accuracy: 0.8126 - val_loss: 1.0247 - val_accuracy: 0.6926\n", - "Epoch 46/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5182 - accuracy: 0.8149 - val_loss: 1.0246 - val_accuracy: 0.6883\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 47/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5079 - accuracy: 0.8190 - val_loss: 1.0530 - val_accuracy: 0.6888\n", - "Epoch 48/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5088 - accuracy: 0.8188 - val_loss: 1.0607 - val_accuracy: 0.6876\n", - "Epoch 49/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4989 - accuracy: 0.8218 - val_loss: 1.0523 - val_accuracy: 0.6858\n", - "Epoch 50/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.5042 - accuracy: 0.8200 - val_loss: 1.0645 - val_accuracy: 0.6898\n", - "Epoch 51/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4800 - accuracy: 0.8292 - val_loss: 1.0762 - val_accuracy: 0.6812\n", - "Epoch 52/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4853 - accuracy: 0.8262 - val_loss: 1.0960 - val_accuracy: 0.6828\n", - "Epoch 53/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4754 - accuracy: 0.8308 - val_loss: 1.0551 - val_accuracy: 0.6916\n", - "Epoch 54/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4745 - accuracy: 0.8284 - val_loss: 1.1048 - val_accuracy: 0.6768\n", - "Epoch 55/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4770 - accuracy: 0.8309 - val_loss: 1.0978 - val_accuracy: 0.6893\n", - "Epoch 56/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4708 - accuracy: 0.8311 - val_loss: 1.1025 - val_accuracy: 0.6791\n", - "Epoch 57/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4577 - accuracy: 0.8366 - val_loss: 1.1247 - val_accuracy: 0.6792\n", - "Epoch 58/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4693 - accuracy: 0.8321 - val_loss: 1.1224 - val_accuracy: 0.6808\n", - "Epoch 59/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4533 - accuracy: 0.8385 - val_loss: 1.1161 - val_accuracy: 0.6830\n", - "Epoch 60/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4602 - accuracy: 0.8326 - val_loss: 1.1262 - val_accuracy: 0.6781\n", - "Epoch 61/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4528 - accuracy: 0.8379 - val_loss: 1.2267 - val_accuracy: 0.6654\n", - "Epoch 62/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4533 - accuracy: 0.8354 - val_loss: 1.1433 - val_accuracy: 0.6901\n", - "Epoch 63/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4373 - accuracy: 0.8418 - val_loss: 1.1481 - val_accuracy: 0.6857\n", - "Epoch 64/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4442 - accuracy: 0.8391 - val_loss: 1.1446 - val_accuracy: 0.6854\n", - "Epoch 65/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4247 - accuracy: 0.8480 - val_loss: 1.1511 - val_accuracy: 0.6856\n", - "Epoch 66/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4395 - accuracy: 0.8406 - val_loss: 1.1960 - val_accuracy: 0.6791\n", - "Epoch 67/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4402 - accuracy: 0.8394 - val_loss: 1.2087 - val_accuracy: 0.6852\n", - "Epoch 68/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4247 - accuracy: 0.8464 - val_loss: 1.1801 - val_accuracy: 0.6837\n", - "Epoch 69/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4219 - accuracy: 0.8460 - val_loss: 1.2674 - val_accuracy: 0.6683\n", - "Epoch 70/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4184 - accuracy: 0.8494 - val_loss: 1.2206 - val_accuracy: 0.6828\n", - "Epoch 71/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4116 - accuracy: 0.8505 - val_loss: 1.1856 - val_accuracy: 0.6782\n", - "Epoch 72/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4177 - accuracy: 0.8481 - val_loss: 1.2790 - val_accuracy: 0.6791\n", - "Epoch 73/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4135 - accuracy: 0.8505 - val_loss: 1.2457 - val_accuracy: 0.6806\n", - "Epoch 74/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4046 - accuracy: 0.8528 - val_loss: 1.2291 - val_accuracy: 0.6852\n", - "Epoch 75/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4132 - accuracy: 0.8500 - val_loss: 1.2248 - val_accuracy: 0.6866\n", - "Epoch 76/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4116 - accuracy: 0.8501 - val_loss: 1.2619 - val_accuracy: 0.6793\n", - "Epoch 77/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4146 - accuracy: 0.8500 - val_loss: 1.2497 - val_accuracy: 0.6780\n", - "Epoch 78/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3922 - accuracy: 0.8579 - val_loss: 1.2788 - val_accuracy: 0.6718\n", - "Epoch 79/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4084 - accuracy: 0.8499 - val_loss: 1.2568 - val_accuracy: 0.6876\n", - "Epoch 80/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3976 - accuracy: 0.8559 - val_loss: 1.3637 - val_accuracy: 0.6652\n", - "Epoch 81/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.4061 - accuracy: 0.8511 - val_loss: 1.2873 - val_accuracy: 0.6775\n", - "Epoch 82/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3781 - accuracy: 0.8623 - val_loss: 1.3062 - val_accuracy: 0.6756\n", - "Epoch 83/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3825 - accuracy: 0.8606 - val_loss: 1.2976 - val_accuracy: 0.6825\n", - "Epoch 84/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3913 - accuracy: 0.8571 - val_loss: 1.4069 - val_accuracy: 0.6528\n", - "Epoch 85/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3876 - accuracy: 0.8591 - val_loss: 1.3395 - val_accuracy: 0.6753\n", - "Epoch 86/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3879 - accuracy: 0.8580 - val_loss: 1.3092 - val_accuracy: 0.6741\n", - "Epoch 87/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3695 - accuracy: 0.8665 - val_loss: 1.3327 - val_accuracy: 0.6762\n", - "Epoch 88/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3835 - accuracy: 0.8608 - val_loss: 1.3579 - val_accuracy: 0.6775\n", - "Epoch 89/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3816 - accuracy: 0.8619 - val_loss: 1.3944 - val_accuracy: 0.6622\n", - "Epoch 90/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3804 - accuracy: 0.8609 - val_loss: 1.3264 - val_accuracy: 0.6854\n", - "Epoch 91/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3718 - accuracy: 0.8647 - val_loss: 1.3646 - val_accuracy: 0.6713\n", - "Epoch 92/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3676 - accuracy: 0.8661 - val_loss: 1.3926 - val_accuracy: 0.6759\n", - "Epoch 93/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3767 - accuracy: 0.8623 - val_loss: 1.3605 - val_accuracy: 0.6701\n", - "Epoch 94/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3813 - accuracy: 0.8612 - val_loss: 1.3938 - val_accuracy: 0.6659\n", - "Epoch 95/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3631 - accuracy: 0.8667 - val_loss: 1.4130 - val_accuracy: 0.6749\n", - "Epoch 96/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3604 - accuracy: 0.8694 - val_loss: 1.3780 - val_accuracy: 0.6832\n", - "Epoch 97/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3657 - accuracy: 0.8666 - val_loss: 1.4425 - val_accuracy: 0.6719\n", - "Epoch 98/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3726 - accuracy: 0.8636 - val_loss: 1.4077 - val_accuracy: 0.6699\n", - "Epoch 99/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3671 - accuracy: 0.8663 - val_loss: 1.4207 - val_accuracy: 0.6769\n", - "Epoch 100/100\n", - "200/200 [==============================] - 1s 5ms/step - loss: 0.3529 - accuracy: 0.8706 - val_loss: 1.4817 - val_accuracy: 0.6716\n", - "Finished training.\n" - ] - } - ], - "source": [ - "#@markdown Train a simple model on CIFAR10 with Keras.\n", - "\n", - "dataset = 'cifar10'\n", - "num_classes = 10\n", - "num_conv = 3\n", - "activation = 'relu'\n", - "lr = 0.02\n", - "momentum = 0.9\n", - "batch_size = 250\n", - "epochs = 100 # Privacy risks are especially visible with lots of epochs.\n", - "\n", - "\n", - "def small_cnn(input_shape: Tuple[int],\n", - " num_classes: int,\n", - " num_conv: int,\n", - " activation: Text = 'relu') -> tf.keras.models.Sequential:\n", - " \"\"\"Setup a small CNN for image classification.\n", - "\n", - " Args:\n", - " input_shape: Integer tuple for the shape of the images.\n", - " num_classes: Number of prediction classes.\n", - " num_conv: Number of convolutional layers.\n", - " activation: The activation function to use for conv and dense layers.\n", - "\n", - " Returns:\n", - " The Keras model.\n", - " \"\"\"\n", - " model = tf.keras.models.Sequential()\n", - " model.add(tf.keras.layers.Input(shape=input_shape))\n", - "\n", - " # Conv layers\n", - " for _ in range(num_conv):\n", - " model.add(tf.keras.layers.Conv2D(32, (3, 3), activation=activation))\n", - " model.add(tf.keras.layers.MaxPooling2D())\n", - "\n", - " model.add(tf.keras.layers.Flatten())\n", - " model.add(tf.keras.layers.Dense(64, activation=activation))\n", - " model.add(tf.keras.layers.Dense(num_classes))\n", - " return model\n", - "\n", - "\n", - "print('Loading the dataset.')\n", - "train_ds = tfds.as_numpy(\n", - " tfds.load(dataset, split=tfds.Split.TRAIN, batch_size=-1))\n", - "test_ds = tfds.as_numpy(\n", - " tfds.load(dataset, split=tfds.Split.TEST, batch_size=-1))\n", - "x_train = train_ds['image'].astype('float32') / 255.\n", - "y_train_indices = train_ds['label'][:, np.newaxis]\n", - "x_test = test_ds['image'].astype('float32') / 255.\n", - "y_test_indices = test_ds['label'][:, np.newaxis]\n", - "\n", - "# Convert class vectors to binary class matrices.\n", - "y_train = tf.keras.utils.to_categorical(y_train_indices, num_classes)\n", - "y_test = tf.keras.utils.to_categorical(y_test_indices, num_classes)\n", - "\n", - "input_shape = x_train.shape[1:]\n", - "\n", - "model = small_cnn(\n", - " input_shape, num_classes, num_conv=num_conv, activation=activation)\n", - "\n", - "print('learning rate %f', lr)\n", - "\n", - "optimizer = tf.keras.optimizers.SGD(lr=lr, momentum=momentum)\n", - "\n", - "loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n", - "model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])\n", - "model.summary()\n", - "model.fit(\n", - " x_train,\n", - " y_train,\n", - " batch_size=batch_size,\n", - " epochs=epochs,\n", - " validation_data=(x_test, y_test),\n", - " shuffle=True)\n", - "print('Finished training.')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ee-zjGGGV1DC" - }, - "source": [ - "## Calculate logits, probabilities and loss values for training and test sets.\n", - "\n", - "We will use these values later in the membership inference attack to separate training and test samples." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "cellView": "both", - "colab": {}, - "colab_type": "code", - "id": "um9r0tSiPx4u" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Predict on train...\n", - "Predict on test...\n", - "Apply softmax to get probabilities from logits...\n", - "Compute losses...\n" - ] - } - ], - "source": [ - "print('Predict on train...')\n", - "logits_train = model.predict(x_train, batch_size=batch_size)\n", - "print('Predict on test...')\n", - "logits_test = model.predict(x_test, batch_size=batch_size)\n", - "\n", - "print('Apply softmax to get probabilities from logits...')\n", - "prob_train = special.softmax(logits_train, axis=1)\n", - "prob_test = special.softmax(logits_test, axis=1)\n", - "\n", - "print('Compute losses...')\n", - "cce = tf.keras.backend.categorical_crossentropy\n", - "constant = tf.keras.backend.constant\n", - "\n", - "loss_train = cce(constant(y_train), constant(prob_train), from_logits=False).numpy()\n", - "loss_test = cce(constant(y_test), constant(prob_test), from_logits=False).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QETxVOHLiHP4" - }, - "source": [ - "## Run membership inference attacks.\n", - "\n", - "We will now execute a membership inference attack against the previously trained CIFAR10 model. This will generate a number of scores, most notably, attacker advantage and AUC for the membership inference classifier.\n", - "\n", - "An AUC of close to 0.5 means that the attack wasn't able to identify training samples, which means that the model doesn't have privacy issues according to this test. Higher values, on the contrary, indicate potential privacy issues." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "B8NIwhVwQT7I" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best-performing attacks over all slices\n", - " THRESHOLD_ATTACK achieved an AUC of 0.74 on slice CORRECTLY_CLASSIFIED=False\n", - " THRESHOLD_ATTACK achieved an advantage of 0.37 on slice CORRECTLY_CLASSIFIED=False\n", - "\n", - "Best-performing attacks over slice: \"Entire dataset\"\n", - " THRESHOLD_ENTROPY_ATTACK achieved an AUC of 0.60\n", - " THRESHOLD_ATTACK achieved an advantage of 0.20\n", - "\n", - "Best-performing attacks over slice: \"CLASS=0\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.64\n", - " LOGISTIC_REGRESSION achieved an advantage of 0.24\n", - "\n", - "Best-performing attacks over slice: \"CLASS=1\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.57\n", - " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.16\n", - "\n", - "Best-performing attacks over slice: \"CLASS=2\"\n", - " THRESHOLD_ENTROPY_ATTACK achieved an AUC of 0.64\n", - " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.26\n", - "\n", - "Best-performing attacks over slice: \"CLASS=3\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.68\n", - " LOGISTIC_REGRESSION achieved an advantage of 0.29\n", - "\n", - "Best-performing attacks over slice: \"CLASS=4\"\n", - " THRESHOLD_ENTROPY_ATTACK achieved an AUC of 0.64\n", - " THRESHOLD_ATTACK achieved an advantage of 0.24\n", - "\n", - "Best-performing attacks over slice: \"CLASS=5\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.63\n", - " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.23\n", - "\n", - "Best-performing attacks over slice: \"CLASS=6\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.63\n", - " LOGISTIC_REGRESSION achieved an advantage of 0.21\n", - "\n", - "Best-performing attacks over slice: \"CLASS=7\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.60\n", - " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.21\n", - "\n", - "Best-performing attacks over slice: \"CLASS=8\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.60\n", - " LOGISTIC_REGRESSION achieved an advantage of 0.19\n", - "\n", - "Best-performing attacks over slice: \"CLASS=9\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.62\n", - " LOGISTIC_REGRESSION achieved an advantage of 0.20\n", - "\n", - "Best-performing attacks over slice: \"CORRECTLY_CLASSIFIED=True\"\n", - " LOGISTIC_REGRESSION achieved an AUC of 0.50\n", - " THRESHOLD_ATTACK achieved an advantage of 0.04\n", - "\n", - "Best-performing attacks over slice: \"CORRECTLY_CLASSIFIED=False\"\n", - " THRESHOLD_ATTACK achieved an AUC of 0.74\n", - " THRESHOLD_ATTACK achieved an advantage of 0.37\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0xklEQVR4nO3deXhN1/7H8feSiJjnOQgNNSQxpVQpRfWntNVWKbe3kxNjDS2X0mou2qpWJ2NRWqVXVF1jRWnNlEbIdMypKSElIsbIvH5/JHITGQQ5Z5+T8309T5/us/eS89kR55u19tprK601QgghHFcxowMIIYQwlhQCIYRwcFIIhBDCwUkhEEIIByeFQAghHJyz0QHuVZUqVbS7u7vRMYQQwq4cOHDgkta6am7H7K4QuLu7ExQUZHQMIYSwK0qpM3kdk6EhIYRwcFIIhBDCwUkhEEIIB2d31whyk5ycTFRUFAkJCUZHKfJcXV1xc3OjePHiRkcRQhSSIlEIoqKiKFu2LO7u7iiljI5TZGmtiY2NJSoqivr16xsdRwhRSCw2NKSU+k4pdVEpZc7juFJKzVRKRSilwpRSre73vRISEqhcubIUAQtTSlG5cmXpeQlRxFjyGsFioHs+x58GGmb8Nwj45kHeTIqAdcj3WYiix2JDQ1rrnUop93ya9AKW6PR1sPcppSoopWpqraMtlUkIIezNsb+vM+6/YVy5dp1eresz+qmHC/09jJw1VBuIzPI6KmNfDkqpQUqpIKVUUExMjFXC3Y81a9aglOLo0aOZ+7Zv384zzzyTrd0bb7zBypUrgfQL3ePHj6dhw4a0atWKdu3asXHjxnzfJzExkZdffhkPDw/atm3L6dOnc7Q5duwYLVq0yPyvXLlyfP311wB88MEHeHt706JFC5566inOnz8PQFxcHC+88ALe3t60adMGsznXUT0hhIVdvpmE31oz7uM38H9f7yQ08gpnrqayMijPe8IeiF1MH9VaL9Ba+2itfapWzfUOaZvg7+9Phw4d8Pf3L/Cf+eCDD4iOjsZsNnPw4EHWrFnD9evX8/0zixYtomLFikRERPDOO+/w7rvv5mjz8MMPExISQkhICAcOHKBUqVK88MILAIwdO5awsDBCQkJ45plnmDJlCgBTp06lRYsWhIWFsWTJEkaNGnUPZy+EuFdaa24kprDt2EU+WGPmmVm7cB+/gVYf/saSvdk/9NvVr8C819pYJIeRs4bOAXWyvHbL2GeXbty4we7du9m2bRvPPvsskydPvuufiY+P59tvv+XUqVOUKFECgOrVq9O3b998/9zatWuZNGkSAC+99BLDhw9Ha53n+P2WLVt46KGHqFevHgDlypXLPHbz5s3MP3f48GHGjx8PQOPGjTl9+jQXLlygevXqdz0XIUTBxCelsOrgOb7YfIy4+OR826rTf/L5G53p3b2zRTMZWQjWAcOVUsuBtsDVwrg+4D5+wwMHy83paT3zPb527Vq6d+9Oo0aNqFy5MgcOHKB169b5/pmIiAjq1q2b7YM5K19fX4YMGYKPj0+2/efOnaNOnfQa6uzsTPny5YmNjaVKlSq5fp3ly5fTv3//bPvef/99lixZQvny5dm2bRsAzZs3Z9WqVTz++OMEBgZy5swZoqKipBAI8QASU1LZfeIS60PPsybkfJ7tnIopnmpaHafYk6ycPYXXX+zBpO8nUbJkSYtntFghUEr5A08AVZRSUcC/geIAWut5QADQA4gA4oE3LZXFGvz9/TOHUvr164e/vz+tW7fO87f0gsy+Wbhw4QPnSkpKYt26dXzyySfZ9n/88cd8/PHHfPLJJ8yePZvJkyczfvx4Ro0aRYsWLfDy8qJly5Y4OTk9cAYhHIXWmvBzVzlwJo7Iy7f48c8zJKWk5dr2sYcq84h7JUZ08cDZqRiRkZEMGTKEyMhIfvH/PscvgJZkyVlD/e9yXANvFfb73u03d0u4fPkyW7duJTw8HKUUqampKKWYPn06lStXJi4uLkf7KlWq4OHhwdmzZ7l27VqevYLc1K5dm8jISNzc3EhJSeHq1atUrlw517YbN26kVatWef5W/8orr9CjRw8mT55MuXLl+P7774H0H+j69evToEGDAucSwtGkpmmir95i94lL+K09RFJq7h/6AGVdnenVohY9PGvymMf/eu9paWl88803+Pn5MWrUKFavXo2Li4s14mcqEncWG23lypW8+uqrzJ8/P3Nfp06d2LVrF23btuX8+fMcOXKEJk2acObMGUJDQ2nRogWlSpXCZDIxatQo5s+fj4uLCzExMWzfvp0+ffrk+X7PPfccP/zwA+3atWPlypV06dIlzx6Gv79/jmGhEydO0LBhQyB9SKtx48YAXLlyhVKlSuHi4sLChQvp2LHjPRUoIRyB1pqZWyL46vfj+bZrXqcCHlXL8HCNMgxoXx9np5xzc44fP46vry/Jycns2LGDpk2bWip2vqQQFAJ/f/8cM3d69+6Nv78/HTt25Mcff+TNN98kISGB4sWLs3DhQsqXLw/ARx99xMSJE2natCmurq6ULl06cxZPXtcITCYTr776Kh4eHlSqVInly5cDcP78eXx9fQkICADSLwT/9ttv2QoUwPjx4zl27BjFihWjXr16zJs3D4AjR47w+uuvo5SiWbNmLFq0qPC/WULYIa01R6Kv8/nmY2w9ejHHcadiimplS9DDqybjuj9MCef8h1RTUlL48ssv+eyzz/jggw8YPny4ocOwKn2Exn74+PjoOx9Mc/u3bWEd8v0WRV1SShpbj15k5pYTHI6+lme7Kb2a0b9NXYrn8tt+XkJDQxkwYACVKlViwYIFVlu3Syl1QGud64UH6REIIUSG6Ku3eOXbPzl56Waebdp7VGbOP1pRodS9jeMnJiby0UcfMX/+fKZNm8abb75pM0u2SCEQQjikawnJHDwTR8TFG+w6cYkdx3OuWuBWsSQ9vWsyrJMH5Uvd/9Lre/fuxWQyZd7oWatWrQeJXuiKTCHI74YqUXjsbShRiNQ0TWjUFb7fc5rAU7FcuJZIKRcn4pNS8/wz73ZvzJBODR74M+XGjRtMnDiRFStWMHPmTHr37m2Tn1NFohC4uroSGxsrS1Fb2O3nEbi6uhodRYg87f0rlh/3nWFDeDRVy5Yg5npijjbxSakoBdXKpt/R/4x3LWqUc+XZ5rWoUb5wfr5/++03Bg0aRMeOHQkPD89zirctKBKFwM3NjaioKGx5Qbqi4vYTyoSwFSmpaawJOc+KoEgCT13OdixrEXAtXoyHa5RjeGcPPGuXo3LpErg4F/5ya3FxcYwZM4YtW7Ywf/58unfPbzV+21AkCkHx4sXliVlCOIiklDT+PBXLfw9EsfPEJS7fTMq1XeeHq/JaO3ea16lAxVLFrTJasHr1aoYPH84LL7yA2WymbNmyFn/PwlAkCoEQomhLTEkl5OwVXl6wL882bhVL0r9NXfr61KFqxpCPtfz999+MGDGCsLAwli9fzuOPP27V939QUgiEEDYlNU1zJT6JgPBoNh26wO6IS7m2a1arHPFJqUx70Yu2DYwZf9das3TpUsaOHcuAAQNYunSpXV5Dk0IghDBUappmQ3g05nNX+f3whXzn8Ls4F2PuP1rxZFPjV8Q9c+YMgwcP5sKFC5lretkrKQRCCKuJT0ohJPIKa4PP81NQZL5ty7o6cz0hhVFdG/J/zWrQtJZtrHuVlpbG3LlzmTRpEmPGjOFf//oXxYvf/z0GtkAKgRDCYv6+msB//jzD/J0n81yOGUAp0Bom9myCa3EnenjVpFJp667AWRDHjh3DZDKhtWb37t2ZCzbaOykEQohCk5SSxrwdf3H072sEhP+dZzuv2uVpVqscvo834KGqpW3+/p/k5GQ+//xzvvjiCyZNmsSwYcMoVswunvRbIFIIhBD3RWvN9mMxzN0eQXxSKofO5704m4tTMd7v2YTnW9R+oKUajBAcHIzJZKJq1aoEBQXh7u5udKRCJ4VACHFPbiSmMG5laL6/8QP0buXGgA7uNKtV3krJCldCQgJTpkxh4cKFTJ8+nddee83mey73SwqBEOKuzsTeZOzPYUTGxRN9NSHH8Z5eNXmrswe1K5aknKuz3X9g7t69G19fX7y8vAgLC6NGjRpGR7IoKQRCiFylpmlG+gezITw61+Nt61diwas+djfUk5/r168zYcIEVq1axezZs3nxxReNjmQVUgiEENkcv3CdDWHRzNhyIsexxx6qzHs9mtC4RtlcH71ozzZt2sTgwYPp0qULhw4domLFikZHshopBEIILl5PYPqvx/j5QFSOY8WdFAtff4ROjaoakMzyLl++zDvvvMOOHTtYsGABTz31lNGRrE4KgRAOJjk1jQNn4th5PIYzl+PZeuQit5Jzrs3fwaMK459ujGdt+7zYWxArV65k5MiR9OnTB7PZTJkyZYyOZAgpBEIUcVprthy5yJajF/EPPJtvW596FRnexYMnHq5mpXTGiI6O5q233uLIkSP8/PPPtG/f3uhIhpJCIEQRFJ+Uwor9kUxafzjfdo83rELjGmXp0rg67R6y3QenFBatNYsXL+bdd99l0KBBLFu2zC4XiStsUgiEKCJSUtM4fuEGPWbuyvV4c7fyDOhQn4bVytKkZlm7n+J5r06dOsWgQYO4fPkymzdvpkWLFkZHshlSCISwU2lpmr9ibmA+f5X1odFsPXoxR5uHq5elj48bbzzmXuRm+RRUamoqc+bMYcqUKYwdO5YxY8bg7CwffVnJd0MIO3IlPgnzuWsM9z/IlfjkXNuUcnGiY8OqzOjfghLOTlZOaFuOHDmCyWTC2dmZP/74g0aNGhkdySZJIRDChsVcT2SE/0Hik1IJi7qaZztvt/I8612LAR3q41TMsYZ8cpOcnMxnn33GV199xZQpUxgyZEiRWiSusEkhEMJGXIlPYvuxGL787ThnL8fn2/bRBpVISdX8NLidfPDf4cCBAwwYMIBatWpx8OBB6tata3QkmyeFQAgDnYy5wcQ1Zi7dSOT4hRu5tqlSxoXOD1fjH23r0rRWOYcf7snLrVu3mDRpEosXL+aLL77glVdecbgL4vdLCoEQVnT072uM9A8m4uIN0nTubUo4F6Nl3QqM7vYwzWqVo3QJ+Wd6Nzt37sTX15eWLVsSHh5OtWpF+z6IwmbRnzClVHdgBuAELNRaT7vjeF3gB6BCRpvxWusAS2YSwtpS0zQB4dHM3/kX5nO5r9nfqVFVJj/XDPcqpa2czr5du3aN8ePHs27dOmbPns3zzz9vdCS7ZLFCoJRyAuYA3YAoYL9Sap3WOusdLhOBFVrrb5RSTYEAwN1SmYSwpribSfScuYvzuSzb/GSTaozu9jB1KpWkrGvRWb3TmgICAhgyZAhPPfUUZrOZChUqGB3JblmyR9AGiNBanwRQSi0HegFZC4EGbj+Rujxw3oJ5hLCK8Kir+C7Zz4VriTmOvd+jCQM7NjAgVdFx6dIl3nnnHfbs2cP3339P165djY5k9yxZCGoDkVleRwFt72gzCdislBoBlAaezO0LKaUGAYMAmQEgbNK60POM9A/O9dj0l7x5qbWbXLh8QFprVqxYwdtvv03//v0JDw+ndGkZSisMRl+F6g8s1lp/oZRqByxVSnlqrdOyNtJaLwAWAPj4+ORxiU0I69Fas+N4DJ/9eozD0TnH/SuWKk6/NnUZ3a0RxR30jt7CdP78eYYOHUpERASrV6/m0UcfNTpSkWLJQnAOqJPltVvGvqxMQHcArfVepZQrUAXIea+8EDbgSPQ15m7/i/WhOUcxG1Yrw8cveNGybgX58C8kWmsWLVrEhAkTGDZsGCtWrKBEiRJGxypyLFkI9gMNlVL1SS8A/YB/3NHmLNAVWKyUagK4AjEWzCTEPUtL06wNPcc7P4XmevzFlrX56AVPSrkY3cEuWk6ePMnAgQO5du0aW7Zswdvb2+hIRZbFfnK11ilKqeHAJtKnhn6ntT6klJoCBGmt1wFjgG+VUu+QfuH4Da21DP0Im6C1ZqP5b4b952COYy+0rM2ILh40qOqYDzKxpNTUVGbOnMnHH3/M+PHjefvtt2WROAuz6Hc3456AgDv2+WXZPgw49hMhhE3RWvNzUBQ//nkm17V9Pu/TnJdauxmQzDEcOnQIk8mEq6sr+/btw8PDw+hIDkHKrBDAnydjmbjGzImLuS/zsHJIO3zcK1k5leNISkpi2rRpzJo1i48++oiBAwfKInFWJIVAOKy0NM22YxeZsCqci9ezz/kvpmDRG4/Q/qEquDjLB5Il7d+/nwEDBlCvXj2Cg4Nxc5Mel7VJIRAO53pCMr1m7+HkpZs5jn3Yqxn/fLSezPm3gvj4ePz8/Pjxxx/56quv6Nevn3zfDSKFQDiUNcHnePunkBz7J/ZswoD29SkmSzpbxfbt2/H19aVNmzaEh4dTtWpVoyM5NCkEosiLu5nEL2Hn+WDtoWz7B3dswPinG8tvoVZ09epVxo0bR0BAAHPnzuXZZ581OpJACoEowq7eSuafC/8k/FzO2T8hft2oUMrFgFSO65dffmHo0KH06NEDs9lM+fLljY4kMkghEEXO31cTePSTLTn2j+ziQdcm1Wlep4L1QzmwmJgYRo0aRWBgIEuWLKFz585GRxJ3kEIgiozUNM3ENeH4B0Zm2/9og0osfrMNrsXlyV7WpLXG39+f0aNH8+qrrxIWFkapUqWMjiVyIYVA2L3Iy/F0/WIHSanZ1irE75mmvNneXa4BGCAqKoqhQ4dy+vRp1q1bR5s2bYyOJPIhhUDYresJyTwxfTuxN5Oy7W9dryI/DGhDGXnEo9WlpaXx7bffMnHiREaMGMF///tfXFzkWoytk38pwu5EXLzOB2sOsfdkbLb9bzzmzoQejeXh7gaJiIhg4MCBxMfHs23bNjw9PY2OJApICoGwG+FRV3l29u4c+3u3cuPzPt4yBGSQlJQUvv76a6ZNm8b777/PyJEjcXKSYmxPpBAIu7Bk72n87rgPoEvjaszq35LSMgRkmPDwcEwmE2XLluXPP//koYceMjqSuA/yL0jYtMSUVFpO+Y34pNTMfV+93JwXWsp6NEZKTExk6tSpzJ07l08++QSTySQ9MjsmhUDYpJUHovjXzzkfBLNx1OM0qVnOgETitn379mEymfDw8CAkJITatWsbHUk8ICkEwqYcOn+VnjNzXgfw7VCfic80NSCRuO3mzZt88MEH+Pv78/XXX9O3b1/pBRQRUgiEzfgl7DzDlwVn2zemWyOGdfbASRaDM9SWLVsYOHAg7du3Jzw8nCpVqhgdSRQiKQTCcOeu3KL9tK3Z9vkPfJR2D1U2KJG47cqVK4wdO5ZNmzbxzTff0LNnT6MjCQuQQiAMcfFaAjO3nmBt8HmuJ6ZkO7Z1TCd5FrANWLt2LW+99RbPPfccZrOZcuXk2kxRJYVAWE1ammbGlhPsOhHDwbNXchz/tLcXLz9S1/rBRDYXLlxg5MiRBAcHs2zZMjp27Gh0JGFhUgiExaWlaab8cpjFf5zOcax1vYr0ae3GS63dcHaSR0IaSWvNf/7zH8aMGcMbb7zB4sWLKVmypNGxhBVIIRAWtTbkHKOWh+TY/+HznvR/pI58+NuIs2fPMmTIEM6dO8eGDRvw8fExOpKwIikEwiKCTl/mpXl7s+1rWrMcM/q1oGH1sgalEndKS0tj/vz5+Pn5MWrUKN59912KFy9udCxhZVIIRKHacuQCph+Ccuz/872uVC/nakAikZfjx4/j6+tLcnIyO3bsoGlTuU/DUUkhEA8kLU2z43gMM7eeIPiOC8BOxRQLX/Ohc+NqxoQTuUpJSeGLL75g+vTp+Pn58dZbb8kicQ5OCoG4b1Fx8XT4dFuO/WVLOLNq2GMyBGSDQkNDGTBgAJUqVWL//v3Ur1/f6EjCBkghEPfkVlIqAxbvz/EsgHqVS9HTqyYvP1KHepVLG5RO5CUhIYGPPvqIBQsW8Omnn/LGG2/I8hAikxQCUSBap08B/X7P6RzH/J5pyoAO8pulrfrjjz8wmUw0adKE0NBQatasaXQkYWOkEIi7irh4nW5f7UTr/+2rU6kkW8c8QXGZ/mmzbty4wfvvv8/PP//MzJkz6d27t/QCRK6kEIg83UhMYcSyg2w7FpO5r4dXDb5+uSUuzlIAbNnmzZsZPHgwHTt2JDw8nMqVZd0mkTeLFgKlVHdgBuAELNRaT8ulTV9gEqCBUK31PyyZSdyd1pqeM3dzOPpatv2z+rfk2ea1DEolCiIuLo7Ro0ezdetW5s+fT/fu3Y2OJOyAxQqBUsoJmAN0A6KA/UqpdVrrw1naNAQmAO211nFKKZlnaLAJq8LxDzybbV+b+pVYMqANrsVliqEtW7VqFSNGjODFF1/EbDZTtqzM2hIFY8keQRsgQmt9EkAptRzoBRzO0mYgMEdrHQegtb5owTwiDxevJTBhVThbjmb/9jepWY6Nox43KJUoqL///pvhw4djNpv56aef6NChg9GRhJ2x5EBvbSAyy+uojH1ZNQIaKaX2KKX2ZQwl5aCUGqSUClJKBcXExOTWRNynn/afpc3ULTmKQNDEJ6UI2DitNT/88APe3t40atSIkJAQKQLivhh9sdgZaAg8AbgBO5VSXlrrK1kbaa0XAAsAfHx8NOKB3UxMofPn27l4PREA1+LFGPR4A/75aD2qyVIQNu/MmTMMHjyYCxcu8Ouvv9KqVSujIwk7ZskewTmgTpbXbhn7sooC1mmtk7XWp4DjpBcGYSHJqWmsCT5Hs39vyiwCAAc/6Mbopx6WImDj0tLSmD17Nq1bt6ZTp04EBgZKERAPzJI9gv1AQ6VUfdILQD/gzhlBa4D+wPdKqSqkDxWdtGAmh7YiKJJxK8Oy7Xur80OM/b/GBiUS9+Lo0aP4+voCsHv3bho3lr83UTgs1iPQWqcAw4FNwBFghdb6kFJqilLquYxmm4BYpdRhYBswVmsdm/tXFPdLa43vD/uzFYFOjaryy4gOUgTsQHJyMlOnTqVDhw7069ePnTt3ShEQhcqi1wi01gFAwB37/LJsa2B0xn/CAuZuj+CzX49l27dueHu83SoYE0jck+DgYAYMGED16tU5cOAA9erVMzqSKIKMvlgsLKjXnD2ERl7JfP14wyr88GYbihWTZQZsXUJCApMnT2bRokVMnz6d1157TZaHEBYjhaAImrnlBF/+djzbvvXDO+DlVt6gROJe7N69G5PJhLe3N2FhYdSoUcPoSKKIk0JQhKw8EMW/fg7Ntq9/mzp88qK3QYnEvbh+/ToTJkxg9erVzJo1ixdffNHoSMJBSCEoIu4sAmVLOLP6rcfwqCbLDNiDX3/9lcGDB9O1a1fMZjMVK1Y0OpJwIFIIioDhyw7yS1h05utt/3qC+lXk4TD2IDY2ltGjR7Nz504WLlxIt27djI4kHJCsJWznPtl4JFsR2DO+ixQBO6C1ZuXKlXh5eVGhQgXCw8OlCAjD3HOPQClVDOivtf6PBfKIAkpOTeP5OXs4dD59qegyJZwJ8euGszwoxuZFR0fz1ltvceTIEVauXMljjz1mdCTh4PL81FBKlVNKTVBKzVZKPaXSjSD9zt++1oso7rTzeAwN39+YWQRqVyhJ2L+fkiJg47TWfP/99zRv3pymTZsSHBwsRUDYhPx6BEuBOGAv4Au8Byjgea11iOWjiTvdTEzB9MN+9p28nLlvXPeHGdrpIZljbuNOnTrFoEGDuHz5Mps3b6ZFixZGRxIiU36FoIHW2gtAKbUQiAbqaq0TrJJMZPPfA1GMuWNq6Jx/tKKntzyI3JalpqYye/ZsPvzwQ8aNG8fo0aNxdpY5GsK25PcTmXx7Q2udqpSKkiJgfX+ejOXlBfuy7evSuBrzX20tD463cYcPH8bX1xdnZ2f++OMPGjVqZHQkIXKVXyForpS6RvpwEEDJLK+11rqcxdM5sMSUVHp/8wfmc9mfGxz4XldZKtrGJScn8+mnnzJjxgymTJnC4MGDKVZMirawXXkWAq21PKDWIFfik2gx5bds+/47tB2t61UyKJEoqAMHDjBgwABq167NgQMHqFu3rtGRhLirPAuBUsoVGAJ4AGHAdxlLSwsLWhtyjlHLQzJfd21cjXkyDGTzbt26xaRJk1i8eDFffPEFr7zyilzAF3Yjv6GhH0i/TrAL6AE0A0ZZI5Qjik9Koanfpmz7HnGvyKI3HjEokSioHTt2MHDgQFq1akV4eDjVqlUzOpIQ9yS/QtA0y6yhRUCgdSI5nr9ibtD1ix3Z9q0a9hit6sp6M7bs2rVrvPvuu6xfv545c+bQq1cvoyMJcV/yG2/IOmtIhoQs5JOAI9mKQA+vGpye1lOKgI0LCAjA09OTlJQUzGazFAFh1/LrEbTImCUE6TOFZNZQIRu3MpQVQVGZr5f5tuUxjyoGJhJ3c+nSJd5++2327t3L999/T9euXY2OJMQDy68QhGqtW1otiQO5mZhC9xk7ibx8K3PfiY+flgvCNkxrzYoVK3j77bfp378/YWFhlC4ti/uJoiG/QqCtlsKBHDwbx4tz/8i279QnPWSGiQ07f/48Q4cOJSIigtWrV/Poo48aHUmIQpVfIaimlMrzofJa6y8tkKdIu7MIvOxTh6kvekkRsFFaaxYtWsR7773H0KFDWbFiBSVKlDA6lhCFLr9C4ASU4X93FosHlLUI/PleV6rLHcI266+//mLgwIFcv36dLVu24OXlZXQkISwmv0IQrbWeYrUkRdyM309kbv/Ht60UARuVmprKjBkzmDp1KhMmTGDUqFGySJwo8vL7CZeeQCF5fs4eQiKvZL5uLzODbJLZbMZkMlGyZEn27duHh4eH0ZGEsIr8pqnIvLhCsDbkXGYRKO3ixPGPnjY2kMghKSmJyZMn07lzZ0wmE1u3bpUiIBxKfovOXc7rmLg7rTUT15j5z59nM/cdmtLdwEQiN4GBgZhMJurVq0dwcDBubm5GRxLC6mTw00LqTwjI9nrXuM4GJRG5iY+Px8/Pjx9//JGvvvqKfv36yewt4bDkDiYLmLTuUOZ2nUolCXyvK3UqlTIwkchq27ZteHt7Ex0dTXh4OP3795ciIBya9AgK2YRV4fgH/m84aNe4LgamEVldvXqVcePGERAQwNy5c3n22WeNjiSETZAeQSHqO29vtiJwaPL/GZhGZLV+/Xo8PT1RSmE2m6UICJGF9AgKSc+Zuzh0Pn2NviY1y7Fx1OMGJxIAMTExjBo1isDAQJYsWULnznKtRog7WbRHoJTqrpQ6ppSKUEqNz6ddb6WUVkr5WDKPpQSfjcssAoAUARugtWbZsmV4eXlRu3ZtwsLCpAgIkQeL9QiUUk7AHKAbEAXsV0qt01ofvqNdWdKffPanpbJYktaaF7IsHXFyag8D0wiAyMhIhg4dytmzZ1m/fj2PPCJPeRMiP5bsEbQBIrTWJ7XWScByILend3wIfAokWDCLxXT+fHvm9g8D2lCsmMw+MUpaWhrz58+nVatWtGnThqCgICkCQhSAJa8R1AYis7yOAtpmbaCUagXU0VpvUEqNzesLKaUGAYMA6tata4Go9+fDXw5zOjYegD6t3ejUqKrBiRzXiRMnGDhwIAkJCWzfvp1mzZoZHUkIu2HYrCGlVDHgS2DM3dpqrRdorX201j5Vq9rGh+2V+CQW7T6V+Xp6n+YGpnFcKSkpfP7557Rr145evXqxZ88eKQJC3CNL9gjOAXWyvHbL2HdbWcAT2J5xM08NYJ1S6jmtdZAFcxWKrM8ZPjxFpokaISwsDJPJRLly5QgMDKRBgwZGRxLCLlmyR7AfaKiUqq+UcgH6AetuH9RaX9VaV9Fau2ut3YF9gF0Uge92nyL2ZhIAn/b2opSLzMK1psTERPz8/HjyyScZMmQIv//+uxQBIR6AxT7BtNYpSqnhwCbSH3Lzndb6kFJqChCktV6X/1ewTWdibzLll/9NfOrrUyef1qKw7du3D5PJRMOGDQkJCaFWrVpGRxLC7ln0V1mtdQAQcMc+vzzaPmHJLIXlmVm7M7d3jH1C1qixkps3bzJx4kSWL1/OjBkz6NOnj3zvhSgkssTEPUhN01xPSAGgr48b9SqXNjiRY7j9qMhLly5hNpvp27evFAEhCpEMbt+DF+buydye+oI8w9bSrly5wr/+9S82b97MvHnz6NFDbtYTwhKkR1BAQacvExZ1FYCXferg7CTfOktau3Ytnp6euLi4YDabpQgIYUHSIygArTUvzdub+frTl7wNTFO0XbhwgZEjRxIcHMyyZcvo2LGj0ZGEKPLk19oCePyzbZnb371hl+vi2TytNUuXLsXb25v69esTGhoqRUAIK5EewV18sfkYUXG3AHCrWJIujasbnKjoOXv2LEOGDOH8+fMEBATQunVroyMJ4VCkR5CPLUcuMGtrBAAVShVn97vytLHClJaWxty5c2ndujXt27dn//79UgSEMID0CPJh+uF/NzkfmNjNwCRFz/Hjx/H19SUlJYWdO3fSpEkToyMJ4bCkR5CHaRuPZm5vHdMJJ1leulCkpKTw6aef8thjj/HSSy+xa9cuKQJCGEx6BLm4Gp/MvB1/AVC3UikaVC1jcKKiISQkBJPJROXKldm/fz/169c3OpIQAukR5GrUT8GZ26uHPWZgkqIhISGB999/n6eeeooRI0awadMmKQJC2BDpEeRi+7EYAHzqVaRymRIGp7Fvf/zxByaTiSZNmhAaGkrNmjWNjiSEuIMUgjtkvTYw+x+tDExi327cuMF7773HypUrmTVrFr179zY6khAiDzI0lEVCcmrmtQEX52LUKO9qcCL7tHnzZry8vLh27Rpms1mKgBA2TnoEWYz5OTRzO8RPpoveq7i4OEaPHs22bduYP38+//d/8uQ2IeyB9Aiy2BAWDUAb90ry1LF7tGrVKjw9PSlTpgzh4eFSBISwI/Jpl2HXiZjM7dmvtDQwiX35+++/GT58OGazmZ9++okOHToYHUkIcY+kR5Dhy9+OA9CoehmqlZVrA3ejtWbx4sV4e3vTqFEjQkJCpAgIYaekRwAs3XeG4LNXAPj3s82MDWMHTp8+zeDBg7l48SKbNm2iZUvpQQlhz6RHAHywxpy53d6jioFJbFtaWhqzZs3Cx8eHJ554gsDAQCkCQhQBDt8j8A88m7m9a1xnA5PYtqNHj+Lr6wvA7t27ady4scGJhBCFxeF7BLdnCpV1daZOpVIGp7E9ycnJTJ06lQ4dOtC/f3927twpRUCIIsbhewSnLt0EYEinhwxOYnsOHjyIyWSievXqHDhwgHr16hkdSQhhAQ7dI4i8HM+5K+lPH3u0QWWD09iOW7duMWHCBJ5++mneeecdNm7cKEVAiCLMoXsES/edydxuVbeCcUFsyO7duzGZTHh7exMWFkb16vJoTiGKOocuBD8HRQLwert6KOXYD565fv06EyZMYPXq1cyaNYsXX3zR6EhCCCtx2KGhhORU4uKTAejaxLF/6/3111/x9PQkPj4es9ksRUAIB+OwPYLv9pzK3O7YqKqBSYwTGxvL6NGj2blzJwsXLqRbN1loTwhH5LA9gsPnrwFQvZzjPXhGa83PP/+Mp6cnFStWJDw8XIqAEA7MYXsEkZfjAejrU8fgJNYVHR3NsGHDOHbsGKtWraJdu3ZGRxJCGMyiPQKlVHel1DGlVIRSanwux0crpQ4rpcKUUluUUlaboxgadRWAxjXKWestDaW15rvvvqN58+Z4enoSHBwsRUAIAViwR6CUcgLmAN2AKGC/Umqd1vpwlmbBgI/WOl4pNRT4DHjZUpluO51xExlA2waVLP12hjt16hSDBg0iLi6O3377jebNmxsdSQhhQyzZI2gDRGitT2qtk4DlQK+sDbTW27TW8Rkv9wFuFsyT6bXvAgGoXNqFKkX44fSpqanMmDGDRx55hG7durFv3z4pAkKIHCx5jaA2EJnldRTQNp/2JmBjbgeUUoOAQQB169Z9oFCpaZqzGdcH/vlo0b1b9vDhw5hMJlxcXPjjjz9o1KiR0ZGEEDbKJmYNKaX+CfgA03M7rrVeoLX20Vr7VK36YFM9952Mzdx++8mGD/S1bFFSUhIffvghnTp14vXXX2fbtm1SBIQQ+bJkj+AckHVKjlvGvmyUUk8C7wOdtNaJFswDwKHzVzPelyJ3N3FQUBAmk4natWtz8OBB6tRxrBlRQoj7Y8kewX6goVKqvlLKBegHrMvaQCnVEpgPPKe1vmjBLJmi4tIXmXvas4Y13s4qbt26xbhx4+jZsyfjxo1jw4YNUgSEEAVmsUKgtU4BhgObgCPACq31IaXUFKXUcxnNpgNlgJ+VUiFKqXV5fLlCszviEgAPVS1j6beyih07duDt7c3Zs2cJDw/nlVdeKXI9HSGEZVn0hjKtdQAQcMc+vyzbT1ry/XOTmqYBqFWhpLXfulBdu3aNd999l/Xr1zNnzhx69ep19z8khBC5sImLxdaiteZMbPqMoZZ2vOz0hg0b8PT0JDU1FbPZLEVACPFAHGqJiR3HYzK33SuXNjDJ/bl06RJvv/02e/fuZfHixXTp0sXoSEKIIsChegS3n0YG4FrcycAk90ZrzfLly/H09KR69eqEhYVJERBCFBqH6hGcuHADsK/nE587d45hw4YRERHB2rVrads2v3vyhBDi3jlUj2DL0QsAPFTV9oeFtNZ8++23tGjRgpYtW3Lw4EEpAkIIi3CoHkF8YioAtW18xtBff/3FwIEDuXHjBlu3bsXLy8voSEKIIsyhegSxN5MAqFOplMFJcpeamsqXX35J27Zt6dmzJ3v37pUiIISwOIfpEWitM7erl3M1MEnuzGYzJpOJUqVKsW/fPjw8PIyOJIRwEA7TI7h2KyVz28XZdk47KSmJyZMn07lzZ0wmE1u2bJEiIISwKofpESSnpQFQ0oamjQYGBmIymXB3dyc4OBg3N6s8jkEIIbJxmEJwe2mJMq7Gn3J8fDx+fn78+OOPfPXVV/Tr10/WBxJCGMZ2xkgsLCWjEDgXM/YDd9u2bXh7exMdHU14eDj9+/eXIiCEMJTxvx5bSVpGIXAyqBBcvXqVsWPHsnHjRr755hueeeYZQ3IIIcSdHK5HYEQhWL9+PZ6enhQrVgyz2SxFQAhhUxymR3AjIX3WkJMVh2FiYmIYOXIk+/fvZ+nSpTzxxBNWe28hhCgoh+kRJKWmzxo6eemmxd9La82yZcvw8vLCzc2NsLAwKQJCCJvlMD2CpJT0QtCmfiWLvk9kZCRDhw7l7NmzrF+/nkceecSi7yeEEA/KYXoENxLTh4ZKWOhmsrS0NObNm0erVq1o27YtQUFBUgSEEHbBYXoEsTcSAbiWkHKXlvfuxIkTDBw4kISEBLZv306zZs0K/T2EEMJSHKZHcHu2ULlCvKEsJSWF6dOn065dO55//nn27NkjRUAIYXccpkdwOWPlUbeKhbPyaFhYGCaTifLlyxMYGEiDBg0K5esKIYS1OUyP4FLG0FBCcuoDfZ3ExET8/Pzo2rUrQ4YM4bfffpMiIISwaw7TIyjl4pzx//tfdG7fvn2YTCYaNmxIaGgotWrVKqx4QghhGIcpBCkZq4/WuI9nEdy8eZOJEyeyfPlyZsyYQZ8+fWR9ICFEkeEwQ0MpqRmLzjnd2yn//vvveHl5ERsbi9lspm/fvlIEhBBFisP0CI5duA5AcaeCfYhfuXKFMWPG8PvvvzNv3jyefvppS8YTQgjDOEyPoELJ4gDExSfdte2aNWto1qwZrq6uhIeHSxEQQhRpDtMjuD2c06BKmTzbXLhwgREjRhASEoK/vz8dO3a0VjwhhDCMw/QIUvNZhlprzdKlS/H29qZBgwaEhoZKERBCOAyH6RGk6vRCUOyOQnD27FkGDx5MdHQ0AQEBtG7d2oh4QghhGIfpEWQ+oSxjiCgtLY05c+bQqlUrHn/8cfbv3y9FQAjhkCzaI1BKdQdmAE7AQq31tDuOlwCWAK2BWOBlrfVpS2T539AQHDt2DF9fX1JTU9m1axdNmjSxxFsKIYRdsFiPQCnlBMwBngaaAv2VUk3vaGYC4rTWHsBXwKeWypOWMTS0evUq2rdvT9++faUICCEElu0RtAEitNYnAZRSy4FewOEsbXoBkzK2VwKzlVJK64xP7UJ0/WY8AOGhYQQFBeHu7l7YbyGEEHbJktcIagORWV5HZezLtY3WOgW4ClS+8wsppQYppYKUUkExMTH3FaZSmZKUckrj008+kiIghBBZ2MWsIa31AmABgI+Pz331Fr55rU2hZhJCiKLCkj2Cc0CdLK/dMvbl2kYp5QyUJ/2isRBCCCuxZCHYDzRUStVXSrkA/YB1d7RZB7yesf0SsNUS1weEEELkzWJDQ1rrFKXUcGAT6dNHv9NaH1JKTQGCtNbrgEXAUqVUBHCZ9GIhhBDCiix6jUBrHQAE3LHPL8t2AtDHkhmEEELkz2HuLBZCCJE7KQRCCOHgpBAIIYSDk0IghBAOTtnbbE2lVAxw5j7/eBXgUiHGsQdyzo5BztkxPMg519NaV83tgN0VggehlArSWvsYncOa5Jwdg5yzY7DUOcvQkBBCODgpBEII4eAcrRAsMDqAAeScHYOcs2OwyDk71DUCIYQQOTlaj0AIIcQdpBAIIYSDK5KFQCnVXSl1TCkVoZQan8vxEkqpnzKO/6mUcjcgZqEqwDmPVkodVkqFKaW2KKXqGZGzMN3tnLO0662U0kopu59qWJBzVkr1zfi7PqSUWmbtjIWtAD/bdZVS25RSwRk/3z2MyFlYlFLfKaUuKqXMeRxXSqmZGd+PMKVUqwd+U611kfqP9CWv/wIaAC5AKND0jjbDgHkZ2/2An4zObYVz7gyUytge6gjnnNGuLLAT2Af4GJ3bCn/PDYFgoGLG62pG57bCOS8AhmZsNwVOG537Ac+5I9AKMOdxvAewEVDAo8CfD/qeRbFH0AaI0Fqf1FonAcuBXne06QX8kLG9EuiqlFJWzFjY7nrOWuttWuv4jJf7SH9inD0ryN8zwIfAp0CCNcNZSEHOeSAwR2sdB6C1vmjljIWtIOesgXIZ2+WB81bMV+i01jtJfz5LXnoBS3S6fUAFpVTNB3nPolgIagORWV5HZezLtY3WOgW4ClS2SjrLKMg5Z2Ui/TcKe3bXc87oMtfRWm+wZjALKsjfcyOgkVJqj1Jqn1Kqu9XSWUZBznkS8E+lVBTpzz8ZYZ1ohrnXf+93ZRcPrxeFRyn1T8AH6GR0FktSShUDvgTeMDiKtTmTPjz0BOm9vp1KKS+t9RUjQ1lYf2Cx1voLpVQ70p966Km1TjM6mL0oij2Cc0CdLK/dMvbl2kYp5Ux6dzLWKuksoyDnjFLqSeB94DmtdaKVslnK3c65LOAJbFdKnSZ9LHWdnV8wLsjfcxSwTmudrLU+BRwnvTDYq4KcswlYAaC13gu4kr44W1FVoH/v96IoFoL9QEOlVH2llAvpF4PX3dFmHfB6xvZLwFadcRXGTt31nJVSLYH5pBcBex83hrucs9b6qta6itbaXWvtTvp1kee01kHGxC0UBfnZXkN6bwClVBXSh4pOWjFjYSvIOZ8FugIopZqQXghirJrSutYBr2XMHnoUuKq1jn6QL1jkhoa01ilKqeHAJtJnHHyntT6klJoCBGmt1wGLSO8+RpB+UaafcYkfXAHPeTpQBvg547r4Wa31c4aFfkAFPOcipYDnvAl4Sil1GEgFxmqt7ba3W8BzHgN8q5R6h/QLx2/Y8y92Sil/0ot5lYzrHv8GigNoreeRfh2kBxABxANvPvB72vH3SwghRCEoikNDQggh7oEUAiGEcHBSCIQQwsFJIRBCCAcnhUAIIRycFAIhCkgplaqUCsnyn7tS6gml1NWM10eUUv/OaJt1/1Gl1OdG5xciL0XuPgIhLOiW1rpF1h0ZS5jv0lo/o5QqDYQopdZnHL69vyQQrJRarbXeY93IQtyd9AiEKCRa65vAAcDjjv23gBAecGEwISxFCoEQBVcyy7DQ6jsPKqUqk76m0aE79lckfb2fndaJKcS9kaEhIQoux9BQhseVUsFAGjAtYwmEJzL2h5JeBL7WWv9ttaRC3AMpBEI8uF1a62fy2q+Uqg/sU0qt0FqHWDmbEHclQ0NCWFjGctDTgHeNziJEbqQQCGEd84COGbOMhLApsvqoEEI4OOkRCCGEg5NCIIQQDk4KgRBCODgpBEII4eCkEAghhIOTQiCEEA5OCoEQQji4/weVUey99GLtmwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackInputData\n", - "from tensorflow_privacy.privacy.membership_inference_attack.data_structures import SlicingSpec\n", - "from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackType\n", - "\n", - "import tensorflow_privacy.privacy.membership_inference_attack.plotting as plotting\n", - "\n", - "labels_train = np.argmax(y_train, axis=1)\n", - "labels_test = np.argmax(y_test, axis=1)\n", - "\n", - "input = AttackInputData(\n", - " logits_train = logits_train,\n", - " logits_test = logits_test,\n", - " loss_train = loss_train,\n", - " loss_test = loss_test,\n", - " labels_train = labels_train,\n", - " labels_test = labels_test\n", - ")\n", - "\n", - "# Run several attacks for different data slices\n", - "attacks_result = mia.run_attacks(input,\n", - " SlicingSpec(\n", - " entire_dataset = True,\n", - " by_class = True,\n", - " by_classification_correctness = True\n", - " ),\n", - " attack_types = [\n", - " AttackType.THRESHOLD_ATTACK,\n", - " AttackType.THRESHOLD_ENTROPY_ATTACK,\n", - " AttackType.LOGISTIC_REGRESSION])\n", - "\n", - "# Plot the ROC curve of the best classifier\n", - "fig = plotting.plot_roc_curve(\n", - " attacks_result.get_result_with_max_auc().roc_curve)\n", - "\n", - "# Print a user-friendly summary of the attacks\n", - "print(attacks_result.summary(by_slices = True))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E9zwsPGFujVq" - }, - "source": [ - "## Compute privacy risk score\n", - "\n", - "This part shows how to use the privacy risk score. (The code is preliminary, we can improve it later.)\n", - "\n", - "For each data slice, we compute privacy risk scores for both training and test data. We then set a threshold on risk scores (an input is inferred as a member if and only if its risk score is higher than the threshold) and compute the attack precision and recall values" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "For slice type: Entire dataset\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5597992410331742, 0.9146)\n", - "\n", - "For slice type: CLASS=0\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6191360891778913, 0.2666)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5744543458607775, 0.8896)\n", - "\n", - "For slice type: CLASS=1\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6059544658493871, 0.0692)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5467082383978894, 0.9118)\n", - "\n", - "For slice type: CLASS=2\n", - "with 0.8 as the threshold on privacy risk score, the precision-recall pair is (0.868421052631579, 0.0066)\n", - "with 0.7 as the threshold on privacy risk score, the precision-recall pair is (0.868421052631579, 0.0066)\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6181660399190049, 0.4274)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5929882636172134, 0.7882)\n", - "\n", - "For slice type: CLASS=3\n", - "with 1 as the threshold on privacy risk score, the precision-recall pair is (1.0, 0.0016)\n", - "with 0.9 as the threshold on privacy risk score, the precision-recall pair is (1.0, 0.0016)\n", - "with 0.8 as the threshold on privacy risk score, the precision-recall pair is (1.0, 0.0016)\n", - "with 0.7 as the threshold on privacy risk score, the precision-recall pair is (0.7972972972972973, 0.0118)\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6332665330661322, 0.316)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5862524785194977, 0.887)\n", - "\n", - "For slice type: CLASS=4\n", - "with 0.7 as the threshold on privacy risk score, the precision-recall pair is (0.7222222222222222, 0.013)\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.61644212262854, 0.4484)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5814285714285714, 0.814)\n", - "\n", - "For slice type: CLASS=5\n", - "with 0.8 as the threshold on privacy risk score, the precision-recall pair is (0.8387096774193549, 0.0052)\n", - "with 0.7 as the threshold on privacy risk score, the precision-recall pair is (0.7452229299363058, 0.0234)\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6262230919765166, 0.32)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5760075737084122, 0.8518)\n", - "\n", - "For slice type: CLASS=6\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6118458884416331, 0.2128)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5705128205128205, 0.712)\n", - "\n", - "For slice type: CLASS=7\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6164383561643836, 0.18)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5659122874312748, 0.8852)\n", - "\n", - "For slice type: CLASS=8\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5507837390085383, 0.8644)\n", - "\n", - "For slice type: CLASS=9\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6332288401253918, 0.202)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5841666666666666, 0.701)\n", - "\n", - "For slice type: CORRECTLY_CLASSIFIED=True\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.5220848110408277, 0.6059067835717582)\n", - "\n", - "For slice type: CORRECTLY_CLASSIFIED=False\n", - "with 0.7 as the threshold on privacy risk score, the precision-recall pair is (0.7104532829471387, 0.3436936936936937)\n", - "with 0.6 as the threshold on privacy risk score, the precision-recall pair is (0.6816679555870855, 0.624024024024024)\n", - "with 0.5 as the threshold on privacy risk score, the precision-recall pair is (0.6396320935483625, 0.8274774774774775)\n", - "\n" - ] - } - ], - "source": [ - "from tensorflow_privacy.privacy.membership_inference_attack.dataset_slicing import get_single_slice_specs\n", - "from tensorflow_privacy.privacy.membership_inference_attack.dataset_slicing import get_slice\n", - "slicing_spec = SlicingSpec(\n", - " entire_dataset = True,\n", - " by_class = True,\n", - " by_classification_correctness = True\n", - " )\n", - "input_slice_specs = get_single_slice_specs(slicing_spec, 10)\n", - "for single_slice_spec in input_slice_specs:\n", - " \n", - " attack_input_slice = get_slice(input, single_slice_spec)\n", - " risk_score_results = mia._compute_privacy_risk_score(attack_input_slice)\n", - " print(f\"For slice type: {str(risk_score_results.slice_spec)}\")\n", - " risk_score_results.print_results()\n", - " print()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "last_runtime": { - "build_target": "//learning/deepmind/public/tools/ml_python:ml_notebook", - "kind": "private" - }, - "name": "Membership inference codelab", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/tensorflow_privacy/privacy/membership_inference_attack/codelabs/privacy_risk_score_codelab.ipynb b/tensorflow_privacy/privacy/membership_inference_attack/codelabs/privacy_risk_score_codelab.ipynb new file mode 100644 index 0000000..9809a87 --- /dev/null +++ b/tensorflow_privacy/privacy/membership_inference_attack/codelabs/privacy_risk_score_codelab.ipynb @@ -0,0 +1,804 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1eiwVljWpzM7" + }, + "source": [ + "Copyright 2020 The TensorFlow Authors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "cellView": "both", + "colab": {}, + "colab_type": "code", + "id": "4rmwPgXeptiS" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YM2gRaJMqvMi" + }, + "source": [ + "# Assess privacy risks with TensorFlow Privacy Membership Inference Attacks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-B5ZvlSqqLaR" + }, + "source": [ + "\n", + " \n", + " \n", + "
\n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "9rMuytY7Nn8P" + }, + "source": [ + "##Overview\n", + "In this codelab we'll train a simple image classification model on the CIFAR10 dataset, and then use the \"membership inference attack\" against this model to assess if the attacker is able to \"guess\" whether a particular sample was present in the training set. We further compute each sample's probability of being in the training set, denoted as the privacy risk score (https://arxiv.org/abs/2003.10595)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "FUWqArj_q8vs" + }, + "source": [ + "## Setup\n", + "First, set this notebook's runtime to use a GPU, under Runtime > Change runtime type > Hardware accelerator. Then, begin importing the necessary libraries." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "cellView": "form", + "colab": {}, + "colab_type": "code", + "id": "Lr1pwHcbralz" + }, + "outputs": [], + "source": [ + "#@title Import statements.\n", + "import numpy as np\n", + "from typing import Tuple, Text\n", + "from scipy import special\n", + "\n", + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "\n", + "# Set verbosity.\n", + "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n", + "from warnings import simplefilter\n", + "from sklearn.exceptions import ConvergenceWarning\n", + "simplefilter(action=\"ignore\", category=ConvergenceWarning)\n", + "simplefilter(action=\"ignore\", category=FutureWarning)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ucw81ar6ru-6" + }, + "source": [ + "### Install TensorFlow Privacy." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "cellView": "both", + "colab": {}, + "colab_type": "code", + "id": "zcqAmiGH90kl" + }, + "outputs": [], + "source": [ + "!pip3 install git+https://github.com/tensorflow/privacy\n", + "\n", + "from tensorflow_privacy.privacy.membership_inference_attack import membership_inference_attack as mia" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "pBbcG86th_sW" + }, + "source": [ + "## Train a model" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "cellView": "form", + "colab": {}, + "colab_type": "code", + "id": "vCyOWyyhXLib" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading the dataset.\n", + "learning rate %f 0.02\n", + "Model: \"sequential\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "conv2d (Conv2D) (None, 30, 30, 32) 896 \n", + "_________________________________________________________________\n", + "max_pooling2d (MaxPooling2D) (None, 15, 15, 32) 0 \n", + "_________________________________________________________________\n", + "conv2d_1 (Conv2D) (None, 13, 13, 32) 9248 \n", + "_________________________________________________________________\n", + "max_pooling2d_1 (MaxPooling2 (None, 6, 6, 32) 0 \n", + "_________________________________________________________________\n", + "conv2d_2 (Conv2D) (None, 4, 4, 32) 9248 \n", + "_________________________________________________________________\n", + "max_pooling2d_2 (MaxPooling2 (None, 2, 2, 32) 0 \n", + "_________________________________________________________________\n", + "flatten (Flatten) (None, 128) 0 \n", + "_________________________________________________________________\n", + "dense (Dense) (None, 64) 8256 \n", + "_________________________________________________________________\n", + "dense_1 (Dense) (None, 10) 650 \n", + "=================================================================\n", + "Total params: 28,298\n", + "Trainable params: 28,298\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Epoch 1/100\n", + "200/200 [==============================] - 2s 8ms/step - loss: 2.0358 - accuracy: 0.2449 - val_loss: 1.7640 - val_accuracy: 0.3603\n", + "Epoch 2/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 1.6132 - accuracy: 0.4126 - val_loss: 1.4455 - val_accuracy: 0.4848\n", + "Epoch 3/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 1.4018 - accuracy: 0.4956 - val_loss: 1.3218 - val_accuracy: 0.5261\n", + "Epoch 4/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 1.2939 - accuracy: 0.5394 - val_loss: 1.2748 - val_accuracy: 0.5392\n", + "Epoch 5/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 1.2074 - accuracy: 0.5735 - val_loss: 1.1824 - val_accuracy: 0.5801\n", + "Epoch 6/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 1.1278 - accuracy: 0.6020 - val_loss: 1.1652 - val_accuracy: 0.5881\n", + "Epoch 7/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 1.0936 - accuracy: 0.6150 - val_loss: 1.1092 - val_accuracy: 0.6069\n", + "Epoch 8/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 1.0410 - accuracy: 0.6349 - val_loss: 1.0702 - val_accuracy: 0.6230\n", + "Epoch 9/100\n", + "200/200 [==============================] - 1s 6ms/step - loss: 0.9952 - accuracy: 0.6493 - val_loss: 1.0984 - val_accuracy: 0.6175\n", + "Epoch 10/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.9682 - accuracy: 0.6600 - val_loss: 1.0346 - val_accuracy: 0.6408\n", + "Epoch 11/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.9370 - accuracy: 0.6719 - val_loss: 1.0140 - val_accuracy: 0.6442\n", + "Epoch 12/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.9146 - accuracy: 0.6792 - val_loss: 0.9974 - val_accuracy: 0.6564\n", + "Epoch 13/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.8839 - accuracy: 0.6880 - val_loss: 1.0525 - val_accuracy: 0.6343\n", + "Epoch 14/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.8554 - accuracy: 0.7000 - val_loss: 1.0280 - val_accuracy: 0.6574\n", + "Epoch 15/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.8444 - accuracy: 0.7030 - val_loss: 0.9596 - val_accuracy: 0.6697\n", + "Epoch 16/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.8255 - accuracy: 0.7095 - val_loss: 0.9406 - val_accuracy: 0.6798\n", + "Epoch 17/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.8038 - accuracy: 0.7172 - val_loss: 1.0018 - val_accuracy: 0.6632\n", + "Epoch 18/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.8045 - accuracy: 0.7172 - val_loss: 0.9343 - val_accuracy: 0.6810\n", + "Epoch 19/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.7672 - accuracy: 0.7308 - val_loss: 0.9666 - val_accuracy: 0.6752\n", + "Epoch 20/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.7660 - accuracy: 0.7309 - val_loss: 0.9823 - val_accuracy: 0.6676\n", + "Epoch 21/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.7396 - accuracy: 0.7391 - val_loss: 0.9548 - val_accuracy: 0.6777\n", + "Epoch 22/100\n", + "200/200 [==============================] - 1s 6ms/step - loss: 0.7285 - accuracy: 0.7415 - val_loss: 0.9568 - val_accuracy: 0.6856\n", + "Epoch 23/100\n", + "200/200 [==============================] - 1s 6ms/step - loss: 0.7235 - accuracy: 0.7454 - val_loss: 0.9302 - val_accuracy: 0.6886\n", + "Epoch 24/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.7017 - accuracy: 0.7525 - val_loss: 0.9557 - val_accuracy: 0.6867\n", + "Epoch 25/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6921 - accuracy: 0.7562 - val_loss: 0.9379 - val_accuracy: 0.6921\n", + "Epoch 26/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6752 - accuracy: 0.7606 - val_loss: 0.9930 - val_accuracy: 0.6720\n", + "Epoch 27/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6769 - accuracy: 0.7590 - val_loss: 0.9484 - val_accuracy: 0.6923\n", + "Epoch 28/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6629 - accuracy: 0.7656 - val_loss: 0.9388 - val_accuracy: 0.6917\n", + "Epoch 29/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6571 - accuracy: 0.7672 - val_loss: 0.9473 - val_accuracy: 0.6898\n", + "Epoch 30/100\n", + "200/200 [==============================] - 1s 6ms/step - loss: 0.6494 - accuracy: 0.7690 - val_loss: 0.9819 - val_accuracy: 0.6849\n", + "Epoch 31/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6554 - accuracy: 0.7663 - val_loss: 0.9548 - val_accuracy: 0.6901\n", + "Epoch 32/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6218 - accuracy: 0.7806 - val_loss: 0.9712 - val_accuracy: 0.6741\n", + "Epoch 33/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6199 - accuracy: 0.7808 - val_loss: 0.9795 - val_accuracy: 0.6824\n", + "Epoch 34/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6096 - accuracy: 0.7825 - val_loss: 0.9969 - val_accuracy: 0.6802\n", + "Epoch 35/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.6156 - accuracy: 0.7806 - val_loss: 0.9656 - val_accuracy: 0.6839\n", + "Epoch 36/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5957 - accuracy: 0.7885 - val_loss: 0.9679 - val_accuracy: 0.6905\n", + "Epoch 37/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5996 - accuracy: 0.7862 - val_loss: 1.0140 - val_accuracy: 0.6847\n", + "Epoch 38/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5943 - accuracy: 0.7890 - val_loss: 0.9588 - val_accuracy: 0.6992\n", + "Epoch 39/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5895 - accuracy: 0.7924 - val_loss: 1.0227 - val_accuracy: 0.6876\n", + "Epoch 40/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5755 - accuracy: 0.7959 - val_loss: 0.9813 - val_accuracy: 0.6912\n", + "Epoch 41/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5767 - accuracy: 0.7946 - val_loss: 1.0326 - val_accuracy: 0.6819\n", + "Epoch 42/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5606 - accuracy: 0.7993 - val_loss: 1.1191 - val_accuracy: 0.6622\n", + "Epoch 43/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5489 - accuracy: 0.8054 - val_loss: 1.0608 - val_accuracy: 0.6795\n", + "Epoch 44/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5465 - accuracy: 0.8066 - val_loss: 1.0195 - val_accuracy: 0.6887\n", + "Epoch 45/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5403 - accuracy: 0.8073 - val_loss: 1.0288 - val_accuracy: 0.6901\n", + "Epoch 46/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5375 - accuracy: 0.8092 - val_loss: 1.0732 - val_accuracy: 0.6793\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 47/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5312 - accuracy: 0.8089 - val_loss: 1.0477 - val_accuracy: 0.6851\n", + "Epoch 48/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5348 - accuracy: 0.8073 - val_loss: 1.0623 - val_accuracy: 0.6823\n", + "Epoch 49/100\n", + "200/200 [==============================] - 1s 6ms/step - loss: 0.5392 - accuracy: 0.8071 - val_loss: 1.0662 - val_accuracy: 0.6852\n", + "Epoch 50/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5270 - accuracy: 0.8115 - val_loss: 1.0746 - val_accuracy: 0.6802\n", + "Epoch 51/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5112 - accuracy: 0.8167 - val_loss: 1.0665 - val_accuracy: 0.6789\n", + "Epoch 52/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5082 - accuracy: 0.8185 - val_loss: 1.0820 - val_accuracy: 0.6839\n", + "Epoch 53/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5062 - accuracy: 0.8195 - val_loss: 1.1127 - val_accuracy: 0.6836\n", + "Epoch 54/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5046 - accuracy: 0.8194 - val_loss: 1.1327 - val_accuracy: 0.6810\n", + "Epoch 55/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4952 - accuracy: 0.8237 - val_loss: 1.0908 - val_accuracy: 0.6868\n", + "Epoch 56/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.5086 - accuracy: 0.8181 - val_loss: 1.0766 - val_accuracy: 0.6876\n", + "Epoch 57/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4831 - accuracy: 0.8266 - val_loss: 1.1173 - val_accuracy: 0.6884\n", + "Epoch 58/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4883 - accuracy: 0.8230 - val_loss: 1.1164 - val_accuracy: 0.6855\n", + "Epoch 59/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4881 - accuracy: 0.8249 - val_loss: 1.1151 - val_accuracy: 0.6828\n", + "Epoch 60/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4891 - accuracy: 0.8250 - val_loss: 1.1092 - val_accuracy: 0.6842\n", + "Epoch 61/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4806 - accuracy: 0.8279 - val_loss: 1.1259 - val_accuracy: 0.6821\n", + "Epoch 62/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4816 - accuracy: 0.8267 - val_loss: 1.1461 - val_accuracy: 0.6736\n", + "Epoch 63/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4599 - accuracy: 0.8349 - val_loss: 1.1267 - val_accuracy: 0.6877\n", + "Epoch 64/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4650 - accuracy: 0.8325 - val_loss: 1.1498 - val_accuracy: 0.6827\n", + "Epoch 65/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4720 - accuracy: 0.8303 - val_loss: 1.1468 - val_accuracy: 0.6819\n", + "Epoch 66/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4769 - accuracy: 0.8293 - val_loss: 1.1620 - val_accuracy: 0.6857\n", + "Epoch 67/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4621 - accuracy: 0.8326 - val_loss: 1.1993 - val_accuracy: 0.6746\n", + "Epoch 68/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4496 - accuracy: 0.8381 - val_loss: 1.1665 - val_accuracy: 0.6803\n", + "Epoch 69/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4463 - accuracy: 0.8378 - val_loss: 1.2094 - val_accuracy: 0.6858\n", + "Epoch 70/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4453 - accuracy: 0.8399 - val_loss: 1.2300 - val_accuracy: 0.6736\n", + "Epoch 71/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4554 - accuracy: 0.8365 - val_loss: 1.1662 - val_accuracy: 0.6776\n", + "Epoch 72/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4416 - accuracy: 0.8411 - val_loss: 1.2145 - val_accuracy: 0.6784\n", + "Epoch 73/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4474 - accuracy: 0.8381 - val_loss: 1.2112 - val_accuracy: 0.6781\n", + "Epoch 74/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4365 - accuracy: 0.8422 - val_loss: 1.2111 - val_accuracy: 0.6738\n", + "Epoch 75/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4354 - accuracy: 0.8431 - val_loss: 1.2956 - val_accuracy: 0.6663\n", + "Epoch 76/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4268 - accuracy: 0.8457 - val_loss: 1.2118 - val_accuracy: 0.6826\n", + "Epoch 77/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4190 - accuracy: 0.8479 - val_loss: 1.2825 - val_accuracy: 0.6732\n", + "Epoch 78/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4423 - accuracy: 0.8407 - val_loss: 1.2410 - val_accuracy: 0.6757\n", + "Epoch 79/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4301 - accuracy: 0.8438 - val_loss: 1.2291 - val_accuracy: 0.6786\n", + "Epoch 80/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4337 - accuracy: 0.8418 - val_loss: 1.2695 - val_accuracy: 0.6854\n", + "Epoch 81/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4209 - accuracy: 0.8480 - val_loss: 1.2480 - val_accuracy: 0.6764\n", + "Epoch 82/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4267 - accuracy: 0.8456 - val_loss: 1.2956 - val_accuracy: 0.6762\n", + "Epoch 83/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4112 - accuracy: 0.8517 - val_loss: 1.2998 - val_accuracy: 0.6745\n", + "Epoch 84/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4202 - accuracy: 0.8487 - val_loss: 1.2688 - val_accuracy: 0.6775\n", + "Epoch 85/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4338 - accuracy: 0.8434 - val_loss: 1.3085 - val_accuracy: 0.6786\n", + "Epoch 86/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4105 - accuracy: 0.8525 - val_loss: 1.3298 - val_accuracy: 0.6762\n", + "Epoch 87/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4154 - accuracy: 0.8493 - val_loss: 1.2965 - val_accuracy: 0.6755\n", + "Epoch 88/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4042 - accuracy: 0.8543 - val_loss: 1.3223 - val_accuracy: 0.6790\n", + "Epoch 89/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4088 - accuracy: 0.8523 - val_loss: 1.3251 - val_accuracy: 0.6754\n", + "Epoch 90/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4008 - accuracy: 0.8557 - val_loss: 1.2946 - val_accuracy: 0.6830\n", + "Epoch 91/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4057 - accuracy: 0.8530 - val_loss: 1.3121 - val_accuracy: 0.6815\n", + "Epoch 92/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4049 - accuracy: 0.8543 - val_loss: 1.3541 - val_accuracy: 0.6765\n", + "Epoch 93/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4090 - accuracy: 0.8529 - val_loss: 1.2951 - val_accuracy: 0.6746\n", + "Epoch 94/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4057 - accuracy: 0.8545 - val_loss: 1.3573 - val_accuracy: 0.6743\n", + "Epoch 95/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4067 - accuracy: 0.8524 - val_loss: 1.3811 - val_accuracy: 0.6710\n", + "Epoch 96/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.3892 - accuracy: 0.8591 - val_loss: 1.3791 - val_accuracy: 0.6712\n", + "Epoch 97/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.3978 - accuracy: 0.8550 - val_loss: 1.3702 - val_accuracy: 0.6680\n", + "Epoch 98/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4045 - accuracy: 0.8554 - val_loss: 1.4202 - val_accuracy: 0.6670\n", + "Epoch 99/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.3891 - accuracy: 0.8602 - val_loss: 1.3683 - val_accuracy: 0.6712\n", + "Epoch 100/100\n", + "200/200 [==============================] - 1s 5ms/step - loss: 0.4081 - accuracy: 0.8506 - val_loss: 1.3715 - val_accuracy: 0.6707\n", + "Finished training.\n" + ] + } + ], + "source": [ + "#@markdown Train a simple model on CIFAR10 with Keras.\n", + "\n", + "dataset = 'cifar10'\n", + "num_classes = 10\n", + "num_conv = 3\n", + "activation = 'relu'\n", + "lr = 0.02\n", + "momentum = 0.9\n", + "batch_size = 250\n", + "epochs = 100 # Privacy risks are especially visible with lots of epochs.\n", + "\n", + "\n", + "def small_cnn(input_shape: Tuple[int],\n", + " num_classes: int,\n", + " num_conv: int,\n", + " activation: Text = 'relu') -> tf.keras.models.Sequential:\n", + " \"\"\"Setup a small CNN for image classification.\n", + "\n", + " Args:\n", + " input_shape: Integer tuple for the shape of the images.\n", + " num_classes: Number of prediction classes.\n", + " num_conv: Number of convolutional layers.\n", + " activation: The activation function to use for conv and dense layers.\n", + "\n", + " Returns:\n", + " The Keras model.\n", + " \"\"\"\n", + " model = tf.keras.models.Sequential()\n", + " model.add(tf.keras.layers.Input(shape=input_shape))\n", + "\n", + " # Conv layers\n", + " for _ in range(num_conv):\n", + " model.add(tf.keras.layers.Conv2D(32, (3, 3), activation=activation))\n", + " model.add(tf.keras.layers.MaxPooling2D())\n", + "\n", + " model.add(tf.keras.layers.Flatten())\n", + " model.add(tf.keras.layers.Dense(64, activation=activation))\n", + " model.add(tf.keras.layers.Dense(num_classes))\n", + " return model\n", + "\n", + "\n", + "print('Loading the dataset.')\n", + "train_ds = tfds.as_numpy(\n", + " tfds.load(dataset, split=tfds.Split.TRAIN, batch_size=-1))\n", + "test_ds = tfds.as_numpy(\n", + " tfds.load(dataset, split=tfds.Split.TEST, batch_size=-1))\n", + "x_train = train_ds['image'].astype('float32') / 255.\n", + "y_train_indices = train_ds['label'][:, np.newaxis]\n", + "x_test = test_ds['image'].astype('float32') / 255.\n", + "y_test_indices = test_ds['label'][:, np.newaxis]\n", + "\n", + "# Convert class vectors to binary class matrices.\n", + "y_train = tf.keras.utils.to_categorical(y_train_indices, num_classes)\n", + "y_test = tf.keras.utils.to_categorical(y_test_indices, num_classes)\n", + "\n", + "input_shape = x_train.shape[1:]\n", + "\n", + "model = small_cnn(\n", + " input_shape, num_classes, num_conv=num_conv, activation=activation)\n", + "\n", + "print('learning rate %f', lr)\n", + "\n", + "optimizer = tf.keras.optimizers.SGD(lr=lr, momentum=momentum)\n", + "\n", + "loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n", + "model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])\n", + "model.summary()\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " batch_size=batch_size,\n", + " epochs=epochs,\n", + " validation_data=(x_test, y_test),\n", + " shuffle=True)\n", + "print('Finished training.')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ee-zjGGGV1DC" + }, + "source": [ + "## Calculate logits, probabilities and loss values for training and test sets.\n", + "\n", + "We will use these values later in the membership inference attack and privacy risk score analysis to separate training and test samples." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "cellView": "both", + "colab": {}, + "colab_type": "code", + "id": "um9r0tSiPx4u" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predict on train...\n", + "Predict on test...\n", + "Apply softmax to get probabilities from logits...\n", + "Compute losses...\n" + ] + } + ], + "source": [ + "print('Predict on train...')\n", + "logits_train = model.predict(x_train, batch_size=batch_size)\n", + "print('Predict on test...')\n", + "logits_test = model.predict(x_test, batch_size=batch_size)\n", + "\n", + "print('Apply softmax to get probabilities from logits...')\n", + "prob_train = special.softmax(logits_train, axis=1)\n", + "prob_test = special.softmax(logits_test, axis=1)\n", + "\n", + "print('Compute losses...')\n", + "cce = tf.keras.backend.categorical_crossentropy\n", + "constant = tf.keras.backend.constant\n", + "\n", + "loss_train = cce(constant(y_train), constant(prob_train), from_logits=False).numpy()\n", + "loss_test = cce(constant(y_test), constant(prob_test), from_logits=False).numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "QETxVOHLiHP4" + }, + "source": [ + "## Run membership inference attacks.\n", + "\n", + "We will now execute a membership inference attack against the previously trained CIFAR10 model. This will generate a number of scores, most notably, attacker advantage and AUC for the membership inference classifier.\n", + "\n", + "An AUC of close to 0.5 means that the attack wasn't able to identify training samples, which means that the model doesn't have privacy issues according to this test. Higher values, on the contrary, indicate potential privacy issues." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "B8NIwhVwQT7I" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best-performing attacks over all slices\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.74 on slice CORRECTLY_CLASSIFIED=False\n", + " LOGISTIC_REGRESSION achieved an advantage of 0.40 on slice CORRECTLY_CLASSIFIED=False\n", + "\n", + "Best-performing attacks over slice: \"Entire dataset\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.61\n", + " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.21\n", + "\n", + "Best-performing attacks over slice: \"CLASS=0\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.66\n", + " LOGISTIC_REGRESSION achieved an advantage of 0.25\n", + "\n", + "Best-performing attacks over slice: \"CLASS=1\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.58\n", + " THRESHOLD_ATTACK achieved an advantage of 0.17\n", + "\n", + "Best-performing attacks over slice: \"CLASS=2\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.66\n", + " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.31\n", + "\n", + "Best-performing attacks over slice: \"CLASS=3\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.71\n", + " LOGISTIC_REGRESSION achieved an advantage of 0.35\n", + "\n", + "Best-performing attacks over slice: \"CLASS=4\"\n", + " THRESHOLD_ATTACK achieved an AUC of 0.64\n", + " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.25\n", + "\n", + "Best-performing attacks over slice: \"CLASS=5\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.65\n", + " THRESHOLD_ENTROPY_ATTACK achieved an advantage of 0.26\n", + "\n", + "Best-performing attacks over slice: \"CLASS=6\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.60\n", + " THRESHOLD_ATTACK achieved an advantage of 0.16\n", + "\n", + "Best-performing attacks over slice: \"CLASS=7\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.61\n", + " LOGISTIC_REGRESSION achieved an advantage of 0.25\n", + "\n", + "Best-performing attacks over slice: \"CLASS=8\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.58\n", + " THRESHOLD_ATTACK achieved an advantage of 0.16\n", + "\n", + "Best-performing attacks over slice: \"CLASS=9\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.59\n", + " LOGISTIC_REGRESSION achieved an advantage of 0.19\n", + "\n", + "Best-performing attacks over slice: \"CORRECTLY_CLASSIFIED=True\"\n", + " LOGISTIC_REGRESSION achieved an AUC of 0.51\n", + " THRESHOLD_ATTACK achieved an advantage of 0.05\n", + "\n", + "Best-performing attacks over slice: \"CORRECTLY_CLASSIFIED=False\"\n", + " THRESHOLD_ENTROPY_ATTACK achieved an AUC of 0.75\n", + " LOGISTIC_REGRESSION achieved an advantage of 0.40\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0g0lEQVR4nO3deXzM1/7H8dexxr4EQWJfaklii622qlKlrbZal99te9XEVmupolVX1VWlrarSUlotV2yluKLUvqZJyDYoYktC7BFLRLbz+yORhixCMvPNZD7Px8PDdznJvL8R85nzXc5RWmuEEELYrwJGBxBCCGEsKQRCCGHnpBAIIYSdk0IghBB2TgqBEELYuUJGB3hcFSpU0DVr1jQ6hhBC2JRDhw5d1VpXzGifzRWCmjVr4u/vb3QMIYSwKUqpc5ntk1NDQghh56QQCCGEnZNCIIQQds7mrhFkJD4+noiICGJjY42Oku85ODjg4uJC4cKFjY4ihMgl+aIQREREUKpUKWrWrIlSyug4+ZbWmmvXrhEREUGtWrWMjiOEyCUWOzWklPpRKXVZKWXOZL9SSn2jlApVSgUrpZo/6WvFxsbi6OgoRcDClFI4OjpKz0uIfMaS1wiWAN2z2P8CUC/lzyDgu5y8mBQB65CfsxD5j8VODWmt9yilambRpBfwi04eB9tHKVVWKVVFax1pqUxCCGFLwq/H8Pnvf3HpZiwRV6J5o3VtxnR7Ktdfx8i7hpyB8DTrESnb0lFKDVJK+Sul/K9cuWKVcE/it99+QynFX3/9lbpt165dvPjiiw+069+/P2vWrAGSL3RPmDCBevXq0bx5c9q2bcvmzZuzfJ179+7xj3/8g7p169K6dWvOnj2brs3x48dp2rRp6p/SpUvz9ddfP9Dmyy+/RCnF1atXAZg1a1Zqe1dXVwoWLMj169ef4CchhHgSd+MSmbzezAtz9lJzwiY6zNzJ/4Ij8TsbReSdJH47HGaR17WJi8Va64XAQgAPD488O5OOl5cX7du3x8vLi08++SRbX/Pxxx8TGRmJ2WymaNGiXLp0id27d2f5NYsXL6ZcuXKEhoayYsUKxo8fz8qVKx9o89RTTxEYGAhAYmIizs7OvPrqq6n7w8PD2bp1K9WrV0/dNm7cOMaNGwfAxo0bmT17NuXLl8/WcQghnpz5fDQvzt2X6f4yiVGM7NmS1vUqW+T1jSwE54FqadZdUrbZpNu3b7Nv3z527tzJSy+9lK1CEBMTww8//MCZM2coWrQoAE5OTvTp0yfLr1u/fj1TpkwB4PXXX2f48OForTM9f799+3bq1KlDjRo1Ure99957zJw5k169emX4NV5eXvTr1++RxyCEeDy37yVw8NQ1vEMi2R96lcu37qVr07FOWS74/I+zh3ez+OvP6NSpp0UzGVkINgDDlVIrgNZAdG5cH6g5YVOOg2Xk7Iys/yHWr19P9+7dqV+/Po6Ojhw6dIgWLVpk+TWhoaFUr16d0qVLZ7jf09OTIUOG4OHh8cD28+fPU61acg0tVKgQZcqU4dq1a1SoUCHD77NixYoH3tTXr1+Ps7MzTZo0ybB9TEwMv//+O99++22W+YUQjxafmMTF6Fh+OXiWH/aeybLtkE61qXTlEBPHDaB///5sPLCdYsWKWTyjxQqBUsoLeAaooJSKAP4NFAbQWn8PeAM9gFAgBnjHUlmswcvLi1GjRgHQt29fvLy8aNGiRaaf0rNz982iRYtynCsuLo4NGzbw2WefAclv8tOnT2fr1q2Zfs3GjRtp166dnBYS4jHcS0jkz9PXOX/jLl9vO0HlMsUICr+R5dd0qFeBF92r0MOtCjeuXGTIkCGEh4ezadOmdB8ALcmSdw1leV4h5W6hYbn9uo/65G4J169fZ8eOHYSEhKCUIjExEaUUs2bNwtHRkaioqHTtK1SoQN26dQkLC+PmzZuZ9goy4uzsTHh4OC4uLiQkJBAdHY2jo2OGbTdv3kzz5s1xcnIC4NSpU5w5cya1NxAREUHz5s3x9fWlcuXk848P9yCEEH+7fS+BfSevcOV2HEcv3KRgAVjmk/4i7qWb6U/5vNK0Kp/0cqVMsb+fzE9KSmLBggVMnjyZUaNGsW7dOooUKWLRY3iYTVwszuvWrFnDW2+9xYIFC1K3derUib1799K6dWsuXLjAsWPHaNiwIefOnSMoKIimTZtSvHhxTCYTo0aNYsGCBRQpUoQrV66wa9cu3njjjUxf7+WXX+bnn3+mbdu2rFmzhmeffTbTHsbD5/rd3Ny4fPly6vr9Yb3vn1aKjo5m9+7dLFu2LKc/FiHyhfjEJPaFXmXAEj90Nm5VKVhA0bWhE9Udi/NKU2cqlipKhZJFMvw/euLECTw9PYmPj2f37t00atTIAkfwaFIIcoGXlxfjx49/YFvv3r3x8vKiY8eOLFu2jHfeeYfY2FgKFy7MokWLKFOmDADTpk1j0qRJNGrUCAcHB0qUKMHUqVOBzK8RmEwm3nrrLerWrUv58uVZsWIFABcuXMDT0xNvb28A7ty5wx9//PFAgXqUdevW0a1bN0qUKPHEPw8hbJXWml3Hr3Du2h3WBV7g3LU73IiJz7R97+YuJCYl4epcBseSRXi1mUu2XichIYGvvvqKmTNn8vHHHzN8+HAKFiyYW4fx2JTOTonLQzw8PPTDE9Pc/7QtrEN+3iK/0Vqz5MBZPtl4NNM2TaqVxaNGOcZ0rU+Jok/+GTooKIgBAwZQvnx5Fi5caLVxu5RSh7TWGV54kB6BEMLuXLhxl7d/9OV81F0KFlDcvpeQrs3QZ+rwlFMp2tR2xKl00RwPr3Lv3j2mTZvGggULmDFjBu+8806eGbJFCoEQIl+LS0ji+p045mw/yeFzURy/dCvTtm7OZVj4dguqlMndWzYPHjyIyWRKfdCzatWqufr9cyrfFIKsHqgSucfWTiUK+xMTl0BIRDQfrgvh1JU7FCygSEzK+Pe2VoUSTHihAa7OZahc2oGCBXL3PeT27dtMmjSJVatW8c0339C7d+88+T6VLwqBg4MD165dk6GoLez+fAQODg5GRxEiVURUDF9sOc7pq3cIjohOtz8xSVO8SEFi4hKpVaEEgzvWpnODSjiVtuzv8R9//MGgQYPo2LEjISEhmd7inRfki0Lg4uJCREQEeXlAuvzi/gxlQhjp+MVb/HzwLMv/zHwQthqOxXEpV4x/v9SY+k6lrJYtKiqKsWPHsn37dhYsWED37lmNxp835ItCULhwYZkxS4h87M69BMKjYlgXcJ4Fu09n2KZZ9bL8q21NujZyytFdPTmxbt06hg8fzquvvorZbKZUKesVoJzIF4VACJF/aK05cuEmO/66jHdIJH9dzPzi7tRejeneuDKVLHya51EuXrzIiBEjCA4OZsWKFXTo0MHQPI9LCoEQwnB34xJZ6RfGct8wTly6nWm7FjXK0bRaWQa0r4VzWcsPxvYoWmuWLl3KuHHjGDBgAEuXLrXJa2hSCIQQVpeQmMTqQxH4nbnO2oCMR58voKB1LUdqVSzBW21q0LBK9sfjsoZz584xePBgLl26lDqml62SQiCEsLirt+/xW8B5dp+4wt6TVzNt51C4AKb2tRjWuS7Fi+TNt6ekpCTmz5/PlClTGDt2LO+//z6FCxd+9BfmYXnzJy2EsHnh12Pw/Nk/ywe4SjsUomP9inSqX5FXmzlTqKCRs+c+2vHjxzGZTGit2bdvHw0aNDA6Uq6QQiCEyBVaa0avDMTn9LUMh2AG6OlehecaVqJVLcc8cY4/u+Lj4/niiy/48ssvmTJlCu+++y4FCuTtovU4pBAIIZ7YjZg4vt52kt8Cz2c6SufM3u683LQqDoWNG10zJwICAjCZTFSsWBF/f39q1qxpdKRcJ4VACPFYYuMTWRdwnolrQzLc37JmOT5+sRE1HEs8MAGLrYmNjWXq1KksWrSIWbNm8fbbb+fbkQukEAghHunQuescCL1GUMQNth27nG5/46ql6d3chTc8XCjlYLtv/vft27cPT09P3NzcCA4OTp29L7+SQiCEeEB8YhJnrt5h4Z7TeIdEEhOXmGG7+k4lGd+9AV0aOlk5oeXcunWLiRMnsnbtWr799ltee+01oyNZhRQCIezY3bhEJqwNxv9sFDfvxnMrg3H503rmqYq84FqZ11tUy/WROo22ZcsWBg8ezLPPPsuRI0coV66c0ZGsRgqBEHbm5KVbDFl2iFNX7jyybYWSRfi0lyudG1Sy2Yu9j3L9+nXee+89du/ezcKFC+nWrZvRkaxOCoEQ+ZjWmh/2nubP09e5dS8B3zPXM2zXtZET//CoRrXyxanhWDzfvuk/bM2aNYwcOZI33ngDs9lMyZIljY5kCCkEQuQzUXfiOBp5k18PR7D2cMbDN0DyuD3f9GtG1TIO+fZumMxERkYybNgwjh07xurVq2nXrp3RkQwlhUAIG3cjJo5TV+7wv+AL/LT/bKbt3n2mDk9VLkXrWo5ULmN7A6PlBq01S5YsYfz48QwaNIjly5fb5CBxuU0KgRA2al1ABO+tDMpwX8mihXBzLkNUTBwfv9iIdnUrWDld3nPmzBkGDRrE9evX2bp1K02bNjU6Up4hhUAIGxIbn8jQZYfYeTz9bHyta5XnTlwC/2pbk9dbuNjd6Z7MJCYmMm/ePKZOncq4ceMYO3YshQrJW19a8tMQwgZERMXw5qI/OXst5oHt9SqVZPnANlQsVdSgZHnbsWPHMJlMFCpUiAMHDlC/fn2jI+VJUgiEyKOi7sTx0W8heIdcTLevvlNJ5v1fc+pZcS5eWxIfH8/MmTOZPXs2U6dOZciQIflqkLjcJoVAiDxi+7FLbDZfZO3hCCqWKprhCJ6vNXPm01dcDZuT1xYcOnSIAQMGULVqVQ4fPkz16tWNjpTnyW+TEAZJSEzi620n2Rd6lcDwGw/su18EKpd2oEKpInzYoyFP15ELvlm5e/cuU6ZMYcmSJXz55Zf885//lOsk2SSFQAgrSkhMYsmBs/wvODLdm/99zzd2oksDJ15rnvcnaskr9uzZg6enJ82aNSMkJIRKlSoZHcmmWLQQKKW6A3OAgsAirfWMh/ZXB34Gyqa0maC19rZkJiGsLTY+ES/fMFb6hfPXxYxn6xrZpR5vtqlOpVJyT/vjuHnzJhMmTGDDhg18++23vPLKK0ZHskkWKwRKqYLAPKArEAH4KaU2aK2Ppmk2CViltf5OKdUI8AZqWiqTENagtcbvbBRHLkTzw57TXIiOzbDd573deK25C4XlU/8T8fb2ZsiQIXTr1g2z2UzZsmWNjmSzLNkjaAWEaq1PAyilVgC9gLSFQAOlU5bLABcsmEcIizlx6RYLdp9mfeB5EpJ0hm2qly/ORz0b0q2Rk5y7zoGrV6/y3nvvsX//fn766Se6dOlidCSbZ8lC4AyEp1mPAFo/1GYKsFUpNQIoATyX0TdSSg0CBgFyB4DIU+4lJNJx5s4M7/CpULIozauXpWn1sgzqUFvO9+eQ1ppVq1YxevRo+vXrR0hICCVKlDA6Vr5g9MXifsASrfWXSqm2wFKllKvWOiltI631QmAhgIeHR8Yft4Swkj+OXmLR3tOEXY8h8qHTPl++0QRX5zLUrlhCTvnkogsXLjB06FBCQ0NZt24dbdq0MTpSvmLJQnAeqJZm3SVlW1omoDuA1vqgUsoBqACknwtPCIMkJmmORd5k1/HLfLH1RIZtujVy4rs3W+S7yVqMprVm8eLFTJw4kXfffZdVq1ZRtKg8RZ3bLFkI/IB6SqlaJBeAvsD/PdQmDOgCLFFKNQQcgPSDqAhhgKg7cYxcEcDek1fT7StTrDCjutSjUdXSeNQoJ6d9LOD06dMMHDiQmzdvsn37dtzd3Y2OlG9ZrBBorROUUsOBLSTfGvqj1vqIUmoq4K+13gCMBX5QSr1H8oXj/lprOfUjDBGXkMQIr8OcuXqHE5dup9vfpnZ5qpQpxpttatCihv1MY2htiYmJfPPNN/znP/9hwoQJjB49WgaJszCL/nRTngnwfmjb5DTLRwH7nhFCGGrX8cssOXCW6LvxBITdSLffoXAB3J3L8oupld3M2mWkI0eOYDKZcHBwwMfHh7p16xodyS5ImRV26UZMHE2n/pHhvubVyzLi2Xq0reMob/5WEhcXx4wZM5g7dy7Tpk1j4MCBMkicFUkhEHbjzr0EZm05zpIDZ9Pt+6D7UzxdpwJNXMrIPf5W5ufnx4ABA6hRowYBAQG4uLgYHcnuSCEQ+V5EVAzvrw7C53T6idv/2bo6/3nVzYBUIiYmhsmTJ7Ns2TJmz55N3759pQgbRAqByJe01qzyD2f8ryHp9vXxcGFQxzrUrVTSgGQCYNeuXXh6etKqVStCQkKoWLGi0ZHsmhQCkW/cio1n65FLrA2IYH/otQf2lShSkPEvNODN1jUoIPf6GyY6OpoPPvgAb29v5s+fz0svvWR0JIEUApEPrA88zzKfc/idjcpw/3LP1jwtk7cb7n//+x9Dhw6lR48emM1mypQpY3QkkUIKgbBZS33O8fFv5nTb61QsQd+W1XnBrTIu5YobkEykdeXKFUaNGoWvry+//PILnTt3NjqSeIgUAmFTIqJi8PINY97OU+n2/Tasndz1k4dorfHy8mLMmDG89dZbBAcHU7y4FOa8SAqBsAmx8Yl4TNvG7XsJ6fZtGtmexlXlNENeEhERwdChQzl79iwbNmygVatWRkcSWZBCIPKshMQkDpy6xkr/cDYFR6ZuL1m0EJ3qV+Sz3m6UdihsYELxsKSkJH744QcmTZrEiBEj+PXXXylSpIjRscQjSCEQec7Ovy7jHRLJmsMRPDzyVB8PF2a+3sSYYCJLoaGhDBw4kJiYGHbu3Imrq6vRkUQ2SSEQecaB0Kv836I/M9z3cpOqTOrZkEqlZU7fvCYhIYGvv/6aGTNm8NFHHzFy5EgKFpShOWyJFAJhqPvz+/ZZcDDdvnHPP4Vnh1oULSRvKnlVSEgIJpOJUqVK8eeff1KnTh2jI4knIIVAGOLCjbv8uO8Mi/adSbfPa2Ab2tZxNCCVyK579+4xffp05s+fz2effYbJZJK7tWyYFAJhVbuOX6b/T37ptjeqUppRz9Xj+caVDUglHoePjw8mk4m6desSGBiIs7Oz0ZFEDkkhEFYxZ9tJZm9LP83jcw0r8XqLanR3lQKQ1925c4ePP/4YLy8vvv76a/r06SO9gHxCCoGwqC1HLjJ5vZlLN+89sP39bvUZ1rmuvJHYiO3btzNw4EDatWtHSEgIFSrIkB35iRQCYRH7Tl7lzcUP3gFUoWRRto/tRJlicu+/rbhx4wbjxo1jy5YtfPfdd/Ts2dPoSMICpBCIXBOfmMQHa4JZF3A+3b6t73WkvlMpA1KJJ7V+/XqGDRvGyy+/jNlspnTp0kZHEhYihUDkim93nOSLremvAawY1IY2teUOIFty6dIlRo4cSUBAAMuXL6djx45GRxIWJoVA5NgLc/ZyLPJm6vpLTarysTz8ZXO01vz3v/9l7Nix9O/fnyVLllCsWDGjYwkrkEIgnsjlW7FM+98xth+7xJ24xNTtG4e3x81FBoCzNWFhYQwZMoTz58+zadMmPDw8jI4krEgKgXgsm4IjeX91EHfjE9PtO/LJ85QoKr9StiQpKYkFCxYwefJkRo0axfjx4ylcWC7m2xv5Xyuy7Z2ffNl5/MoD215pWpXeLVx4uk4FCsoUkDblxIkTeHp6Eh8fz+7du2nUqJHRkYRBpBCIR9pz4gpv/+j7wDbvkR1oVFXuIrFFCQkJfPnll8yaNYvJkyczbNgwGSTOzkkhEBnSWhMZHcuH60LYlaYX0MOtMvP/2cLAZCIngoKCGDBgAOXLl8fPz49atWoZHUnkAVIIRDoZPQwGsH5YO5pUK2v9QCLHYmNjmTZtGgsXLuTzzz+nf//+8lS3SCWFQKT65eBZJq8/km5746ql+XlAKyqULGpAKpFTBw4cwGQy0bBhQ4KCgqhSpYrRkUQeI4VAcC8hkacm/Z5u+4/9PXi2gZMBiURuuH37Nh999BGrV6/mm2++oXfv3tILEBmSQmDnDp66Rr8ffB7YttyzNU/XlUHFbNnWrVsZPHgwHTt2JCQkBEdHebpbZM6ihUAp1R2YAxQEFmmtZ2TQpg8wBdBAkNb6/yyZSfxt0d7TTNt0LHX9uYaV+OFtD/nUaMOioqIYM2YMO3bsYMGCBXTv3t3oSMIGWKwQKKUKAvOArkAE4KeU2qC1PpqmTT1gItBOax2llKpkqTzib+HXY+gwc+cD2xa97cFzjeQ0kC1bu3YtI0aM4LXXXsNsNlOqlAzyJ7LHkj2CVkCo1vo0gFJqBdALOJqmzUBgntY6CkBrfdmCeexebHwiH60z8+vhiAe2+096Ti4E27CLFy8yfPhwzGYzK1eupH379kZHEjamgAW/tzMQnmY9ImVbWvWB+kqp/Uopn5RTSekopQYppfyVUv5XrlzJqIl4hNDLt2nw8e8PFIGPejTk7IyeUgRslNaan3/+GXd3d+rXr09gYKAUAfFEjL5YXAioBzwDuAB7lFJuWusbaRtprRcCCwE8PDy0lTPavIG/+PPH0Uup6y83qcrsfzSVISFs2Llz5xg8eDCXLl3i999/p3nz5kZHEjbMkj2C80C1NOsuKdvSigA2aK3jtdZngBMkFwaRC67dvkfNCZseKALLB7bmm37NpAjYqKSkJL799ltatGhBp06d8PX1lSIgcsySPQI/oJ5SqhbJBaAv8PAdQb8B/YCflFIVSD5VdNqCmexCcMQNpm06hu+Z6w9sPzHtBYoUsmTtF5b0119/4enpCcC+ffto0KCBwYlEfmGxdwWtdQIwHNgCHANWaa2PKKWmKqVeTmm2BbimlDoK7ATGaa2vWSqTPXjnJ19e/nb/A0Xgg+5PcXZGTykCNio+Pp7p06fTvn17+vbty549e6QIiFxl0WsEWmtvwPuhbZPTLGtgTMofkQNaa2pNfOBHzaSeDXmzTQ0cCsvIkrYqICCAAQMG4OTkxKFDh6hRo4bRkUQ+ZPTFYpELvtx6nLk7QlPXSxUthN+k56QA2LDY2Fg++eQTFi9ezKxZs3j77bflQT9hMVIIbFhQ+A36LvR5YLawhlVKs3lUBwNTiZzat28fJpMJd3d3goODqVy5stGRRD4nhcAGnb9xl3YzdqTbvnvcM9RwLGFAIpEbbt26xcSJE1m3bh1z587ltddeMzqSsBNSCGzMtqOX8PzF/4FtKwe1oXVtGVTMlv3+++8MHjyYLl26YDabKVeunNGRhB2RQmBD4hKSUotA4YKKJe+0op2MEmrTrl27xpgxY9izZw+LFi2ia9euRkcSdkjuJ7QRF27cpf6kzanrm0d1kCJgw7TWrFmzBjc3N8qWLUtISIgUAWGYx+4RKKUKAP201v+1QB6RgZFeAWwIupC6/mozZ+pWkpElbVVkZCTDhg3j2LFjrFmzhqefftroSMLOZdojUEqVVkpNVEp9q5TqppKNIPnJ3z7Wi2i/4hOTaDT59weKQB8PF2b/o6lxocQT01rz008/0aRJExo1akRAQIAUAZEnZNUjWApEAQcBT+BDQAGvaK0DLR/NvmmtaT19OzFxybeG1ncqyaaRHShcUM7m2aIzZ84waNAgrl+/ztatW2natKnRkYRIldW7Sm2tdX+t9QKSxwNqBDwvRcA6Xpm3n+t34gAY07U+W9/rJEXABiUmJjJnzhxatmxJ165d+fPPP6UIiDwnqx5B/P0FrXWiUipCax1rhUx2b/BSf4IiogFoVKU0I7vIgKy26OjRo3h6elKoUCEOHDhA/fr1jY4kRIay+ojZRCl1Uyl1Syl1C3BPs37TWgHtjd/Z62w5kjxsdIWSRfGWp4RtTnx8PNOmTaNTp0689dZb7Nq1S4qAyNMy7RForWWgGgO88f3B1OUDE541MIl4EocOHWLAgAE4Oztz6NAhqlevbnQkIR4pq7uGHJRSo1PuGhqklJKHzyzs9e8OpC6vGdJWho22IXfv3mX8+PH06NGDcePGsWnTJikCwmZk9U7zM+ABhAA9gC+tkshOvb86CP9zUUDyHUIeNcsbnEhk1+7du2nSpAnnzp0jJCSEN998U0YKFTYlq0/5jbTWbgBKqcWAr3Ui2RetNd/vPs2aQ8mTylcqVZSt73UyOJXIjps3bzJ+/Hg2btzIvHnz6NWrl9GRhHgi2b1rKEE+4eS+VX7hfPBr8APb9nzQ2aA04nF4e3szZMgQnn/+ecxmM2XLljU6khBPLKtC0DTN3UEKKJayrkieXKy0xdPlY/tOXn2gCLSoUY5fBrSSyWTyuKtXrzJ69GgOHjzITz/9RJcuXYyOJESOZVUIgrTWzayWxI5cuHGXNxf/mbq+eVQHGlaRupqXaa1ZtWoVo0ePpl+/fgQHB1OihMz9IPKHrAqBtloKO3Ly0i26zt6Tum7+5HlKFpUbsvKyCxcuMHToUEJDQ1m3bh1t2rQxOpIQuSqrd6BKSqlMJ5XXWn9lgTz5Wvj1mAeKwLjnn5IikIdprVm8eDEffvghQ4cOZdWqVRQtWtToWELkuqzehQoCJUm+JiByKDY+kQ4zd6auj+/egKHP1DEwkcjKqVOnGDhwILdu3WL79u24ubkZHUkIi8mqEERqradaLUk+tj7wPKNWBKauL3irBc83lgnJ86L7g8RNnz6diRMnMmrUKAoVkl6byN+y+g2XnkAumLrxKD/uP5O6Pv+fzaUI5FFmsxmTyUSxYsXw8fGhbt26RkcSwiqyerJY7ovLobnbTz5QBIL+3Y0eblUMTCQyEhcXxyeffELnzp0xmUzs2LFDioCwK1kNOnfdmkHym1b/2cblW/cA8KhRjpWD21KwgHSy8hpfX19MJhM1atQgICAAFxcXoyMJYXVy8tMC3lr8Z2oRqF6+OGuGynSEeU1MTAyTJ09m2bJlzJ49m759+8r4QMJuyfCWuSgxSdP/J1/2nrwKQNUyDjJkRB60c+dO3N3diYyMJCQkhH79+kkREHZNegS5qNe8fZjP/z1njxSBvCU6OpoPPvgAb29v5s+fz0svvWR0JCHyBOkR5JILN+6mFoE6FUtwdkZPCskcw3nGxo0bcXV1RSmF2WyWIiBEGtIjyCWvzNufurxldEcDk4i0rly5wqhRo/D19eWXX36hc2fppQnxMIt+ZFVKdVdKHVdKhSqlJmTRrrdSSiulPCyZx1J+PRSRenH4p3daSk8gD9Bas3z5ctzc3HB2diY4OFiKgBCZsFiPQClVEJgHdAUiAD+l1Aat9dGH2pUCRgF/pv8ueZ93SCRjVwcBUEBB56cqGZxIhIeHM3ToUMLCwti4cSMtW7Y0OpIQeZolP7q2AkK11qe11nHACiCjKZw+BT4HYi2YxSJi4hJ497+HU9eDpzxvYBqRlJTEggULaN68Oa1atcLf31+KgBDZYMlrBM5AeJr1CKB12gZKqeZANa31JqXUuMy+kVJqEDAIyFMTgk/ZcCR12X/SczKSqIFOnjzJwIEDiY2NZdeuXTRu3NjoSELYDMNOZiulCgBfAWMf1VZrvVBr7aG19qhYsaLlw2XDxehYVvknzzPct2U1KpSU4YmNkJCQwBdffEHbtm3p1asX+/fvlyIgxGOy5EfY80C1NOsuKdvuKwW4ArtSHuapDGxQSr2stfa3YK5c0eaz7anLH/VsaGAS+xUcHIzJZKJ06dL4+vpSu3ZtoyMJYZMs2SPwA+oppWoppYoAfYEN93dqraO11hW01jW11jUBH8AmisDlm39fznirTQ1KORQ2MI39uXfvHpMnT+a5555jyJAhbNu2TYqAEDlgsR6B1jpBKTUc2ELyJDc/aq2PKKWmAv5a6w1Zf4e8q93nO1KXP33F1cAk9sfHxweTyUS9evUIDAykatWqRkcSwuZZ9Oqm1tob8H5o2+RM2j5jySy5JfTyLeITk6dz7t1cRqq0ljt37jBp0iRWrFjBnDlzeOONN2R8ICFyiTz59Jie++rvOYc/e02mL7SG+1NFXr16FbPZTJ8+faQICJGL5H7Hx3Ax+u9rAysGtaFIIamjlnTjxg3ef/99tm7dyvfff0+PHj2MjiREviTvZI/h7R//fvi5TW1HA5Pkf+vXr8fV1ZUiRYpgNpulCAhhQdIjyKbV/uGcuHQbSL5TSFjGpUuXGDlyJAEBASxfvpyOHWUAPyEsTXoE2aC15psdJ1PXp/aSB5Zym9aapUuX4u7uTq1atQgKCpIiIISVSI8gGzYEXSD8+l0Ato3pKBcqc1lYWBhDhgzhwoULeHt706JFC6MjCWFXpEfwCElJmlErAgEoWqgAdSuVMjZQPpKUlMT8+fNp0aIF7dq1w8/PT4qAEAaQHsEj9FlwMHX5V5mEPtecOHECT09PEhIS2LNnDw0byjAdQhhFegRZ8D97Hf9zUQD0a1UNV+cyBieyfQkJCXz++ec8/fTTvP766+zdu1eKgBAGkx5BFl7//u/ewPRX5eGxnAoMDMRkMuHo6Iifnx+1atUyOpIQAukRZGreztDU5Zm93eUCcQ7Exsby0Ucf0a1bN0aMGMGWLVukCAiRh0iPIAOHzkUxa8txACqVKkqfltUe8RUiMwcOHMBkMtGwYUOCgoKoUqWK0ZGEEA+RQvAQrTW9vzuQuu4zsYuBaWzX7du3+fDDD1mzZg1z586ld+/eRkcSQmRCTg095P3VwanLM193p0ABOSX0uLZu3Yqbmxs3b97EbDZLERAij5MeQRqHw6L49XDy9JOta5Wnj4ecEnocUVFRjBkzhp07d7JgwQKef/55oyMJIbJBegRpvDb/71NCywe2MTCJ7Vm7di2urq6ULFmSkJAQKQJC2BDpEaRRpFAB4hKS+LRXYwrKKaFsuXjxIsOHD8dsNrNy5Urat29vdCQhxGOSHkGKBbtPEZeQBEDvFjLz2KNorVmyZAnu7u7Ur1+fwMBAKQJC2CjpEaT4Ye8ZAIoVLkjxIvJjycrZs2cZPHgwly9fZsuWLTRr1szoSEKIHJAeAXA3LpGrt+8B8NuwdganybuSkpKYO3cuHh4ePPPMM/j6+koRECIfkI++wEq/sNTlpyrL6KIZ+euvv/D09ARg3759NGjQwOBEQojcIj0C/j4t1KtpVYOT5D3x8fFMnz6d9u3b069fP/bs2SNFQIh8xu57BFprzt9InnSmU/2KBqfJWw4fPozJZMLJyYlDhw5Ro4ZM0SlEfmT3PYJtxy6nLvdq6mxgkrzj7t27TJw4kRdeeIH33nuPzZs3SxEQIh+z+x7Bj/vOpC7LswPJ5/9NJhPu7u4EBwfj5ORkdCQhhIXZdSFISEzi4OlrAHz/ZnOD0xjr1q1bTJw4kXXr1jF37lxee+01oyMJIazErk8NHQ67kbr8fOPKxgUx2O+//46rqysxMTGYzWYpAkLYGbvuEczdcRIA57LF7HLimWvXrjFmzBj27NnDokWL6Nq1q9GRhBAGsOsewd6TVwHo3dy+LhJrrVm9ejWurq6UK1eOkJAQKQJC2DG77RFcjI5NXR7YsbaBSawrMjKSd999l+PHj7N27Vratm1rdCQhhMEs2iNQSnVXSh1XSoUqpSZksH+MUuqoUipYKbVdKWW1exQ3Bl0AoEyxwpRyKGytlzWM1poff/yRJk2a4OrqSkBAgBQBIQRgwR6BUqogMA/oCkQAfkqpDVrro2maBQAeWusYpdRQYCbwD0tlSmvJgbMAdKhXwRovZ6gzZ84waNAgoqKi+OOPP2jSpInRkYQQeYglewStgFCt9WmtdRywAuiVtoHWeqfWOiZl1QewyvjPWmsu3Uw+NdSubv4tBImJicyZM4eWLVvStWtXfHx8pAgIIdKx5DUCZyA8zXoE0DqL9iZgc0Y7lFKDgEEA1atXz3Gwq7fjSEjSALzaLH9eKD569Cgmk4kiRYpw4MAB6tevb3QkIUQelSfuGlJKvQl4ALMy2q+1Xqi19tBae1SsmPPxgHb+dTnldcGhcMEcf7+8JC4ujk8//ZROnTrxr3/9i507d0oREEJkyZI9gvNA2tnfXVK2PUAp9RzwEdBJa33PgnlS/XnmOgCuVctY4+Wsxt/fH5PJhLOzM4cPH6ZatWqP/iIhhN2zZI/AD6inlKqllCoC9AU2pG2glGoGLABe1lpfzuB7WMRvgcn1aMIL+WM45bt37/LBBx/Qs2dPPvjgAzZt2iRFQAiRbRYrBFrrBGA4sAU4BqzSWh9RSk1VSr2c0mwWUBJYrZQKVEptyOTb5Zrb9xJITLk+0Kx6WUu/nMXt3r0bd3d3wsLCCAkJ4Z///KddPiUthHhyFn2gTGvtDXg/tG1ymuXnLPn6GdkQeCF12ZbnJr558ybjx49n48aNzJs3j169ej36i4QQIgN54mKxNR0OiwLAzdl2rw9s2rQJV1dXEhMTMZvNUgSEEDliux+Jn1BIRDQAT9d1NDjJ47t69SqjR4/m4MGDLFmyhGeffdboSEKIfMDuegSnrtwGoFm1ssYGeQxaa1asWIGrqytOTk4EBwdLERBC5Bq76hEkJenUB8kaVbGNU0Pnz5/n3XffJTQ0lPXr19O6dVbP5AkhxOOzqx5BeFRM6rJzuWIGJnk0rTU//PADTZs2pVmzZhw+fFiKgBDCIuyqRxCccn2gTsUSeXp+4lOnTjFw4EBu377Njh07cHNzMzqSECIfs6sewZ17CQCULV7E4CQZS0xM5KuvvqJ169b07NmTgwcPShEQQlicXfUIjl+6BeTNW0fNZjMmk4nixYvj4+ND3bp1jY4khLATdtUj8DubPMbQvYREg5P8LS4ujk8++YTOnTtjMpnYvn27FAEhhFXZVY8gMSn570ZVShsbJIWvry8mk4maNWsSEBCAi4tVpmMQQogH2FUhuHk3HoDWtY19mCwmJobJkyezbNkyZs+eTd++fWV8ICGEYezq1ND5G3cBqFiyqGEZdu7cibu7O5GRkYSEhNCvXz8pAkIIQ9lNjyAuISl1uZSD9Q87OjqacePGsXnzZr777jtefPFFq2cQQoiM2E2P4G783xeICxW07mFv3LgRV1dXChQogNlsliIghMhT7KZHEHUnDoAKJa33DMGVK1cYOXIkfn5+LF26lGeeecZqry2EENllNz2CsOvJw0tcvR1n8dfSWrN8+XLc3NxwcXEhODhYioAQIs+ymx5B6OXkUUerly9u0dcJDw9n6NChhIWFsXHjRlq2bGnR1xNCiJyymx7B/WsElioESUlJfP/99zRv3pzWrVvj7+8vRUAIYRPspkeQlDL8dH2nUrn+vU+ePMnAgQOJjY1l165dNG7cONdfQwghLMVuegTxKYUgN28dTUhIYNasWbRt25ZXXnmF/fv3SxEQQtgcu+kRJKSML1G4YO48vBUcHIzJZKJMmTL4+vpSu3btXPm+QghhbXbTIzhz9Q6Q82cI7t27x+TJk+nSpQtDhgzhjz/+kCIghLBpdtMjKJxSAKJTxht6Ej4+PphMJurVq0dQUBBVq1bNrXhCCGEYuykERQslFwLnso8/ReWdO3eYNGkSK1asYM6cObzxxhsyPpAQIt+wm1NDKdeKUwtCdm3btg03NzeuXbuG2WymT58+UgSEEPmK3fQIknRyJSiQzTfxGzduMHbsWLZt28b333/PCy+8YMl4QghhGDvqESQXguxMWv/bb7/RuHFjHBwcCAkJkSIghMjX7KhHkPx3Vh2CS5cuMWLECAIDA/Hy8qJjx47WCSeEEAayux5BRqeGtNYsXboUd3d3ateuTVBQkBQBIYTdsJ8eQVLGhSAsLIzBgwcTGRmJt7c3LVq0MCKeEEIYxg57BCnrSUnMmzeP5s2b06FDB/z8/KQICCHskkV7BEqp7sAcoCCwSGs946H9RYFfgBbANeAfWuuzlshy/xpBgQKK48eP4+npSWJiInv37qVhw4aWeEkhhLAJFusRKKUKAvOAF4BGQD+lVKOHmpmAKK11XWA28Lml8uiUHsG6tb/Srl07+vTpI0VACCGwbI+gFRCqtT4NoJRaAfQCjqZp0wuYkrK8BvhWKaX0/XftXHT7TvIMZcFBwfj7+1OzZs3cfgkhhLBJlrxG4AyEp1mPSNmWYRutdQIQDTg+/I2UUoOUUv5KKf8rV648UZhypYpRvGASM6Z/KkVACCHSsIm7hrTWC4GFAB4eHk/UW/jurVa5mkkIIfILS/YIzgPV0qy7pGzLsI1SqhBQhuSLxkIIIazEkoXAD6inlKqllCoC9AU2PNRmA/CvlOXXgR2WuD4ghBAicxY7NaS1TlBKDQe2kHz76I9a6yNKqamAv9Z6A7AYWKqUCgWuk1wshBBCWJFFrxForb0B74e2TU6zHAu8YckMQgghsmY3TxYLIYTImBQCIYSwc1IIhBDCzkkhEEIIO6ds7W5NpdQV4NwTfnkF4GouxrEFcsz2QY7ZPuTkmGtorStmtMPmCkFOKKX8tdYeRuewJjlm+yDHbB8sdcxyakgIIeycFAIhhLBz9lYIFhodwAByzPZBjtk+WOSY7eoagRBCiPTsrUcghBDiIVIIhBDCzuXLQqCU6q6UOq6UClVKTchgf1Gl1MqU/X8qpWoaEDNXZeOYxyiljiqlgpVS25VSNYzImZsedcxp2vVWSmmllM3fapidY1ZK9Un5tz6ilFpu7Yy5LRu/29WVUjuVUgEpv989jMiZW5RSPyqlLiulzJnsV0qpb1J+HsFKqeY5flGtdb76Q/KQ16eA2kARIAho9FCbd4HvU5b7AiuNzm2FY+4MFE9ZHmoPx5zSrhSwB/ABPIzObYV/53pAAFAuZb2S0bmtcMwLgaEpy42As0bnzuExdwSaA+ZM9vcANgMKaAP8mdPXzI89glZAqNb6tNY6DlgB9HqoTS/g55TlNUAXpZSyYsbc9shj1lrv1FrHpKz6kDxjnC3Lzr8zwKfA50CsNcNZSHaOeSAwT2sdBaC1vmzljLktO8esgdIpy2WAC1bMl+u01ntInp8lM72AX3QyH6CsUqpKTl4zPxYCZyA8zXpEyrYM22itE4BowNEq6SwjO8eclonkTxS27JHHnNJlrqa13mTNYBaUnX/n+kB9pdR+pZSPUqq71dJZRnaOeQrwplIqguT5T0ZYJ5phHvf/+yPZxOT1Ivcopd4EPIBORmexJKVUAeAroL/BUaytEMmnh54hude3RynlprW+YWQoC+sHLNFaf6mUakvyrIeuWusko4PZivzYIzgPVEuz7pKyLcM2SqlCJHcnr1klnWVk55hRSj0HfAS8rLW+Z6VslvKoYy4FuAK7lFJnST6XusHGLxhn5985AtigtY7XWp8BTpBcGGxVdo7ZBKwC0FofBBxIHpwtv8rW//fHkR8LgR9QTylVSylVhOSLwRsearMB+FfK8uvADp1yFcZGPfKYlVLNgAUkFwFbP28MjzhmrXW01rqC1rqm1romyddFXtZa+xsTN1dk53f7N5J7AyilKpB8qui0FTPmtuwccxjQBUAp1ZDkQnDFqimtawPwdsrdQ22AaK11ZE6+Yb47NaS1TlBKDQe2kHzHwY9a6yNKqamAv9Z6A7CY5O5jKMkXZfoalzjnsnnMs4CSwOqU6+JhWuuXDQudQ9k85nwlm8e8BeimlDoKJALjtNY229vN5jGPBX5QSr1H8oXj/rb8wU4p5UVyMa+Qct3j30BhAK319yRfB+kBhAIxwDs5fk0b/nkJIYTIBfnx1JAQQojHIIVACCHsnBQCIYSwc1IIhBDCzkkhEEIIOyeFQIhsUkolKqUC0/ypqZR6RikVnbJ+TCn175S2abf/pZT6wuj8QmQm3z1HIIQF3dVaN027IWUI871a6xeVUiWAQKXUxpTd97cXAwKUUuu01vutG1mIR5MegRC5RGt9BzgE1H1o+10gkBwODCaEpUghECL7iqU5LbTu4Z1KKUeSxzQ68tD2ciSP97PHOjGFeDxyakiI7Et3aihFB6VUAJAEzEgZAuGZlO1BJBeBr7XWF62WVIjHIIVAiJzbq7V+MbPtSqlagI9SapXWOtDK2YR4JDk1JISFpQwHPQMYb3QWITIihUAI6/ge6Jhyl5EQeYqMPiqEEHZOegRCCGHnpBAIIYSdk0IghBB2TgqBEELYOSkEQghh56QQCCGEnZNCIIQQdu7/ASqW2GJUmPDPAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackInputData\n", + "from tensorflow_privacy.privacy.membership_inference_attack.data_structures import SlicingSpec\n", + "from tensorflow_privacy.privacy.membership_inference_attack.data_structures import AttackType\n", + "\n", + "import tensorflow_privacy.privacy.membership_inference_attack.plotting as plotting\n", + "\n", + "labels_train = np.argmax(y_train, axis=1)\n", + "labels_test = np.argmax(y_test, axis=1)\n", + "\n", + "input = AttackInputData(\n", + " logits_train = logits_train,\n", + " logits_test = logits_test,\n", + " loss_train = loss_train,\n", + " loss_test = loss_test,\n", + " labels_train = labels_train,\n", + " labels_test = labels_test\n", + ")\n", + "\n", + "# Run several attacks for different data slices\n", + "attacks_result = mia.run_attacks(input,\n", + " SlicingSpec(\n", + " entire_dataset = True,\n", + " by_class = True,\n", + " by_classification_correctness = True\n", + " ),\n", + " attack_types = [\n", + " AttackType.THRESHOLD_ATTACK,\n", + " AttackType.THRESHOLD_ENTROPY_ATTACK,\n", + " AttackType.LOGISTIC_REGRESSION])\n", + "\n", + "# Plot the ROC curve of the best classifier\n", + "fig = plotting.plot_roc_curve(\n", + " attacks_result.get_result_with_max_auc().roc_curve)\n", + "\n", + "# Print a user-friendly summary of the attacks\n", + "print(attacks_result.summary(by_slices = True))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "E9zwsPGFujVq" + }, + "source": [ + "## Compute privacy risk score\n", + "\n", + "This part shows how to use the privacy risk score.\n", + "\n", + "For each data slice, we compute privacy risk scores for both training and test data. We then set a threshold on risk scores (an input is inferred as a member if and only if its risk score is higher than the threshold) and compute the attack precision and recall values" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Privacy risk score analysis over slice: \"Entire dataset\"\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.56143, 0.92450)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=0\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.62567, 0.16380)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.56207, 0.89200)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=1\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.61458, 0.29340)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.56183, 0.79240)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=2\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.63724, 0.56740)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.61159, 0.84400)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=3\"\n", + " with 1.00000 as the threshold on privacy risk score, the precision-recall pair is (1.00000, 0.00100)\n", + " with 0.90000 as the threshold on privacy risk score, the precision-recall pair is (1.00000, 0.00100)\n", + " with 0.80000 as the threshold on privacy risk score, the precision-recall pair is (1.00000, 0.00100)\n", + " with 0.70000 as the threshold on privacy risk score, the precision-recall pair is (0.77273, 0.00340)\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.65255, 0.35120)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.60833, 0.77660)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=4\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.61174, 0.41280)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.59076, 0.74920)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=5\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.60540, 0.34520)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.58148, 0.89060)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=6\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.60929, 0.10760)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.57011, 0.69360)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=7\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.61250, 0.29400)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.56678, 0.85040)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=8\"\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.54417, 0.92520)\n", + "\n", + "Privacy risk score analysis over slice: \"CLASS=9\"\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.61538, 0.08640)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.56456, 0.78440)\n", + "\n", + "Privacy risk score analysis over slice: \"CORRECTLY_CLASSIFIED=True\"\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.52433, 0.59775)\n", + "\n", + "Privacy risk score analysis over slice: \"CORRECTLY_CLASSIFIED=False\"\n", + " with 0.70000 as the threshold on privacy risk score, the precision-recall pair is (0.70957, 0.36504)\n", + " with 0.60000 as the threshold on privacy risk score, the precision-recall pair is (0.68111, 0.67130)\n", + " with 0.50000 as the threshold on privacy risk score, the precision-recall pair is (0.63809, 0.87166)\n" + ] + } + ], + "source": [ + "# compute privacy risk scores on all given data slices\n", + "risk_score_results = mia.privacy_risk_score_analysis(input,\n", + " SlicingSpec(\n", + " entire_dataset = True,\n", + " by_class = True,\n", + " by_classification_correctness = True))\n", + "# print the summary of privacy risk score analysis\n", + "print(risk_score_results.summary())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "last_runtime": { + "build_target": "//learning/deepmind/public/tools/ml_python:ml_notebook", + "kind": "private" + }, + "name": "Membership inference codelab", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "metadata": { + "collapsed": false + }, + "source": [] + } + } + }, + "nbformat": 4, + "nbformat_minor": 1 +}