From 7eea74a6a1cf15e2d2bd890722400edd0e470db8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 5 May 2022 15:29:04 -0700 Subject: [PATCH] COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/privacy/pull/230 from npapernot:hyperparam 8835b9c4072e3e598aa49d605e7643a2c2e65988 PiperOrigin-RevId: 446832781 --- research/hyperparameters_2022/README.md | 30 + research/hyperparameters_2022/figure7.py | 199 ++++++ .../figure7_default_values.py | 47 ++ research/hyperparameters_2022/lr_acc.json | 14 + .../hyperparameters_2022/rdp_accountant.py | 622 ++++++++++++++++++ tensorflow_privacy/privacy/analysis/BUILD | 5 +- .../analysis/compute_noise_from_budget_lib.py | 18 +- tutorials/BUILD | 2 + tutorials/mnist_lr_tutorial.py | 15 +- 9 files changed, 935 insertions(+), 17 deletions(-) create mode 100644 research/hyperparameters_2022/README.md create mode 100644 research/hyperparameters_2022/figure7.py create mode 100644 research/hyperparameters_2022/figure7_default_values.py create mode 100644 research/hyperparameters_2022/lr_acc.json create mode 100644 research/hyperparameters_2022/rdp_accountant.py diff --git a/research/hyperparameters_2022/README.md b/research/hyperparameters_2022/README.md new file mode 100644 index 0000000..8e9ad12 --- /dev/null +++ b/research/hyperparameters_2022/README.md @@ -0,0 +1,30 @@ +# Hyperparameter Tuning with Renyi Differential Privacy + +### Nicolas Papernot and Thomas Steinke + +This repository contains the code used to reproduce some of the experiments in +our +[ICLR 2022 paper on hyperparameter tuning with differential privacy](https://openreview.net/forum?id=-70L8lpp9DF). + +You can reproduce Figure 7 in the paper by running `figure7.py`. It loads by +default values used to plot the figure contained in the paper, and we also +included a dictionary `lr_acc.json` containing the accuracy of a large number of +ML models trained with different learning rates. If you'd like to try our +approach to fine-tune your own parameters, you will have to modify the code that +interacts with this dictionary (`lr_acc` in the code from `figure7.py`). + +## Citing this work + +If you use this repository for academic research, you are highly encouraged +(though not required) to cite our paper: + +``` +@inproceedings{ +papernot2022hyperparameter, +title={Hyperparameter Tuning with Renyi Differential Privacy}, +author={Nicolas Papernot and Thomas Steinke}, +booktitle={International Conference on Learning Representations}, +year={2022}, +url={https://openreview.net/forum?id=-70L8lpp9DF} +} +``` diff --git a/research/hyperparameters_2022/figure7.py b/research/hyperparameters_2022/figure7.py new file mode 100644 index 0000000..a2d6a33 --- /dev/null +++ b/research/hyperparameters_2022/figure7.py @@ -0,0 +1,199 @@ +# Copyright 2022, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Code for reproducing Figure 7 of paper.""" + +import math +import matplotlib.pyplot as plt +import numpy as np +import rdp_accountant + +# pylint: disable=bare-except +# pylint: disable=g-import-not-at-top +# pylint: disable=g-multiple-import +# pylint: disable=missing-function-docstring +# pylint: disable=redefined-outer-name + +#################################################### +# This file loads default values to reproduce +# figure 7 from the paper. If you'd like to +# provide your own value, modify the variables +# in the if statement controlled by this variable. +#################################################### + +load_values_to_reproduce_paper_fig = True + + +def repeat_logarithmic_rdp(orders, rdp, gamma): + n = len(orders) + assert len(rdp) == n + assert min(orders) >= 1 + rdp_out = [None] * n + for i in range(n): + if orders[i] == 1: + continue # unfortunately the formula doesn't work in this case + for j in range(n): + # Compute (orders[i],eps)-RDP bound on A_gamma given that Q satisfies + # (orders[i],rdp[i])-RDP and (orders[j],rdp[j])-RDP + eps = rdp[i] + ( + 1 - 1 / orders[j]) * rdp[j] + math.log(1 / gamma - 1) / orders[j] + ( + math.log(1 / gamma - 1) - math.log(math.log(1 / gamma))) / ( + orders[i] - 1) + if rdp_out[i] is None or eps < rdp_out[i]: + rdp_out[i] = eps + return rdp_out + + +def repeat_geometric_rdp(orders, rdp, gamma): + n = len(orders) + assert len(rdp) == n + assert min(orders) >= 1 + rdp_out = [None] * n + for i in range(n): + if orders[i] == 1: + continue # formula doesn't work in this case + for j in range(n): + eps = rdp[i] + 2 * (1 - 1 / orders[j]) * rdp[j] + ( + 2 / orders[j] + 1 / (orders[i] - 1)) * math.log(1 / gamma) + if rdp_out[i] is None or eps < rdp_out[i]: + rdp_out[i] = eps + return rdp_out + + +def repeat_negativebinomial_rdp(orders, rdp, gamma, eta): + n = len(orders) + assert len(rdp) == n + assert min(orders) >= 1 + assert 0 < gamma < 1 + assert eta > 0 + rdp_out = [None] * n + # foo = log(eta/(1-gamma^eta)) + foo = math.log(eta) - math.log1p(-math.pow(gamma, eta)) + for i in range(n): + if orders[i] == 1: + continue # forumla doesn't work for lambda=1 + for j in range(n): + eps = rdp[i] + (1 + eta) * (1 - 1 / orders[j]) * rdp[j] - ( + (1 + eta) / orders[j] + 1 / + (orders[i] - 1)) * math.log(gamma) + foo / (orders[i] - 1) + ( + 1 + eta) * math.log1p(-gamma) / orders[j] + if rdp_out[i] is None or eps < rdp_out[i]: + rdp_out[i] = eps + return rdp_out + + +def repeat_poisson_rdp(orders, rdp, tau): + n = len(orders) + assert len(rdp) == n + assert min(orders) >= 1 + rdp_out = [None] * n + for i in range(n): + if orders[i] == 1: + continue # forumula doesn't work with lambda=1 + _, delta, _ = rdp_accountant.get_privacy_spent( + orders, rdp, target_eps=math.log1p(1 / (orders[i] - 1))) + rdp_out[i] = rdp[i] + tau * delta + math.log(tau) / (orders[i] - 1) + return rdp_out + + +if load_values_to_reproduce_paper_fig: + from figure7_default_values import orders, rdp, lr_acc, num_trials, lr_rates, gammas, non_private_acc +else: + orders = [] # Complete with the list of orders + rdp = [] # Complete with the list of RDP + lr_acc = {} # Complete with a dictionary such that keys + # are learning rates and values are the + # corresponding model's accuracy + num_trials = 1000 # num_trials to average results over + lr_rates = np.asarray([]) # 1D array of learning rate candidates + gammas = np.asarray( + []) # 1D array of gamma parameters to the random distributions. + non_private_acc = 1. # accuracy of a non-private run (for plotting only) + +for dist_id in range(4): + res_x = np.zeros_like(gammas) + res_y = np.zeros_like(res_x) + res_y_max = non_private_acc * np.ones_like(res_x) + for gamma_id, gamma in enumerate(gammas): + expected = (1 / gamma - 1) / np.log(1 / gamma) + best_acc_trials = [] + for trial in range(num_trials): + if dist_id == 0: + K = np.random.logseries(1 - gamma) + label = 'logarithmic distribution $\\eta=0$' + color = 'b' + eps = repeat_logarithmic_rdp(orders, rdp, gamma) + elif dist_id == 1: + if load_values_to_reproduce_paper_fig and gamma < 1e-4: + continue + K = np.random.geometric(gamma) + label = 'geometric distribution $\\eta=1$' + color = 'g' + eps = repeat_geometric_rdp(orders, rdp, gamma) + elif dist_id == 2: + if load_values_to_reproduce_paper_fig and gamma < 1e-07: + continue + eta = 0.5 + K = 0 + while K == 0: + K = np.random.negative_binomial(eta, gamma) + label = 'negative binomial $\\eta=0.5$' + color = 'k' + eps = repeat_negativebinomial_rdp(orders, rdp, gamma, eta) + elif dist_id == 3: + if load_values_to_reproduce_paper_fig and gamma < 0.0015: + continue + gamma_factor = 100 + K = np.random.poisson(gamma * gamma_factor) + label = 'poisson distribution' + color = 'm' + eps = repeat_poisson_rdp(orders, rdp, gamma * gamma_factor) + best_acc = 0. + best_lr = -1. + for k in range(K): + # pick a hyperparam candidate uniformly at random + j = np.random.randint(0, len(lr_rates)) + lr_candidate = lr_rates[j] + try: + acc = lr_acc[str(lr_candidate)] + except: + print('lr - acc pair missing for ' + str(lr_candidate)) + acc = 0. + if best_acc < acc: + best_acc = acc + best_lr = lr_candidate + best_acc_trials.append(best_acc) + try: + res_x[gamma_id] = np.min(eps) + res_y[gamma_id] = np.mean(best_acc_trials) + except: + print('skipping ' + str(gamma_id)) + if dist_id == 0: + plt.hlines( + res_y_max[0], + xmin=-1., + xmax=20., + color='r', + label='baseline (non-private search)') + if dist_id >= 1: + res_x = res_x[2:] + res_y = res_y[2:] + plt.plot(res_x, res_y, label=label, color=color) + +if load_values_to_reproduce_paper_fig: + plt.xlim([0.5, 8.]) + plt.ylim([0.85, 0.97]) +plt.xlabel('Privacy budget') +plt.ylabel('Model Accuracy for Best Hyperparameter') +plt.legend(loc='lower right') +plt.savefig('rdp_hyper_search.pdf', bbox_inches='tight') diff --git a/research/hyperparameters_2022/figure7_default_values.py b/research/hyperparameters_2022/figure7_default_values.py new file mode 100644 index 0000000..4bb3c52 --- /dev/null +++ b/research/hyperparameters_2022/figure7_default_values.py @@ -0,0 +1,47 @@ +# Copyright 2022, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Default values for generating Figure 7.""" + +import json +import numpy as np + +orders = ([1.25, 1.5, 1.75, 2., 2.25, 2.5, 3., 3.5, 4., 4.5] + + list(range(5, 64)) + [128, 256, 512]) +rdp = [ + 2.04459751e-01, 2.45818210e-01, 2.87335988e-01, 3.29014798e-01, + 3.70856385e-01, 4.12862542e-01, 4.97375951e-01, 5.82570265e-01, + 6.68461534e-01, 7.55066706e-01, 8.42403732e-01, 1.01935100e+00, + 1.19947313e+00, 1.38297035e+00, 1.57009549e+00, 1.76124790e+00, + 1.95794503e+00, 2.19017390e+00, 4.48407479e+00, 3.08305394e+02, + 4.98610133e+03, 1.11363692e+04, 1.72590079e+04, 2.33487231e+04, + 2.94091123e+04, 3.54439803e+04, 4.14567914e+04, 4.74505356e+04, + 5.34277419e+04, 5.93905358e+04, 6.53407051e+04, 7.12797586e+04, + 7.72089762e+04, 8.31294496e+04, 8.90421151e+04, 9.49477802e+04, + 1.00847145e+05, 1.06740819e+05, 1.12629335e+05, 1.18513163e+05, + 1.24392717e+05, 1.30268362e+05, 1.36140424e+05, 1.42009194e+05, + 1.47874932e+05, 1.53737871e+05, 1.59598221e+05, 1.65456171e+05, + 1.71311893e+05, 1.77165542e+05, 1.83017260e+05, 1.88867175e+05, + 1.94715404e+05, 2.00562057e+05, 2.06407230e+05, 2.12251015e+05, + 2.18093495e+05, 2.23934746e+05, 2.29774840e+05, 2.35613842e+05, + 2.41451813e+05, 2.47288808e+05, 2.53124881e+05, 2.58960080e+05, + 2.64794449e+05, 2.70628032e+05, 2.76460867e+05, 2.82292992e+05, + 2.88124440e+05, 6.66483142e+05, 1.41061455e+06, 2.89842152e+06 +] +with open("lr_acc.json", "r") as dict_f: + lr_acc = json.load(dict_f) +num_trials = 1000 +lr_rates = np.logspace(np.log10(1e-4), np.log10(1.), num=1000)[-400:] +gammas = np.asarray( + [1e-07, 8e-06, 1e-04, 0.00024, 0.0015, 0.0035, 0.025, 0.05, 0.1, 0.2, 0.5]) +non_private_acc = 0.9594 diff --git a/research/hyperparameters_2022/lr_acc.json b/research/hyperparameters_2022/lr_acc.json new file mode 100644 index 0000000..39fe4bc --- /dev/null +++ b/research/hyperparameters_2022/lr_acc.json @@ -0,0 +1,14 @@ +// Copyright 2022, The TensorFlow Privacy Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +{"0.0018082449348779518": 0.1289, "0.011430311291144786": 0.3575, "0.001476281471909391": 0.1313, "0.0016188596901781992": 0.1285, "0.00326222200971167": 0.1571, "0.13524808704178753": 0.9296, "0.4442706749606883": 0.9563, "0.1124737178364752": 0.9071, "0.05791122647641758": 0.7135, "0.00014065272421052364": 0.1029, "0.00013186214013947485": 0.1074, "0.00010765446128423159": 0.1366, "0.0007671581176779302": 0.1089, "0.02119953457536071": 0.4397, "0.024569164629827903": 0.4574, "0.44019351852088745": 0.9549, "0.04556786265841064": 0.6142, "0.007548799281653432": 0.3069, "0.00010471768194855202": 0.1415, "0.12108297502320395": 0.9153, "0.0013339056900390587": 0.1205, "0.002298059988758851": 0.1558, "0.01208042134677329": 0.3796, "0.3560832552629278": 0.959, "0.000897331581458352": 0.1113, "0.30163343472591975": 0.9562, "0.0001270878709202058": 0.1251, "0.4962444877628913": 0.9572, "0.00018040819287193827": 0.1028, "0.0508987019351968": 0.6508, "0.0026880010215376074": 0.1552, "0.0012975171686575868": 0.12, "0.00276338529005317": 0.1611, "0.022612800663372772": 0.4427, "0.007479522515621821": 0.3007, "0.14560059950206486": 0.9326, "0.5196557243827657": 0.9572, "0.2010496416260499": 0.9503, "0.8708431497690724": 0.9554, "0.7583677914997191": 0.9507, "0.007760503335133571": 0.3026, "0.0008648423275731727": 0.1113, "0.00026572011053245065": 0.1028, "0.0011942000281335325": 0.1162, "0.26756384445520454": 0.9566, "0.09098272894455559": 0.8689, "0.017467962151272456": 0.4224, "0.0022353696459097966": 0.15, "0.008126619200091946": 0.3202, "0.016527920614648955": 0.4179, "0.7796360130405229": 0.9577, "0.002095662399480433": 0.1393, "0.0016642601764859037": 0.1274, "0.00012476595526308696": 0.1453, "0.0004620241371751313": 0.1084, "0.08608647696149245": 0.8557, "0.025728259674479323": 0.4649, "0.005517492376129128": 0.2202, "0.0006379766808606282": 0.1061, "0.002920555512182745": 0.1695, "0.00010280447320933097": 0.1144, "0.2029118018046678": 0.9505, "0.1524695727017573": 0.9348, "0.0034796979038876916": 0.1384, "0.0001722805447131394": 0.1028, "0.00025847135074695635": 0.1028, "0.00013431611700460153": 0.1046, "0.0595353313081437": 0.7563, "0.00011067360180959734": 0.1663, "0.5646141419303667": 0.9566, "0.0008335293965098195": 0.1098, "0.00019421746814890265": 0.1028, "0.4126820845702952": 0.9568, "0.6543586018883236": 0.9559, "0.0036438589837635444": 0.1515, "0.23954073587208768": 0.9544, "0.04232789065573549": 0.6013, "0.003002461709085549": 0.1373, "0.29071233772725785": 0.9541, "0.2066880249629082": 0.9488, "0.04005000757873611": 0.5788, "0.27506760079080644": 0.9576, "0.010327947319189514": 0.3406, "0.0038157646612712484": 0.1832, "0.0003842240846055061": 0.1106, "0.17030650292528443": 0.9431, "0.3187891292677646": 0.9561, "0.39409016404034475": 0.9561, "0.04951020159556351": 0.6786, "0.0062776601058065": 0.2696, "0.01288576213185518": 0.3752, "0.8316104153230962": 0.9544, "0.058447611316336315": 0.6902, "0.4014242490499322": 0.9559, "0.6078323128297229": 0.959, "0.07850456200204509": 0.8312, "0.03895745615775501": 0.5672, "0.003115254223555488": 0.1419, "0.09704808877380307": 0.8821, "0.9549485639791966": 0.9465, "0.01829204504846294": 0.4151, "0.00015002933220192182": 0.1028, "0.4567301270168747": 0.9542, "0.0004452958509942655": 0.1031, "0.00146273335620113": 0.1229, "0.0010115911122238297": 0.117, "0.9461848194722002": 0.9505, "0.1567455410205595": 0.939, "0.045989920905224385": 0.6048, "0.3158635408267819": 0.9594, "0.00217438947560008": 0.1412, "0.0010399609139541202": 0.1154, "0.4361537789208006": 0.9502, "0.0009660174799522646": 0.1125, "0.00029955493343598134": 0.1028, "0.00042917323784221574": 0.103, "0.21844360711494262": 0.9527, "0.0009056428379445294": 0.1111, "0.005831305113526219": 0.2301, "0.2934049709215787": 0.9541, "0.0005160748710385908": 0.1034, "0.1233363497913776": 0.9187, "0.09267593301146883": 0.877, "0.8015006961565405": 0.9519, "0.0061064075422320365": 0.2578, "0.22664980792736927": 0.9552, "0.0005981040962380944": 0.1047, "0.9119267598459299": 0.948, "0.0003703126675869927": 0.1047, "0.0023843904700937206": 0.1448, "0.04684580115873045": 0.6412, "0.5912508413831875": 0.9586, "0.1327770829355429": 0.9237, "0.28018665564591955": 0.9558, "0.016077044216738237": 0.4081, "0.0059398266939203566": 0.2415, "0.01387209780541621": 0.4043, "0.009862658461312821": 0.3361, "0.0837380653526649": 0.8465, "0.019692202554791715": 0.4305, "0.00021494746734379805": 0.1028, "0.06650018030431118": 0.7852, "0.0002757023325609583": 0.1073, "0.730909932860291": 0.9551, "0.00741088151564157": 0.2961, "0.09976977642363201": 0.8678, "0.00263889081445751": 0.1694, "0.021395888713434216": 0.431, "0.0029476062551248585": 0.1679, "0.002406475150015424": 0.1494, "0.23086779941871696": 0.9548, "0.005268921421350677": 0.2117, "0.00099310918137498": 0.1128, "0.00016003103137387001": 0.1028, "0.013369837418249465": 0.3909, "0.0002860595535175742": 0.1062, "0.05737976414214134": 0.7405, "0.05898896425508499": 0.7536, "0.0006680743915695614": 0.1063, "0.0010890229622637305": 0.1169, "0.1267952846786434": 0.9211, "0.0008490415204088747": 0.1122, "0.26026478819690047": 0.9572, "0.02847439166467248": 0.4797, "0.0018589566796356884": 0.1331, "0.006573820143409586": 0.2664, "0.01091535935331391": 0.3595, "0.0009839952296278227": 0.1126, "0.2555097090352507": 0.9556, "0.02669478494034321": 0.4713, "0.00032848573660300434": 0.1032, "0.18504070195423022": 0.9475, "0.19737643263002555": 0.9516, "0.12796968682159415": 0.9235, "0.011643031329208768": 0.3644, "0.003232283978181381": 0.1404, "0.04514967772036102": 0.6271, "0.00022508800520954617": 0.1028, "0.005125186927053333": 0.1906, "0.01578331405652118": 0.4175, "0.07707027114212295": 0.8008, "0.07496781874966876": 0.8248, "0.008829699955494083": 0.3229, "0.009331897715733239": 0.33, "0.0009140310748756232": 0.1134, "0.004301635758106795": 0.2233, "0.000263281546564802": 0.1028, "0.1937703337477989": 0.9467, "0.0003737425742391064": 0.1029, "0.01699144172034626": 0.4161, "0.0435149650092505": 0.6146, "0.00031659241119835204": 0.1028, "0.6134621717992506": 0.9554, "0.015072253093107555": 0.4072, "0.00018891927762076664": 0.1028, "0.5148867450137492": 0.9573, "0.0037807466635993493": 0.1756, "0.006220048825634711": 0.2555, "0.0006321218475812448": 0.1057, "0.0008890965989529158": 0.1112, "0.005724766239702178": 0.2132, "0.15819734815786": 0.9394, "0.00033459891205499747": 0.1028, "0.002614673211801092": 0.1483, "0.0010991097009294972": 0.1149, "0.012534242654613994": 0.3829, "0.004804870439655134": 0.2239, "0.00016604286571875296": 0.1028, "0.01795784647002095": 0.4172, "0.46096044868284336": 0.9584, "0.026942037136818822": 0.4739, "0.013005112521734086": 0.3944, "0.00036691423784024937": 0.1031, "0.0007886728615614156": 0.1102, "0.0007192118872221193": 0.1098, "0.00014593657991557562": 0.1029, "0.5008407989848213": 0.9594, "0.00016151326935030913": 0.1029, "0.009505565920101192": 0.3449, "0.04311561990318229": 0.6016, "0.012767507043192657": 0.3837, "0.006050367879391224": 0.2536, "0.161141427725302": 0.9403, "0.15388177500383463": 0.9391, "0.3833395101766601": 0.9566, "0.002566902715491952": 0.1683, "0.00014728827239075018": 0.1029, "0.11887076977119033": 0.9108, "0.45253862781701665": 0.9549, "0.0019646866461804444": 0.1374, "0.011750871309048075": 0.3624, "0.02005867779508234": 0.4311, "0.001775208011717636": 0.1336, "0.00016913295170296487": 0.1028, "0.022405278693000182": 0.4412, "0.0016796748720926532": 0.1276, "0.5101615314749833": 0.9579, "0.00040607720257003656": 0.1031, "0.0002314005380130654": 0.1028, "0.0001542368646296629": 0.1029, "0.002001249798969037": 0.1394, "0.044324785912403956": 0.6095, "0.014933932161242518": 0.4041, "0.0030866649433372744": 0.1564, "0.00694771254846024": 0.2818, "0.0008809371904473991": 0.1108, "0.751408106111697": 0.9525, "0.008911482322840203": 0.3217, "0.000872852662384838": 0.1109, "0.00037720424934169976": 0.1036, "0.0011616226326085018": 0.1172, "0.0005555776222398878": 0.1045, "0.7044462277299037": 0.9549, "0.00010666649582795387": 0.1165, "0.0045884041264547615": 0.2451, "0.08451366330684713": 0.8449, "0.012305240043592616": 0.3625, "0.007618717702322999": 0.3012, "0.0001630092360979741": 0.1028, "0.18165997883753268": 0.9482, "0.19920457084538692": 0.9513, "0.008994022174092044": 0.3267, "0.0009749649183484086": 0.1126, "0.000209083769055575": 0.1028, "0.011969557023590429": 0.37, "0.0006036438506075864": 0.1049, "0.00018376562003881704": 0.1028, "0.0005607234882852032": 0.1045, "0.004381689931514195": 0.2571, "0.0006498495354469888": 0.1052, "0.021004982416539152": 0.4459, "0.0001706994934038408": 0.1028, "0.020431973201952704": 0.4345, "0.02324697059985648": 0.4447, "0.10940547072057424": 0.8984, "0.029004304938639917": 0.4854, "0.09885417021919574": 0.8913, "0.0008033501977124734": 0.1101, "0.05633142670601358": 0.7387, "0.0008569055051268347": 0.1097, "0.001560246414366368": 0.129, "0.0010023075482838654": 0.1144, "0.1867545842761076": 0.945, "0.013744790926775366": 0.3896, "0.06899837121430018": 0.8002, "0.060643293954080614": 0.759, "0.0007531420165974376": 0.1096, "0.10740661533334334": 0.9047, "0.33076497807442423": 0.9553, "0.10256779307444219": 0.8893, "0.009954008287621519": 0.3323, "0.7445122910795136": 0.954, "0.5542985515684669": 0.9577, "0.8789090653419954": 0.9569, "0.2105345242766706": 0.9521, "0.6727099135712337": 0.9564, "0.009418331534647951": 0.3323, "0.0005305480525369575": 0.1048, "0.00105931476351837": 0.1146, "0.00024456166834524467": 0.1028, "0.0068839520696454964": 0.2815, "0.0001514189325304352": 0.1028, "0.0021346630333242455": 0.1432, "0.00034398264890229247": 0.106, "0.00013808297652180923": 0.1036, "0.0002033800305846982": 0.1028, "0.008431909292866252": 0.319, "0.0006558685659571436": 0.1056, "0.642403365939419": 0.9577, "0.004070142453219439": 0.2245, "0.07359814475265763": 0.8155, "0.19556507158659492": 0.9432, "0.025026400964179192": 0.4541, "0.034557199367621395": 0.5215, "0.0016952323415541214": 0.1371, "0.53917746403875": 0.9559, "0.00015710723892474503": 0.1028, "0.05184593543892913": 0.6349, "0.0002782559402207126": 0.1041, "0.00029680586086656025": 0.1118, "0.1429404533431761": 0.9329, "0.004032789982193705": 0.2254, "0.00022927693128656486": 0.103, "0.00030795587129142264": 0.1029, "0.0010495932305582276": 0.1136, "0.0020764301072557748": 0.1309, "0.0013713147177539457": 0.1234, "0.032100108955431715": 0.5115, "0.0006205728806776501": 0.1046, "0.0001432702953409831": 0.1029, "0.0003315282342319423": 0.1044, "0.0011403996019700331": 0.1172, "0.00011169868184678226": 0.1751, "0.004985373463873894": 0.2099, "0.10544427935261684": 0.9022, "0.001183240627458378": 0.1173, "0.0002731721598441376": 0.1029, "0.010423606739764012": 0.3538, "0.4165044248545185": 0.9575, "0.07996554525892346": 0.834, "0.005620173848083188": 0.2189, "0.0006092349152400712": 0.1093, "0.001069126339173476": 0.1168, "0.0001393619274224142": 0.1029, "0.02179406984302956": 0.4473, "0.1919920665593285": 0.9494, "0.10642092440647247": 0.8988, "0.029817722900196734": 0.4821, "0.0007462302891391107": 0.1102, "0.03330600343624589": 0.5143, "0.0053669769455404765": 0.1983, "0.14694918006248173": 0.934, "0.012650337203959038": 0.3907, "0.0021150728248687948": 0.1401, "0.08768856094587427": 0.8594, "0.0018418966807997109": 0.1382, "0.34636941773717345": 0.9555, "0.0024287643824604504": 0.1477, "0.02719157943036019": 0.4744, "0.05379361503980703": 0.7198, "0.03686095362172158": 0.5487, "0.155307057393346": 0.9363, "0.00021894767628566206": 0.1028, "0.17188391428171457": 0.9425, "0.002520004993764092": 0.1699, "0.0036776091016010306": 0.1533, "0.0023193450592744274": 0.1432, "0.0009571521538991865": 0.1113, "0.0024739641008868114": 0.1642, "0.00029408201705870637": 0.1044, "0.00546685729972018": 0.204, "0.4827070965603183": 0.9568, "0.390473523688556": 0.9565, "0.00019601634743191858": 0.1028, "0.9375015015145289": 0.9496, "0.1403289084785873": 0.9305, "0.5751217071841613": 0.958, "0.004504573251759458": 0.2344, "0.0012505385872903913": 0.12, "0.008748668120479914": 0.3246, "0.29612254379880343": 0.9573, "0.408894822629486": 0.9566, "0.07923168624866253": 0.8299, "0.25787628875938007": 0.9546, "0.00010865157746525384": 0.1307, "0.6248788072006894": 0.9594, "0.012419213527017834": 0.3668, "0.24175940791691308": 0.9557, "0.02081221569986339": 0.4424, "0.006162966255132942": 0.2643, "0.047717609489387455": 0.6453, "0.0355263467657814": 0.5352, "0.008510007247122246": 0.3206, "0.0003806979871402284": 0.1408, "0.0033537101520029287": 0.1379, "0.06468607661546327": 0.7789, "0.1596626022101425": 0.9387, "0.0028937530190509503": 0.1633, "0.00320262069365765": 0.1461, "0.001227691047988359": 0.1188, "0.0001355601785329369": 0.1036, "0.028738126918510663": 0.4839, "0.0018761746914391195": 0.1286, "0.37982153061907364": 0.9563, "0.010139254075588154": 0.3458, "0.0004371602248248502": 0.1033, "0.0027380251779278576": 0.1669, "0.9908228099003797": 0.9504, "0.0035444556739704356": 0.1452, "0.2725432531281028": 0.9588, "0.0027128978003724646": 0.17, "0.13776507695490536": 0.9294, "0.21445260759716675": 0.9527, "0.7109709432312431": 0.9551, "0.0034161232685855285": 0.1374, "0.22046687352394098": 0.9529, "0.05281079711934331": 0.7061, "0.013493671405883065": 0.3884, "0.9203731996618221": 0.9498, "0.006394488428556937": 0.258, "0.0015317404637020797": 0.1238, "0.00011483124145435111": 0.1636, "0.005220567527846975": 0.2213, "0.004939621743878321": 0.2233, "0.04042095839796306": 0.5829, "0.0011299339380332217": 0.1193, "0.0015459277364194785": 0.1256, "0.038599936176797675": 0.5617, "0.6306665540567405": 0.9558, "0.0004023505548869293": 0.1067, "0.0027889802923804397": 0.1541, "0.125631660247412": 0.918, "0.6365079081295572": 0.9575, "0.0038867766908926675": 0.1962, "0.0039591102664684585": 0.2063, "0.16719497597319902": 0.9427, "0.03520031472796679": 0.5347, "0.00022302232979659386": 0.1028, "0.0015746977146430868": 0.1236, "0.0001368157627967472": 0.1048, "0.00588531577519145": 0.2033, "0.00029138317048327885": 0.1254, "0.13904108340900698": 0.9288, "0.010617591834830001": 0.3454, "0.26267541037238357": 0.956, "0.10447659715608042": 0.8943, "0.1902301188668944": 0.9481, "0.06528521141127848": 0.785, "0.06234401888627864": 0.7667, "0.04079534503452449": 0.5786, "0.0014899550728528536": 0.1235, "0.0012390621569479156": 0.1182, "0.055814462494549605": 0.6643, "0.31009266359319265": 0.9565, "0.0005113387538414326": 0.1036, "0.6915758828738525": 0.9552, "0.017629753752872058": 0.4239, "0.017793043899185772": 0.4218, "0.0003408258547423452": 0.1028, "0.0019828839491270716": 0.1389, "0.00013308347246540747": 0.1042, "0.004760775230226368": 0.2249, "0.028213076759394707": 0.482, "0.0012621213145225473": 0.119, "0.03824569722466999": 0.5636, "0.011325413151528114": 0.3549, "0.02769761935036891": 0.4793, "0.0009483681866285928": 0.1127, "0.36271002523306484": 0.9568, "0.0031732296347349765": 0.1395, "0.5244688749495119": 0.9567, "0.009077326525210224": 0.3185, "0.0003023294684405776": 0.1071, "0.17670435260889467": 0.9431, "0.8164167604921464": 0.953, "0.03151363484866479": 0.5053, "0.013125568357718428": 0.3919, "0.1135154708920999": 0.9087, "0.0001": 0.114, "0.0640924401935645": 0.7821, "0.29886528735503826": 0.9565, "0.015494950393146316": 0.4172, "0.0002514203348142797": 0.1028, "0.021594061521035653": 0.441, "0.00010375666787451859": 0.1221, "0.0003949995461220643": 0.1032, "0.9637934799615786": 0.9489, "0.3593813663804626": 0.9571, "0.11041880508541602": 0.9069, "0.01714881969870539": 0.4204, "0.014393226447194065": 0.4073, "0.21643890860640203": 0.9522, "0.08145371766280746": 0.8368, "0.00011273325637104872": 0.1754, "0.0015176833902834053": 0.1249, "0.0007060717714137773": 0.108, "0.002543345761304648": 0.1625, "0.00014195547660501016": 0.1036, "0.0005454271305329836": 0.1042, "0.07028244264308345": 0.806, "0.01592950212572123": 0.4066, "0.0045462954695324": 0.2401, "0.0264498018242772": 0.4636, "0.003292437333007769": 0.1403, "0.0009224970052592174": 0.1116, "0.0005926151812475553": 0.1054, "0.06008675891719687": 0.7573, "0.0004213321743847289": 0.1029, "0.10351779556301763": 0.8985, "0.0024968784288843266": 0.1494, "0.001216424293857368": 0.1199, "0.05685317913873753": 0.7382, "0.8089243486805938": 0.9417, "0.014130259905995337": 0.3922, "0.737679760252773": 0.9541, "0.004422273980505897": 0.2438, "0.01915500555573528": 0.4339, "0.3044272212064303": 0.9544, "0.6483534286054721": 0.9564, "0.03652267364308175": 0.544, "0.0011723818032865986": 0.1172, "0.17508270317357252": 0.9441, "0.1783410220710008": 0.9416, "0.0033229325163989716": 0.1529, "0.00805203967082547": 0.3137, "0.6665363268124913": 0.9583, "0.1365007806546014": 0.9212, "0.026207066964838526": 0.4691, "0.00025609931002584595": 0.1028, "0.05137013543351344": 0.682, "0.030653952950565267": 0.4986, "0.0002706652070033241": 0.1033, "0.00028870909173592344": 0.1063, "0.005416686911033152": 0.2189, "0.8470868266557402": 0.9553, "0.012192312516491107": 0.3582, "0.0005504789807854967": 0.1044, "0.00024231727942376007": 0.1028, "0.00010186101701559753": 0.1093, "0.5967271195973316": 0.9582, "0.00227697025538168": 0.144, "0.09794696670695395": 0.8825, "0.2651083601908539": 0.9564, "0.00015856239617711372": 0.1029, "0.16414029711444664": 0.9381, "0.2245697995539774": 0.9523, "0.006453715401646702": 0.2575, "0.00022097561147959031": 0.1028, "0.0024512600620333396": 0.1611, "0.00031368698245668765": 0.1365, "0.007342870447166758": 0.2797, "0.1510703304486654": 0.9299, "0.0003376980310825091": 0.1044, "0.0001294529978227916": 0.1165, "0.0031441083031472647": 0.1512, "0.25084150592775384": 0.955, "0.000643885742724042": 0.1115, "0.00010092621909870473": 0.1122, "0.01730765534195726": 0.4129, "0.001911090621689138": 0.132, "0.03269749744511768": 0.5164, "0.0822081575524054": 0.8493, "0.013247139878661175": 0.3818, "0.09440064789417603": 0.8755, "0.0042621588290153245": 0.2326, "0.002590677858688006": 0.1642, "0.00028083319988231727": 0.1029, "0.00017387624002162505": 0.1028, "0.8628512566366896": 0.9541, "0.04860564232142134": 0.6585, "0.823978568452852": 0.9466, "0.0004331483223376398": 0.118, "0.04905583706365046": 0.6656, "0.00017548671496481503": 0.1028, "0.007208715033782135": 0.294, "0.0005711586478126435": 0.1044, "0.0005404216420705915": 0.1037, "0.00023570694139967278": 0.1028, "0.00019783188827841644": 0.1028, "0.00015282140360258693": 0.1028, "0.2776153294436801": 0.9521, "0.03300034791125285": 0.5234, "0.00286719649749377": 0.1693, "0.697981390783066": 0.9563, "0.0002129748535745521": 0.1028, "0.01052015217616159": 0.3404, "0.019332422875550453": 0.432, "0.00034716868189265597": 0.1075, "0.003995780301895269": 0.202, "0.0051726573872160194": 0.2336, "0.0013587299019027091": 0.1208, "0.010715933998226712": 0.3499, "0.08688382635251184": 0.8547, "0.0014097728716289677": 0.1251, "0.006820776732865685": 0.276, "0.01987459549580984": 0.4277, "0.00023789010410788934": 0.1115, "0.025258200269627846": 0.4648, "0.0013095350204826678": 0.1211, "0.4242556430717777": 0.9532, "0.00045778405383766163": 0.1242, "0.00014459729217920195": 0.1029, "0.030372635797033115": 0.4934, "0.027954159990678566": 0.4783, "0.6604193962330305": 0.956, "0.0318055201533292": 0.5006, "0.020244465099768037": 0.4265, "0.005994842503189409": 0.2435, "0.0009310413487069075": 0.1137, "0.00019966424501097934": 0.1028, "0.005672228971644543": 0.2376, "0.03392583382740992": 0.5244, "0.0065134909462728026": 0.2633, "0.1006938631476027": 0.888, "0.0003913745601980384": 0.1036, "0.46522995239601894": 0.9484, "0.07636298261282241": 0.808, "0.0013968351179887398": 0.12, "0.0019287915080207778": 0.1399, "0.7653919388230148": 0.9518, "0.6852291595284065": 0.9562, "0.00015566543592710622": 0.1028, "0.000520854855057766": 0.1036, "0.030937875717301368": 0.5074, "0.00035038422452906756": 0.113, "0.01111849604819271": 0.3578, "0.061771875973384946": 0.761, "0.0053177231778509665": 0.2101, "0.004341478330055093": 0.232, "0.009682466119303124": 0.3318, "0.37288213071828336": 0.9563, "0.24853948574297985": 0.9523, "0.025492146544514233": 0.4547, "0.004894289896114532": 0.2013, "0.07427982482564918": 0.8198, "0.0002071649675602069": 0.1028, "0.016835508029612024": 0.4182, "0.027443433032283655": 0.4781, "0.030093900344497212": 0.48, "0.004184288507901585": 0.2398, "0.019511483468466165": 0.4253, "0.001589282865622978": 0.1225, "0.0003224842498408442": 0.1037, "0.003058338032378432": 0.1438, "0.05429186177618943": 0.7199, "0.010815187025522881": 0.3382, "0.00031080821738690636": 0.1029, "0.0004663034929742731": 0.1044, "0.014526539259467812": 0.4094, "0.001648986944471065": 0.1298, "0.09182542835656282": 0.8481, "0.006634708121092351": 0.2621, "0.000128264983052806": 0.1304, "0.00020526377527092522": 0.1028, "0.0006619433458774394": 0.1063, "0.3495775574363275": 0.9571, "0.02367960067833076": 0.4499, "0.018805040551285814": 0.427, "0.00160400310705682": 0.1245, "0.04193943955667186": 0.5811, "0.5858248200152536": 0.9583, "0.042719939663067766": 0.5895, "0.034240061379714255": 0.528, "0.018124175473742378": 0.4253, "0.00010965792912678099": 0.1701, "0.10840143591783309": 0.9046, "0.001726780903884356": 0.1277, "0.00025374903797335715": 0.1029, "0.0020197857568198785": 0.1432, "0.0671161176749628": 0.7914, "0.00017711210643450888": 0.1028, "0.014796880626863964": 0.4139, "0.013618652367560814": 0.3899, "0.09014776314524918": 0.8274, "0.022199661191199548": 0.4368, "0.312964801067075": 0.9591, "0.77248114514034": 0.9539, "0.00012362095437367691": 0.1177, "0.8870496889654403": 0.9421, "0.0021945290862033138": 0.1451, "0.00047062248498412813": 0.1092, "0.00018546769230846994": 0.1028, "0.0020573743134329114": 0.1379, "0.0010304169949505876": 0.1135, "0.080706201411495": 0.8393, "0.2827817979625341": 0.9564, "0.40514231711146476": 0.9538, "0.002256074066496859": 0.1441, "0.000774263682681127": 0.109, "0.007142559285543126": 0.2873, "0.03720236681413066": 0.5454, "0.0005764488282925873": 0.105, "0.0010790287915161836": 0.1149, "0.28804441533962977": 0.9555, "0.0011195643194838783": 0.116, "0.000732596542821523": 0.1073, "0.08296958520834907": 0.8493, "0.016376240745216875": 0.4061, "0.018461469463245474": 0.4293, "0.03239742629528195": 0.5139, "0.39774030240580366": 0.9562, "0.0011092898648952228": 0.1173, "0.000488302208687788": 0.1058, "0.0004252346334528682": 0.1031, "0.004673795107992464": 0.2388, "0.3247218492073129": 0.9564, "0.0005354620899273607": 0.1059, "0.00024911300260677886": 0.1028, "0.4871780218794631": 0.9513, "0.0017589165903277326": 0.1305, "0.42818517986524113": 0.9559, "0.36946012051993027": 0.9571, "0.01101645949633657": 0.3557, "0.38689007393279756": 0.9547, "0.1291549665014884": 0.9216, "0.00038778284145894574": 0.1096, "0.5342293299538352": 0.9583, "0.005031548945038057": 0.2117, "0.006696160054853215": 0.274, "0.00024009348768606518": 0.1033, "0.0007393819919175866": 0.1079, "0.00048382096649259574": 0.113, "0.0658898955079995": 0.7923, "0.1303512244681509": 0.9245, "0.00018207916800994624": 0.1028, "0.0005659170163246243": 0.1048, "0.794145171902934": 0.9556, "0.20479120966650854": 0.9504, "0.5804485942768978": 0.9579, "0.0006995920165435374": 0.1089, "0.7868571506936851": 0.9516, "0.8549327066268376": 0.9381, "0.0007814350607844542": 0.1092, "0.00031952475057592136": 0.1222, "0.0005817880074344935": 0.1045, "0.004463233926710397": 0.2404, "0.00675818116816111": 0.2924, "0.0070770106611818895": 0.2799, "0.00012024861420374123": 0.167, "0.00019243509752303302": 0.1028, "0.0001486524844997857": 0.1029, "0.4203621683844714": 0.958, "0.06292146109610344": 0.7715, "0.016681005372000592": 0.4223, "0.055302242561929005": 0.711, "0.00024682684522556923": 0.1029, "0.0005066461008921268": 0.1083, "0.03361449000108765": 0.5168, "0.014261137071941283": 0.4081, "0.003851107002325569": 0.1781, "0.009593608287093146": 0.329, "0.00011589483034398106": 0.1743, "0.11562801312073753": 0.9112, "0.00017875255259042355": 0.1028, "0.13400688963639507": 0.9258, "0.0777841107128649": 0.821, "0.1315585624045704": 0.9272, "0.243998629725955": 0.9546, "0.0361874981241128": 0.5443, "0.22874908173557018": 0.955, "0.7175560918936928": 0.9567, "0.05329994080844088": 0.7123, "0.000501996513311008": 0.1047, "0.6191441755977841": 0.9571, "0.016225952870780873": 0.4201, "0.00036354699612933176": 0.1044, "0.001321664183946605": 0.118, "0.0008107909806731687": 0.1103, "0.0017427746784089192": 0.1312, "0.005777790117970507": 0.2485, "0.0003254711605531848": 0.1042, "0.9727203192450538": 0.9485, "0.3763358362286533": 0.9557, "0.25316484786313553": 0.9539, "0.0034477640547344642": 0.1471, "0.0011509622008850323": 0.117, "0.0006148777653810023": 0.1055, "0.30724688427090036": 0.9575, "0.0006868103588995307": 0.1066, "0.18334254825622906": 0.9444, "0.0042230441872066725": 0.242, "0.07159041085964889": 0.8078, "0.00011377741332214914": 0.1628, "0.000356904934567523": 0.103, "0.06963744730628223": 0.7954, "0.0012052609368708425": 0.1187, "0.0010209606623060466": 0.1169, "0.011221477682079803": 0.348, "0.0001906690840512252": 0.1028, "0.023462288481422625": 0.4488, "0.10162650893929952": 0.8913, "0.001422830457214352": 0.1219, "0.8952657125996392": 0.9498, "0.0016338538778098604": 0.1226, "0.03487727474814178": 0.5286, "0.0004793808495089107": 0.1138, "0.0012738113231864786": 0.1224, "0.06836516004510239": 0.7966, "0.0002608653617622548": 0.1052, "0.007275483529196233": 0.3058, "0.11997177354358843": 0.9154, "0.4695390010680058": 0.9513, "0.006335804992658254": 0.2683, "0.02412028207618007": 0.4561, "0.0004098383671757261": 0.1039, "0.3369205705980267": 0.955, "0.4916903577628026": 0.9541, "0.07292272058728312": 0.8153, "0.5054796821191241": 0.954, "0.04473533054498463": 0.6135, "0.003746050032748993": 0.1707, "0.009246257116405733": 0.3301, "0.0029749075472144406": 0.1393, "0.037894709190746674": 0.5594, "0.32174181506763716": 0.9578, "0.34319071974590426": 0.9576, "0.0007601177617955331": 0.1104, "0.00011696827039703847": 0.1614, "0.009772146969725724": 0.3414, "0.16874356777273758": 0.9415, "0.14968392930772556": 0.9355, "0.01023316578330245": 0.3288, "0.4321511127789762": 0.9536, "0.00916140245713852": 0.331, "0.004717084690917017": 0.2264, "0.8393129498166364": 0.9469, "0.08932045998580969": 0.8538, "0.0004136343684063274": 0.1157, "0.5492116483887789": 0.957, "0.0007126115430111746": 0.1068, "0.04155455334718875": 0.5808, "0.018979216428391014": 0.4257, "0.3338285864731761": 0.9553, "0.0007258733650817253": 0.1076, "0.11777987011971193": 0.9132, "0.11669898186171475": 0.9087, "0.04391800892596086": 0.6173, "0.024796728925021577": 0.4611, "0.0006931717276155408": 0.1074, "0.00044942026621191424": 0.118, "0.11456687286348714": 0.4324, "0.07566218500481055": 0.8172, "0.24625859163505467": 0.9553, "0.0295440799888038": 0.4925, "0.03968246104569482": 0.581, "0.08850074914473438": 0.862, "0.18848434090337954": 0.946, "0.001": 0.1126, "0.05": 0.6563, "0.01": 0.3454, "0.1": 0.8785, "0.2": 0.9496, "0.3": 0.9536, "0.4": 0.9574, "0.5": 0.9553, "0.6": 0.9574, "0.7": 0.9576, "0.8": 0.9544, "0.9": 0.9493, "1.0": 0.9455, "0.07225349491787214": 0.7959, "0.003711671819475767": 0.1878, "0.025966559729348696": 0.4663, "0.4782772017727485": 0.955, "0.0014493095741262166": 0.1203, "0.003511927530450728": 0.1688, "0.00020151357338155586": 0.1028, "0.014000583824680975": 0.395, "0.11144152514667881": 0.9111, "0.0004973895958790063": 0.1033, "0.0001056875971184804": 0.1367, "0.05479472336900287": 0.6986, "0.5441714286865893": 0.9542, "0.00035362955013550426": 0.1084, "0.04117319931161679": 0.5918, "0.0012856096069432965": 0.1172, "0.007689283720758305": 0.2984, "0.001346260579298911": 0.1201, "0.010046204213468132": 0.3331, "0.0007959777002314986": 0.1098, "0.008668379930019773": 0.3155, "0.00039865810735804387": 0.1033, "0.004630902801799737": 0.2286, "0.00026818126094530154": 0.105, "0.004107840889965647": 0.2353, "0.000360210656235707": 0.113, "0.020621218039991424": 0.4358, "0.024343688735431104": 0.462, "0.0002834343306151309": 0.1028, "0.27004207188377727": 0.9539, "0.002814812360507581": 0.1588, "0.00047498148032285": 0.1056, "0.0033847728559459832": 0.1442, "0.008277856966198472": 0.3123, "0.0004174655289253135": 0.103, "0.05232614239486662": 0.6815, "0.0001675807864530767": 0.1028, "0.06773775997517752": 0.7845, "0.00012136237983442405": 0.135, "0.9288978720164498": 0.9481, "0.0013840160965731315": 0.1188, "0.23734242500238661": 0.9541, "0.061204983724766966": 0.7598, "0.048159579101923505": 0.6532, "0.00013065201621247213": 0.1043, "0.09353431520292387": 0.8679, "0.14831025143361043": 0.9351, "0.03585539857459817": 0.5372, "0.023033628731421313": 0.4493, "0.6789406812696106": 0.9557, "0.0015037553212997384": 0.1239, "0.0008258799387844272": 0.1106, "0.0001191450698119776": 0.1739, "0.000818300681586739": 0.1098, "0.003030271082866396": 0.165, "0.005568596444286409": 0.2361, "0.004145888496832911": 0.2253, "0.0014360089846512606": 0.1213, "0.23300614106969247": 0.9565, "0.28540097698292377": 0.9575, "0.0028408836901833013": 0.158, "0.015638467583022463": 0.4127, "0.0008412497049736118": 0.1101, "0.00016451905877536624": 0.1028, "0.0022148552337263594": 0.1389, "0.0002271728133026905": 0.1028, "0.0050781521123276704": 0.2097, "0.14426439512181574": 0.9325, "0.0952750047242729": 0.882, "0.0018935521797562953": 0.1315, "0.00021693835183851843": 0.1033, "0.004849374067335233": 0.2103, "0.0004535828825510186": 0.1048, "0.007012063589007177": 0.2726, "0.002663332725174981": 0.1673, "0.00023354381399064793": 0.1028, "0.3528154115380883": 0.9573, "0.14162866162991988": 0.9319, "0.00018718552949655793": 0.1028, "0.0005256791122018419": 0.1038, "0.0017109339072690134": 0.1281, "0.00858882855954625": 0.3218, "0.6022541201461928": 0.9559, "0.16566059589499135": 0.9414, "0.4738879609717651": 0.9569, "0.0004928249570040513": 0.1054, "0.07093341204987996": 0.8002, "0.00011805165285688056": 0.1544, "0.002362508465477945": 0.1448, "0.0001259215613694151": 0.1287, "0.000441209286319119": 0.1269, "0.000939664831495469": 0.1113, "0.050431594871713586": 0.6867, "0.02927294835042816": 0.495, "0.04996877453854884": 0.6722, "0.0006742622241778342": 0.108, "0.23516428844943485": 0.9524, "0.0393182875570577": 0.5622, "0.0019466563433422632": 0.1359, "0.09615746001432096": 0.8865, "0.0035772850993678732": 0.1498, "0.007832382599179196": 0.2919, "0.046415888336127774": 0.6377, "0.22250887981283693": 0.9524, "0.014661086840469845": 0.4048, "0.0017916503273638995": 0.1316, "0.03122442823092858": 0.5061, "0.008201889499202203": 0.3128, "0.00012248646137509307": 0.1259, "0.011859710123376693": 0.3605, "0.20860240892485027": 0.9526, "0.0023408272761782944": 0.1405, "0.002154434690031882": 0.1423, "0.015211855179861048": 0.4101, "0.5293266058360562": 0.954, "0.00021102034285685965": 0.1028, "0.002038493398252464": 0.1443, "0.4483855948021186": 0.9506, "0.3277294849923382": 0.9582, "0.9817298406188841": 0.9513, "0.17999285067824763": 0.9415, "0.018632463119315597": 0.4209, "0.5594325706169377": 0.9557, "0.16263395040481923": 0.9348, "0.023898925662310503": 0.455, "0.0006263207452198692": 0.1063, "0.007978144572076629": 0.3116, "0.015352750287804227": 0.4142, "0.08529644499741025": 0.8363, "0.022822244741868985": 0.4462, "0.9035578346138929": 0.9515, "0.00790492762269642": 0.3067, "0.04727969591600391": 0.6351, "0.5698437059469142": 0.9574, "0.36606951475969024": 0.9575, "0.06350425168595962": 0.7787, "0.011536181017364784": 0.3575, "0.34004119327037063": 0.9576, "0.00030512970171828706": 0.1255, "0.17347593592339308": 0.9441, "0.008354528058382871": 0.3041, "0.0018249932448161524": 0.1337, "0.0039227767589277195": 0.1883, "0.7242022334607315": 0.9536, "0.0006805073696735206": 0.107, "0.021995930680300747": 0.4427, "0.12447871461879062": 0.92, "0.12220446866314887": 0.9068, "0.0005871766390733255": 0.1083, "0.21248453524988828": 0.952, "0.03754694224073337": 0.5471, "0.0036104185971733375": 0.1523} diff --git a/research/hyperparameters_2022/rdp_accountant.py b/research/hyperparameters_2022/rdp_accountant.py new file mode 100644 index 0000000..9de1ace --- /dev/null +++ b/research/hyperparameters_2022/rdp_accountant.py @@ -0,0 +1,622 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""RDP analysis of the Sampled Gaussian Mechanism. + +Functionality for computing Renyi differential privacy (RDP) of an additive +Sampled Gaussian Mechanism (SGM). Its public interface consists of two methods: + compute_rdp(q, noise_multiplier, T, orders) computes RDP for SGM iterated + T times. + get_privacy_spent(orders, rdp, target_eps, target_delta) computes delta + (or eps) given RDP at multiple orders and + a target value for eps (or delta). + +Example use: + +Suppose that we have run an SGM applied to a function with l2-sensitivity 1. +Its parameters are given as a list of tuples (q1, sigma1, T1), ..., +(qk, sigma_k, Tk), and we wish to compute eps for a given delta. +The example code would be: + + max_order = 32 + orders = range(2, max_order + 1) + rdp = np.zeros_like(orders, dtype=float) + for q, sigma, T in parameters: + rdp += rdp_accountant.compute_rdp(q, sigma, T, orders) + eps, _, opt_order = rdp_accountant.get_privacy_spent(rdp, target_delta=delta) +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +import sys + +import numpy as np +from scipy import special +import six + +######################## +# LOG-SPACE ARITHMETIC # +######################## + + +def _log_add(logx, logy): + """Add two numbers in the log space.""" + a, b = min(logx, logy), max(logx, logy) + if a == -np.inf: # adding 0 + return b + # Use exp(a) + exp(b) = (exp(a - b) + 1) * exp(b) + return math.log1p(math.exp(a - b)) + b # log1p(x) = log(x + 1) + + +def _log_sub(logx, logy): + """Subtract two numbers in the log space. Answer must be non-negative.""" + if logx < logy: + raise ValueError("The result of subtraction must be non-negative.") + if logy == -np.inf: # subtracting 0 + return logx + if logx == logy: + return -np.inf # 0 is represented as -np.inf in the log space. + + try: + # Use exp(x) - exp(y) = (exp(x - y) - 1) * exp(y). + return math.log(math.expm1(logx - logy)) + logy # expm1(x) = exp(x) - 1 + except OverflowError: + return logx + + +def _log_sub_sign(logx, logy): + """Returns log(exp(logx)-exp(logy)) and its sign.""" + if logx > logy: + s = True + mag = logx + np.log(1 - np.exp(logy - logx)) + elif logx < logy: + s = False + mag = logy + np.log(1 - np.exp(logx - logy)) + else: + s = True + mag = -np.inf + + return s, mag + + +def _log_print(logx): + """Pretty print.""" + if logx < math.log(sys.float_info.max): + return "{}".format(math.exp(logx)) + else: + return "exp({})".format(logx) + + +def _log_comb(n, k): + return (special.gammaln(n + 1) - special.gammaln(k + 1) - + special.gammaln(n - k + 1)) + + +def _compute_log_a_int(q, sigma, alpha): + """Compute log(A_alpha) for integer alpha. 0 < q < 1.""" + assert isinstance(alpha, six.integer_types) + + # Initialize with 0 in the log space. + log_a = -np.inf + + for i in range(alpha + 1): + log_coef_i = ( + _log_comb(alpha, i) + i * math.log(q) + (alpha - i) * math.log(1 - q)) + + s = log_coef_i + (i * i - i) / (2 * (sigma**2)) + log_a = _log_add(log_a, s) + + return float(log_a) + + +def _compute_log_a_frac(q, sigma, alpha): + """Compute log(A_alpha) for fractional alpha. 0 < q < 1.""" + # The two parts of A_alpha, integrals over (-inf,z0] and [z0, +inf), are + # initialized to 0 in the log space: + log_a0, log_a1 = -np.inf, -np.inf + i = 0 + + z0 = sigma**2 * math.log(1 / q - 1) + .5 + + while True: # do ... until loop + coef = special.binom(alpha, i) + log_coef = math.log(abs(coef)) + j = alpha - i + + log_t0 = log_coef + i * math.log(q) + j * math.log(1 - q) + log_t1 = log_coef + j * math.log(q) + i * math.log(1 - q) + + log_e0 = math.log(.5) + _log_erfc((i - z0) / (math.sqrt(2) * sigma)) + log_e1 = math.log(.5) + _log_erfc((z0 - j) / (math.sqrt(2) * sigma)) + + log_s0 = log_t0 + (i * i - i) / (2 * (sigma**2)) + log_e0 + log_s1 = log_t1 + (j * j - j) / (2 * (sigma**2)) + log_e1 + + if coef > 0: + log_a0 = _log_add(log_a0, log_s0) + log_a1 = _log_add(log_a1, log_s1) + else: + log_a0 = _log_sub(log_a0, log_s0) + log_a1 = _log_sub(log_a1, log_s1) + + i += 1 + if max(log_s0, log_s1) < -30: + break + + return _log_add(log_a0, log_a1) + + +def _compute_log_a(q, sigma, alpha): + """Compute log(A_alpha) for any positive finite alpha.""" + if float(alpha).is_integer(): + return _compute_log_a_int(q, sigma, int(alpha)) + else: + return _compute_log_a_frac(q, sigma, alpha) + + +def _log_erfc(x): + """Compute log(erfc(x)) with high accuracy for large x.""" + try: + return math.log(2) + special.log_ndtr(-x * 2**.5) + except NameError: + # If log_ndtr is not available, approximate as follows: + r = special.erfc(x) + if r == 0.0: + # Using the Laurent series at infinity for the tail of the erfc function: + # erfc(x) ~ exp(-x^2-.5/x^2+.625/x^4)/(x*pi^.5) + # To verify in Mathematica: + # Series[Log[Erfc[x]] + Log[x] + Log[Pi]/2 + x^2, {x, Infinity, 6}] + return (-math.log(math.pi) / 2 - math.log(x) - x**2 - .5 * x**-2 + + .625 * x**-4 - 37. / 24. * x**-6 + 353. / 64. * x**-8) + else: + return math.log(r) + + +def _compute_delta(orders, rdp, eps): + """Compute delta given a list of RDP values and target epsilon. + + Args: + orders: An array (or a scalar) of orders. + rdp: A list (or a scalar) of RDP guarantees. + eps: The target epsilon. + + Returns: + Pair of (delta, optimal_order). + + Raises: + ValueError: If input is malformed. + + """ + orders_vec = np.atleast_1d(orders) + rdp_vec = np.atleast_1d(rdp) + + if eps < 0: + raise ValueError("Value of privacy loss bound epsilon must be >=0.") + if len(orders_vec) != len(rdp_vec): + raise ValueError("Input lists must have the same length.") + + # Basic bound (see https://arxiv.org/abs/1702.07476 Proposition 3 in v3): + # delta = min( np.exp((rdp_vec - eps) * (orders_vec - 1)) ) + + # Improved bound from https://arxiv.org/abs/2004.00010 Proposition 12 (in v4): + logdeltas = [] # work in log space to avoid overflows + for (a, r) in zip(orders_vec, rdp_vec): + if a < 1: + raise ValueError("Renyi divergence order must be >=1.") + if r < 0: + raise ValueError("Renyi divergence must be >=0.") + # For small alpha, we are better of with bound via KL divergence: + # delta <= sqrt(1-exp(-KL)). + # Take a min of the two bounds. + logdelta = 0.5 * math.log1p(-math.exp(-r)) + if a > 1.01: + # This bound is not numerically stable as alpha->1. + # Thus we have a min value for alpha. + # The bound is also not useful for small alpha, so doesn't matter. + rdp_bound = (a - 1) * (r - eps + math.log1p(-1 / a)) - math.log(a) + logdelta = min(logdelta, rdp_bound) + + logdeltas.append(logdelta) + + idx_opt = np.argmin(logdeltas) + return min(math.exp(logdeltas[idx_opt]), 1.), orders_vec[idx_opt] + + +def _compute_eps(orders, rdp, delta): + """Compute epsilon given a list of RDP values and target delta. + + Args: + orders: An array (or a scalar) of orders. + rdp: A list (or a scalar) of RDP guarantees. + delta: The target delta. + + Returns: + Pair of (eps, optimal_order). + + Raises: + ValueError: If input is malformed. + + """ + orders_vec = np.atleast_1d(orders) + rdp_vec = np.atleast_1d(rdp) + + if delta <= 0: + raise ValueError("Privacy failure probability bound delta must be >0.") + if len(orders_vec) != len(rdp_vec): + raise ValueError("Input lists must have the same length.") + + # Basic bound (see https://arxiv.org/abs/1702.07476 Proposition 3 in v3): + # eps = min( rdp_vec - math.log(delta) / (orders_vec - 1) ) + + # Improved bound from https://arxiv.org/abs/2004.00010 Proposition 12 (in v4). + # Also appears in https://arxiv.org/abs/2001.05990 Equation 20 (in v1). + eps_vec = [] + for (a, r) in zip(orders_vec, rdp_vec): + if a < 1: + raise ValueError("Renyi divergence order must be >=1.") + if r < 0: + raise ValueError("Renyi divergence must be >=0.") + + if delta**2 + math.expm1(-r) >= 0: + # In this case, we can simply bound via KL divergence: + # delta <= sqrt(1-exp(-KL)). + eps = 0 # No need to try further computation if we have eps = 0. + elif a > 1.01: + # This bound is not numerically stable as alpha->1. + # Thus we have a min value of alpha. + # The bound is also not useful for small alpha, so doesn't matter. + eps = r + math.log1p(-1 / a) - math.log(delta * a) / (a - 1) + else: + # In this case we can't do anything. E.g., asking for delta = 0. + eps = np.inf + eps_vec.append(eps) + + idx_opt = np.argmin(eps_vec) + return max(0, eps_vec[idx_opt]), orders_vec[idx_opt] + + +def _stable_inplace_diff_in_log(vec, signs, n=-1): + """Replaces the first n-1 dims of vec with the log of abs difference operator. + + Args: + vec: numpy array of floats with size larger than 'n' + signs: Optional numpy array of bools with the same size as vec in case one + needs to compute partial differences vec and signs jointly describe a + vector of real numbers' sign and abs in log scale. + n: Optonal upper bound on number of differences to compute. If negative, all + differences are computed. + + Returns: + The first n-1 dimension of vec and signs will store the log-abs and sign of + the difference. + + Raises: + ValueError: If input is malformed. + """ + + assert vec.shape == signs.shape + if n < 0: + n = np.max(vec.shape) - 1 + else: + assert np.max(vec.shape) >= n + 1 + for j in range(0, n, 1): + if signs[j] == signs[j + 1]: # When the signs are the same + # if the signs are both positive, then we can just use the standard one + signs[j], vec[j] = _log_sub_sign(vec[j + 1], vec[j]) + # otherwise, we do that but toggle the sign + if not signs[j + 1]: + signs[j] = ~signs[j] + else: # When the signs are different. + vec[j] = _log_add(vec[j], vec[j + 1]) + signs[j] = signs[j + 1] + + +def _get_forward_diffs(fun, n): + """Computes up to nth order forward difference evaluated at 0. + + See Theorem 27 of https://arxiv.org/pdf/1808.00087.pdf + + Args: + fun: Function to compute forward differences of. + n: Number of differences to compute. + + Returns: + Pair (deltas, signs_deltas) of the log deltas and their signs. + """ + func_vec = np.zeros(n + 3) + signs_func_vec = np.ones(n + 3, dtype=bool) + + # ith coordinate of deltas stores log(abs(ith order discrete derivative)) + deltas = np.zeros(n + 2) + signs_deltas = np.zeros(n + 2, dtype=bool) + for i in range(1, n + 3, 1): + func_vec[i] = fun(1.0 * (i - 1)) + for i in range(0, n + 2, 1): + # Diff in log scale + _stable_inplace_diff_in_log(func_vec, signs_func_vec, n=n + 2 - i) + deltas[i] = func_vec[0] + signs_deltas[i] = signs_func_vec[0] + return deltas, signs_deltas + + +def _compute_rdp(q, sigma, alpha): + """Compute RDP of the Sampled Gaussian mechanism at order alpha. + + Args: + q: The sampling rate. + sigma: The std of the additive Gaussian noise. + alpha: The order at which RDP is computed. + + Returns: + RDP at alpha, can be np.inf. + """ + if q == 0: + return 0 + + if q == 1.: + return alpha / (2 * sigma**2) + + if np.isinf(alpha): + return np.inf + + return _compute_log_a(q, sigma, alpha) / (alpha - 1) + + +def compute_rdp(q, noise_multiplier, steps, orders): + """Computes RDP of the Sampled Gaussian Mechanism. + + Args: + q: The sampling rate. + noise_multiplier: The ratio of the standard deviation of the Gaussian noise + to the l2-sensitivity of the function to which it is added. + steps: The number of steps. + orders: An array (or a scalar) of RDP orders. + + Returns: + The RDPs at all orders. Can be `np.inf`. + """ + if np.isscalar(orders): + rdp = _compute_rdp(q, noise_multiplier, orders) + else: + rdp = np.array( + [_compute_rdp(q, noise_multiplier, order) for order in orders]) + + return rdp * steps + + +def compute_rdp_sample_without_replacement(q, noise_multiplier, steps, orders): + """Compute RDP of Gaussian Mechanism using sampling without replacement. + + This function applies to the following schemes: + 1. Sampling w/o replacement: Sample a uniformly random subset of size m = q*n. + 2. ``Replace one data point'' version of differential privacy, i.e., n is + considered public information. + + Reference: Theorem 27 of https://arxiv.org/pdf/1808.00087.pdf (A strengthened + version applies subsampled-Gaussian mechanism) + - Wang, Balle, Kasiviswanathan. "Subsampled Renyi Differential Privacy and + Analytical Moments Accountant." AISTATS'2019. + + Args: + q: The sampling proportion = m / n. Assume m is an integer <= n. + noise_multiplier: The ratio of the standard deviation of the Gaussian noise + to the l2-sensitivity of the function to which it is added. + steps: The number of steps. + orders: An array (or a scalar) of RDP orders. + + Returns: + The RDPs at all orders, can be np.inf. + """ + if np.isscalar(orders): + rdp = _compute_rdp_sample_without_replacement_scalar( + q, noise_multiplier, orders) + else: + rdp = np.array([ + _compute_rdp_sample_without_replacement_scalar(q, noise_multiplier, + order) + for order in orders + ]) + + return rdp * steps + + +def _compute_rdp_sample_without_replacement_scalar(q, sigma, alpha): + """Compute RDP of the Sampled Gaussian mechanism at order alpha. + + Args: + q: The sampling proportion = m / n. Assume m is an integer <= n. + sigma: The std of the additive Gaussian noise. + alpha: The order at which RDP is computed. + + Returns: + RDP at alpha, can be np.inf. + """ + + assert (q <= 1) and (q >= 0) and (alpha >= 1) + + if q == 0: + return 0 + + if q == 1.: + return alpha / (2 * sigma**2) + + if np.isinf(alpha): + return np.inf + + if float(alpha).is_integer(): + return _compute_rdp_sample_without_replacement_int(q, sigma, alpha) / ( + alpha - 1) + else: + # When alpha not an integer, we apply Corollary 10 of [WBK19] to interpolate + # the CGF and obtain an upper bound + alpha_f = math.floor(alpha) + alpha_c = math.ceil(alpha) + + x = _compute_rdp_sample_without_replacement_int(q, sigma, alpha_f) + y = _compute_rdp_sample_without_replacement_int(q, sigma, alpha_c) + t = alpha - alpha_f + return ((1 - t) * x + t * y) / (alpha - 1) + + +def _compute_rdp_sample_without_replacement_int(q, sigma, alpha): + """Compute log(A_alpha) for integer alpha, subsampling without replacement. + + When alpha is smaller than max_alpha, compute the bound Theorem 27 exactly, + otherwise compute the bound with Stirling approximation. + + Args: + q: The sampling proportion = m / n. Assume m is an integer <= n. + sigma: The std of the additive Gaussian noise. + alpha: The order at which RDP is computed. + + Returns: + RDP at alpha, can be np.inf. + """ + + max_alpha = 256 + assert isinstance(alpha, six.integer_types) + + if np.isinf(alpha): + return np.inf + elif alpha == 1: + return 0 + + def cgf(x): + # Return rdp(x+1)*x, the rdp of Gaussian mechanism is alpha/(2*sigma**2) + return x * 1.0 * (x + 1) / (2.0 * sigma**2) + + def func(x): + # Return the rdp of Gaussian mechanism + return 1.0 * x / (2.0 * sigma**2) + + # Initialize with 1 in the log space. + log_a = 0 + # Calculates the log term when alpha = 2 + log_f2m1 = func(2.0) + np.log(1 - np.exp(-func(2.0))) + if alpha <= max_alpha: + # We need forward differences of exp(cgf) + # The following line is the numerically stable way of implementing it. + # The output is in polar form with logarithmic magnitude + deltas, _ = _get_forward_diffs(cgf, alpha) + # Compute the bound exactly requires book keeping of O(alpha**2) + + for i in range(2, alpha + 1): + if i == 2: + s = 2 * np.log(q) + _log_comb(alpha, 2) + np.minimum( + np.log(4) + log_f2m1, + func(2.0) + np.log(2)) + elif i > 2: + delta_lo = deltas[int(2 * np.floor(i / 2.0)) - 1] + delta_hi = deltas[int(2 * np.ceil(i / 2.0)) - 1] + s = np.log(4) + 0.5 * (delta_lo + delta_hi) + s = np.minimum(s, np.log(2) + cgf(i - 1)) + s += i * np.log(q) + _log_comb(alpha, i) + log_a = _log_add(log_a, s) + return float(log_a) + else: + # Compute the bound with stirling approximation. Everything is O(x) now. + for i in range(2, alpha + 1): + if i == 2: + s = 2 * np.log(q) + _log_comb(alpha, 2) + np.minimum( + np.log(4) + log_f2m1, + func(2.0) + np.log(2)) + else: + s = np.log(2) + cgf(i - 1) + i * np.log(q) + _log_comb(alpha, i) + log_a = _log_add(log_a, s) + + return log_a + + +def compute_heterogenous_rdp(sampling_probabilities, noise_multipliers, + steps_list, orders): + """Computes RDP of Heteregoneous Applications of Sampled Gaussian Mechanisms. + + Args: + sampling_probabilities: A list containing the sampling rates. + noise_multipliers: A list containing the noise multipliers: the ratio of the + standard deviation of the Gaussian noise to the l2-sensitivity of the + function to which it is added. + steps_list: A list containing the number of steps at each + `sampling_probability` and `noise_multiplier`. + orders: An array (or a scalar) of RDP orders. + + Returns: + The RDPs at all orders. Can be `np.inf`. + """ + assert len(sampling_probabilities) == len(noise_multipliers) + + rdp = 0 + for q, noise_multiplier, steps in zip(sampling_probabilities, + noise_multipliers, steps_list): + rdp += compute_rdp(q, noise_multiplier, steps, orders) + + return rdp + + +def get_privacy_spent(orders, rdp, target_eps=None, target_delta=None): + """Computes delta (or eps) for given eps (or delta) from RDP values. + + Args: + orders: An array (or a scalar) of RDP orders. + rdp: An array of RDP values. Must be of the same length as the orders list. + target_eps: If not `None`, the epsilon for which we compute the + corresponding delta. + target_delta: If not `None`, the delta for which we compute the + corresponding epsilon. Exactly one of `target_eps` and `target_delta` must + be `None`. + + Returns: + A tuple of epsilon, delta, and the optimal order. + + Raises: + ValueError: If target_eps and target_delta are messed up. + """ + if target_eps is None and target_delta is None: + raise ValueError( + "Exactly one out of eps and delta must be None. (Both are).") + + if target_eps is not None and target_delta is not None: + raise ValueError( + "Exactly one out of eps and delta must be None. (None is).") + + if target_eps is not None: + delta, opt_order = _compute_delta(orders, rdp, target_eps) + return target_eps, delta, opt_order + else: + eps, opt_order = _compute_eps(orders, rdp, target_delta) + return eps, target_delta, opt_order + + +def compute_rdp_from_ledger(ledger, orders): + """Computes RDP of Sampled Gaussian Mechanism from ledger. + + Args: + ledger: A formatted privacy ledger. + orders: An array (or a scalar) of RDP orders. + + Returns: + RDP at all orders. Can be `np.inf`. + """ + total_rdp = np.zeros_like(orders, dtype=float) + for sample in ledger: + # Compute equivalent z from l2_clip_bounds and noise stddevs in sample. + # See https://arxiv.org/pdf/1812.06210.pdf for derivation of this formula. + effective_z = sum([ + (q.noise_stddev / q.l2_norm_bound)**-2 for q in sample.queries + ])**-0.5 + total_rdp += compute_rdp(sample.selection_probability, effective_z, 1, + orders) + return total_rdp diff --git a/tensorflow_privacy/privacy/analysis/BUILD b/tensorflow_privacy/privacy/analysis/BUILD index be92179..68a7fe2 100644 --- a/tensorflow_privacy/privacy/analysis/BUILD +++ b/tensorflow_privacy/privacy/analysis/BUILD @@ -45,7 +45,10 @@ py_binary( py_library( name = "compute_noise_from_budget_lib", srcs = ["compute_noise_from_budget_lib.py"], - deps = [":rdp_accountant"], + deps = [ + "@com_google_differential_py//python/dp_accounting:dp_event", + "@com_google_differential_py//python/dp_accounting/rdp:rdp_privacy_accountant", + ], ) py_test( diff --git a/tensorflow_privacy/privacy/analysis/compute_noise_from_budget_lib.py b/tensorflow_privacy/privacy/analysis/compute_noise_from_budget_lib.py index f20d235..18214af 100644 --- a/tensorflow_privacy/privacy/analysis/compute_noise_from_budget_lib.py +++ b/tensorflow_privacy/privacy/analysis/compute_noise_from_budget_lib.py @@ -19,22 +19,18 @@ import math from absl import app from scipy import optimize -from tensorflow_privacy.privacy.analysis.rdp_accountant import compute_rdp # pylint: disable=g-import-not-at-top -from tensorflow_privacy.privacy.analysis.rdp_accountant import get_privacy_spent +from com_google_differential_py.python.dp_accounting import dp_event +from com_google_differential_py.python.dp_accounting.rdp import rdp_privacy_accountant def apply_dp_sgd_analysis(q, sigma, steps, orders, delta): """Compute and print results of DP-SGD analysis.""" - # compute_rdp requires that sigma be the ratio of the standard deviation of - # the Gaussian noise to the l2-sensitivity of the function to which it is - # added. Hence, sigma here corresponds to the `noise_multiplier` parameter - # in the DP-SGD implementation found in privacy.optimizers.dp_optimizer - rdp = compute_rdp(q, sigma, steps, orders) - - eps, _, opt_order = get_privacy_spent(orders, rdp, target_delta=delta) - - return eps, opt_order + accountant = rdp_privacy_accountant.RdpAccountant(orders) + event = dp_event.SelfComposedDpEvent( + dp_event.PoissonSampledDpEvent(q, dp_event.GaussianDpEvent(sigma)), steps) + accountant.compose(event) + return accountant.get_epsilon_and_optimal_order(delta) def compute_noise(n, batch_size, target_epsilon, epochs, delta, noise_lbd): diff --git a/tutorials/BUILD b/tutorials/BUILD index e398d3b..8c0bb1a 100644 --- a/tutorials/BUILD +++ b/tutorials/BUILD @@ -94,6 +94,8 @@ py_binary( "//tensorflow_privacy/privacy/optimizers:dp_optimizer", "//third_party/py/tensorflow:tensorflow_compat_v1_estimator", "//third_party/py/tensorflow:tensorflow_estimator", + "@com_google_differential_py//python/dp_accounting:dp_event", + "@com_google_differential_py//python/dp_accounting/rdp:rdp_privacy_accountant", ], ) diff --git a/tutorials/mnist_lr_tutorial.py b/tutorials/mnist_lr_tutorial.py index 557bf21..cd65a7a 100644 --- a/tutorials/mnist_lr_tutorial.py +++ b/tutorials/mnist_lr_tutorial.py @@ -30,9 +30,10 @@ import numpy as np import tensorflow as tf from tensorflow import estimator as tf_estimator from tensorflow.compat.v1 import estimator as tf_compat_v1_estimator -from tensorflow_privacy.privacy.analysis.rdp_accountant import compute_rdp from tensorflow_privacy.privacy.analysis.rdp_accountant import get_privacy_spent from tensorflow_privacy.privacy.optimizers import dp_optimizer +from com_google_differential_py.python.dp_accounting import dp_event +from com_google_differential_py.python.dp_accounting.rdp import rdp_privacy_accountant GradientDescentOptimizer = tf.compat.v1.train.GradientDescentOptimizer @@ -169,10 +170,14 @@ def print_privacy_guarantees(epochs, batch_size, samples, noise_multiplier): eps, _, _ = get_privacy_spent(orders, rdp, target_delta=delta) print('\t{:g}% enjoy at least ({:.2f}, {})-DP'.format(p * 100, eps, delta)) - # Compute privacy guarantees for the Sampled Gaussian Mechanism. - rdp_sgm = compute_rdp(batch_size / samples, noise_multiplier, - epochs * steps_per_epoch, orders) - eps_sgm, _, _ = get_privacy_spent(orders, rdp_sgm, target_delta=delta) + accountant = rdp_privacy_accountant.RdpAccountant(orders) + event = dp_event.SelfComposedDpEvent( + dp_event.PoissonSampledDpEvent( + batch_size / samples, dp_event.GaussianDpEvent(noise_multiplier)), + epochs * steps_per_epoch) + accountant.compose(event) + eps_sgm = accountant.get_epsilon(target_delta=delta) + print('By comparison, DP-SGD analysis for training done with the same ' 'parameters and random shuffling in each epoch guarantees ' '({:.2f}, {})-DP for all samples.'.format(eps_sgm, delta))