626_privacy/mia_on_model_distillation
2
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

O1: update theoretical plots

Browse source
This commit is contained in:
Akemi Izuko 2024-12-06 18:57:02 -07:00
parent ce3a848eb7
commit e239602148
Signed by: akemi
GPG key ID: 8DE0764E1809E9FC
1 changed files with 86 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

97
one_run_audit/plot.py
View file

@ -1,21 +1,94 @@
import time
import math
import concurrent.futures
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
from equations import get_eps_audit
delta = 1e-5
p_value = 0.05
def compute_y(x_values, p, delta, proportion_correct, key):
return key, [get_eps_audit(x, x, math.floor(x *proportion_correct), delta, p) for x in x_values]
x_values = np.floor((1.5)**np.arange(30)).astype(int)
x_values = np.concatenate([x_values[x_values < 60000], [60000]])
y_values = [get_eps_audit(x, x, x, delta, p_value) for x in tqdm(x_values)]
def get_plots():
final_values = dict()
mul = 1.5 #1.275 #1.5
max = 60000 #2000 #60000
x_values = np.floor((mul)**np.arange(30)).astype(int)
x_values = np.concatenate([x_values[x_values < max], [max]])
with concurrent.futures.ProcessPoolExecutor(max_workers=16) as executor:
start_time = time.time()
futures = [
executor.submit(compute_y, x_values, 0.05, 0.0, 1.0, "y11"),
executor.submit(compute_y, x_values, 0.05, 1e-6, 1.0, "y12"),
executor.submit(compute_y, x_values, 0.05, 1e-4, 1.0, "y13"),
executor.submit(compute_y, x_values, 0.05, 1e-2, 1.0, "y14"),
executor.submit(compute_y, x_values, 0.01, 0.0, 1.0, "y21"),
executor.submit(compute_y, x_values, 0.01, 1e-6, 1.0, "y22"),
executor.submit(compute_y, x_values, 0.01, 1e-4, 1.0, "y23"),
executor.submit(compute_y, x_values, 0.01, 1e-2, 1.0, "y24"),
executor.submit(compute_y, x_values, 0.05, 0.0, 0.9, "y31"),
executor.submit(compute_y, x_values, 0.05, 1e-6, 0.9, "y32"),
executor.submit(compute_y, x_values, 0.05, 1e-4, 0.9, "y33"),
executor.submit(compute_y, x_values, 0.05, 1e-2, 0.9, "y34"),
executor.submit(compute_y, x_values, 0.01, 0.0, 0.9, "y41"),
executor.submit(compute_y, x_values, 0.01, 1e-6, 0.9, "y42"),
executor.submit(compute_y, x_values, 0.01, 1e-4, 0.9, "y43"),
executor.submit(compute_y, x_values, 0.01, 1e-2, 0.9, "y44"),
]
for future in concurrent.futures.as_completed(futures):
k, v = future.result()
final_values[k] = v
print(f"Took: {time.time()-start_time}s")
return final_values, x_values
def plot_to(value_set, x_values, title, fig_name):
plt.xscale('log')
plt.plot(x_values, y_values, marker='o')
plt.xlabel("Number of samples guessed correctly")
plt.ylabel("ε value audited")
plt.title("Maximum possible ε from audit")
plt.plot(x_values, value_set[0], marker='o', label='δ=0')
plt.plot(x_values, value_set[1], marker='o', label='δ=1e-6')
plt.plot(x_values, value_set[2], marker='o', label='δ=1e-4')
plt.plot(x_values, value_set[3], marker='o', label='δ=1e-2')
# 5. Save the plot as a PNG
plt.savefig("/dev/shm/my_plot.png", dpi=300, bbox_inches='tight')
plt.xlabel("Number of samples attacked")
plt.ylabel("Maximum ε lower-bound from audit")
plt.title(title)
plt.legend()
plt.savefig(fig_name, dpi=300, bbox_inches='tight')
def main():
final_values, x_values = get_plots()
plot_to(
[final_values[f"y1{i}"] for i in range(1,5)],
x_values,
"Maximum ε audit with p-value=0.05 and 100% MIA accuracy",
"/dev/shm/plot_05_100.png"
)
plot_to(
[final_values[f"y1{i}"] for i in range(1,5)],
x_values,
"Maximum ε audit with p-value=0.01 and 100% MIA accuracy",
"/dev/shm/plot_01_100.png"
)
plot_to(
[final_values[f"y1{i}"] for i in range(1,5)],
x_values,
"Maximum ε audit with p-value=0.05 and 90% MIA accuracy",
"/dev/shm/plot_05_90.png"
)
plot_to(
[final_values[f"y1{i}"] for i in range(1,5)],
x_values,
"Maximum ε audit with p-value=0.01 and 90% MIA accuracy"
"/dev/shm/plot_01_90.png"
)
if __name__ == '__main__':
main()