626_privacy/tensorflow_privacy
2
0
Fork 1
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Minor changes + tutorial

Browse source
This commit is contained in:
Christopher Choquette Choo 2019-06-19 11:04:18 -04:00
parent f41be2c598
commit 56e16f0a15
5 changed files with 719 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

278
privacy/bolton/losses.py
View file

@ -319,3 +319,281 @@ class StrongConvexBinaryCrossentropy(
return L1L2(l2=self.reg_lambda/2)
# class StrongConvexSparseCategoricalCrossentropy(
# losses.CategoricalCrossentropy,
# StrongConvexMixin
# ):
# """
# Strong Convex version of CategoricalCrossentropy loss using l2 weight
# regularization.
# """
#
# def __init__(self,
# reg_lambda: float,
# C: float,
# radius_constant: float,
# from_logits: bool = True,
# label_smoothing: float = 0,
# reduction: str = losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
# name: str = 'binarycrossentropy',
# dtype=tf.float32):
# """
# Args:
# reg_lambda: Weight regularization constant
# C: Penalty parameter C of the loss term
# radius_constant: constant defining the length of the radius
# reduction: reduction type to use. See super class
# label_smoothing: amount of smoothing to perform on labels
# relaxation of trust in labels, e.g. (1 -> 1-x, 0 -> 0+x)
# name: Name of the loss instance
# dtype: tf datatype to use for tensor conversions.
# """
# if reg_lambda <= 0:
# raise ValueError("reg lambda: {0} must be positive".format(reg_lambda))
# if C <= 0:
# raise ValueError('c: {0}, should be >= 0'.format(C))
# if radius_constant <= 0:
# raise ValueError('radius_constant: {0}, should be >= 0'.format(
# radius_constant
# ))
#
# self.C = C
# self.dtype = dtype
# self.reg_lambda = tf.constant(reg_lambda, dtype=self.dtype)
# super(StrongConvexSparseCategoricalCrossentropy, self).__init__(
# reduction=reduction,
# name=name,
# from_logits=from_logits,
# label_smoothing=label_smoothing,
# )
# self.radius_constant = radius_constant
#
# def call(self, y_true, y_pred):
# """Compute loss
#
# Args:
# y_true: Ground truth values.
# y_pred: The predicted values.
#
# Returns:
# Loss values per sample.
# """
# loss = super()
# loss = loss * self.C
# return loss
#
# def radius(self):
# """See super class.
# """
# return self.radius_constant / self.reg_lambda
#
# def gamma(self):
# """See super class.
# """
# return self.reg_lambda
#
# def beta(self, class_weight):
# """See super class.
# """
# max_class_weight = self.max_class_weight(class_weight, self.dtype)
# return self.C * max_class_weight + self.reg_lambda
#
# def lipchitz_constant(self, class_weight):
# """See super class.
# """
# max_class_weight = self.max_class_weight(class_weight, self.dtype)
# return self.C * max_class_weight + self.reg_lambda * self.radius()
#
# def kernel_regularizer(self):
# """
# l2 loss using reg_lambda as the l2 term (as desired). Required for
# this loss function to be strongly convex.
# :return:
# """
# return L1L2(l2=self.reg_lambda)
#
# class StrongConvexSparseCategoricalCrossentropy(
# losses.SparseCategoricalCrossentropy,
# StrongConvexMixin
# ):
# """
# Strong Convex version of SparseCategoricalCrossentropy loss using l2 weight
# regularization.
# """
#
# def __init__(self,
# reg_lambda: float,
# C: float,
# radius_constant: float,
# from_logits: bool = True,
# label_smoothing: float = 0,
# reduction: str = losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
# name: str = 'binarycrossentropy',
# dtype=tf.float32):
# """
# Args:
# reg_lambda: Weight regularization constant
# C: Penalty parameter C of the loss term
# radius_constant: constant defining the length of the radius
# reduction: reduction type to use. See super class
# label_smoothing: amount of smoothing to perform on labels
# relaxation of trust in labels, e.g. (1 -> 1-x, 0 -> 0+x)
# name: Name of the loss instance
# dtype: tf datatype to use for tensor conversions.
# """
# if reg_lambda <= 0:
# raise ValueError("reg lambda: {0} must be positive".format(reg_lambda))
# if C <= 0:
# raise ValueError('c: {0}, should be >= 0'.format(C))
# if radius_constant <= 0:
# raise ValueError('radius_constant: {0}, should be >= 0'.format(
# radius_constant
# ))
#
# self.C = C
# self.dtype = dtype
# self.reg_lambda = tf.constant(reg_lambda, dtype=self.dtype)
# super(StrongConvexHuber, self).__init__(reduction=reduction,
# name=name,
# from_logits=from_logits,
# label_smoothing=label_smoothing,
# )
# self.radius_constant = radius_constant
#
# def call(self, y_true, y_pred):
# """Compute loss
#
# Args:
# y_true: Ground truth values.
# y_pred: The predicted values.
#
# Returns:
# Loss values per sample.
# """
# loss = super()
# loss = loss * self.C
# return loss
#
# def radius(self):
# """See super class.
# """
# return self.radius_constant / self.reg_lambda
#
# def gamma(self):
# """See super class.
# """
# return self.reg_lambda
#
# def beta(self, class_weight):
# """See super class.
# """
# max_class_weight = self.max_class_weight(class_weight, self.dtype)
# return self.C * max_class_weight + self.reg_lambda
#
# def lipchitz_constant(self, class_weight):
# """See super class.
# """
# max_class_weight = self.max_class_weight(class_weight, self.dtype)
# return self.C * max_class_weight + self.reg_lambda * self.radius()
#
# def kernel_regularizer(self):
# """
# l2 loss using reg_lambda as the l2 term (as desired). Required for
# this loss function to be strongly convex.
# :return:
# """
# return L1L2(l2=self.reg_lambda)
#
#
# class StrongConvexCategoricalCrossentropy(
# losses.CategoricalCrossentropy,
# StrongConvexMixin
# ):
# """
# Strong Convex version of CategoricalCrossentropy loss using l2 weight
# regularization.
# """
#
# def __init__(self,
# reg_lambda: float,
# C: float,
# radius_constant: float,
# from_logits: bool = True,
# label_smoothing: float = 0,
# reduction: str = losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
# name: str = 'binarycrossentropy',
# dtype=tf.float32):
# """
# Args:
# reg_lambda: Weight regularization constant
# C: Penalty parameter C of the loss term
# radius_constant: constant defining the length of the radius
# reduction: reduction type to use. See super class
# label_smoothing: amount of smoothing to perform on labels
# relaxation of trust in labels, e.g. (1 -> 1-x, 0 -> 0+x)
# name: Name of the loss instance
# dtype: tf datatype to use for tensor conversions.
# """
# if reg_lambda <= 0:
# raise ValueError("reg lambda: {0} must be positive".format(reg_lambda))
# if C <= 0:
# raise ValueError('c: {0}, should be >= 0'.format(C))
# if radius_constant <= 0:
# raise ValueError('radius_constant: {0}, should be >= 0'.format(
# radius_constant
# ))
#
# self.C = C
# self.dtype = dtype
# self.reg_lambda = tf.constant(reg_lambda, dtype=self.dtype)
# super(StrongConvexHuber, self).__init__(reduction=reduction,
# name=name,
# from_logits=from_logits,
# label_smoothing=label_smoothing,
# )
# self.radius_constant = radius_constant
#
# def call(self, y_true, y_pred):
# """Compute loss
#
# Args:
# y_true: Ground truth values.
# y_pred: The predicted values.
#
# Returns:
# Loss values per sample.
# """
# loss = super()
# loss = loss * self.C
# return loss
#
# def radius(self):
# """See super class.
# """
# return self.radius_constant / self.reg_lambda
#
# def gamma(self):
# """See super class.
# """
# return self.reg_lambda
#
# def beta(self, class_weight):
# """See super class.
# """
# max_class_weight = self.max_class_weight(class_weight, self.dtype)
# return self.C * max_class_weight + self.reg_lambda
#
# def lipchitz_constant(self, class_weight):
# """See super class.
# """
# max_class_weight = self.max_class_weight(class_weight, self.dtype)
# return self.C * max_class_weight + self.reg_lambda * self.radius()
#
# def kernel_regularizer(self):
# """
# l2 loss using reg_lambda as the l2 term (as desired). Required for
# this loss function to be strongly convex.
# :return:
# """
# return L1L2(l2=self.reg_lambda)

2
privacy/bolton/models.py
View file

@ -170,7 +170,6 @@ class BoltonModel(Model):
self.layers,
class_weight_,
data_size,
self.n_outputs,
batch_size_,
) as _:
out = super(BoltonModel, self).fit(x=x,
@ -223,7 +222,6 @@ class BoltonModel(Model):
self.layers,
class_weight,
data_size,
self.n_outputs,
batch_size
) as _:
out = super(BoltonModel, self).fit_generator(

24
privacy/bolton/optimizers.py
View file

@ -137,7 +137,6 @@ class Bolton(optimizer_v2.OptimizerV2):
'class_weights',
'input_dim',
'n_samples',
'n_outputs',
'layers',
'batch_size',
'_is_init'
@ -166,6 +165,9 @@ class Bolton(optimizer_v2.OptimizerV2):
Returns:
"""
if not self._is_init:
raise Exception('This method must be called from within the optimizer\'s '
'context.')
radius = self.loss.radius()
for layer in self.layers:
weight_norm = tf.norm(layer.kernel, axis=0)
@ -323,7 +325,6 @@ class Bolton(optimizer_v2.OptimizerV2):
layers: list,
class_weights,
n_samples,
n_outputs,
batch_size
):
"""Entry point from context. Accepts required values for bolton method and
@ -338,7 +339,6 @@ class Bolton(optimizer_v2.OptimizerV2):
class_weights: class_weights used, which may either be a scalar or 1D
tensor with dim == n_classes.
n_samples number of rows/individual samples in the training set
n_outputs: number of output classes
batch_size: batch size used.
"""
if epsilon <= 0:
@ -352,20 +352,11 @@ class Bolton(optimizer_v2.OptimizerV2):
self.learning_rate.initialize(self.loss.beta(class_weights),
self.loss.gamma()
)
self.epsilon = _ops.convert_to_tensor_v2(epsilon, dtype=self.dtype)
self.class_weights = _ops.convert_to_tensor_v2(class_weights,
dtype=self.dtype
)
self.n_samples = _ops.convert_to_tensor_v2(n_samples,
dtype=self.dtype
)
self.n_outputs = _ops.convert_to_tensor_v2(n_outputs,
dtype=self.dtype
)
self.epsilon = tf.constant(epsilon, dtype=self.dtype)
self.class_weights = tf.constant(class_weights, dtype=self.dtype)
self.n_samples = tf.constant(n_samples, dtype=self.dtype)
self.layers = layers
self.batch_size = _ops.convert_to_tensor_v2(batch_size,
dtype=self.dtype
)
self.batch_size = tf.constant(batch_size, dtype=self.dtype)
return self
def __exit__(self, *args):
@ -397,6 +388,5 @@ class Bolton(optimizer_v2.OptimizerV2):
self.class_weights = None
self.n_samples = None
self.input_dim = None
self.n_outputs = None
self.layers = None
self._is_init = False

4
privacy/bolton/optimizers_test.py
View file

@ -314,7 +314,7 @@ class BoltonOptimizerTest(keras_parameterized.TestCase):
model.layers[0].kernel = \
model.layers[0].kernel_initializer((model.layer_input_shape[0],
model.n_outputs))
with bolton(noise, epsilon, model.layers, class_weights, 1, 1, 1) as _:
with bolton(noise, epsilon, model.layers, class_weights, 1, 1) as _:
pass
return _ops.convert_to_tensor_v2(bolton.epsilon, dtype=tf.float32)
epsilon = test_run()
@ -349,7 +349,7 @@ class BoltonOptimizerTest(keras_parameterized.TestCase):
model.layers[0].kernel = \
model.layers[0].kernel_initializer((model.layer_input_shape[0],
model.n_outputs))
with bolton(noise, epsilon, model.layers, 1, 1, 1, 1) as _:
with bolton(noise, epsilon, model.layers, 1, 1, 1) as _:
pass
with self.assertRaisesRegexp(ValueError, err_msg): # pylint: disable=deprecated-method
test_run(noise, epsilon)

432
tutorials/bolton_tutorial.ipynb Normal file
View file

@ -0,0 +1,432 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"pycharm": {
"is_executing": false
},
"scrolled": true
},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append('..')\n",
"import tensorflow as tf\n",
"from privacy.bolton import losses\n",
"from privacy.bolton import models"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, we will create a binary classification dataset with a single output dimension.\n",
"The samples for each label are repeated datapoints at different points in space."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"pycharm": {
"is_executing": false,
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(20, 2) (20, 1)\n"
]
}
],
"source": [
"# Parameters for dataset\n",
"n_samples = 10\n",
"input_dim = 2\n",
"n_outputs = 1\n",
"# Create binary classification dataset:\n",
"x_stack = [tf.constant(-1, tf.float32, (n_samples, input_dim)), \n",
" tf.constant(1, tf.float32, (n_samples, input_dim))]\n",
"y_stack = [tf.constant(0, tf.float32, (n_samples, 1)),\n",
" tf.constant(1, tf.float32, (n_samples, 1))]\n",
"x, y = tf.concat(x_stack, 0), tf.concat(y_stack, 0)\n",
"print(x.shape, y.shape)\n",
"generator = tf.data.Dataset.from_tensor_slices((x, y))\n",
"generator = generator.batch(10)\n",
"generator = generator.shuffle(10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, we will explore using the pre-built BoltonModel, which is a thin wrapper around a Keras Model using a single-layer neural network. It automatically uses the Bolton Optimizer which encompasses all the logic required for the Bolton Differential Privacy method.\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"pycharm": {
"is_executing": false
}
},
"outputs": [],
"source": [
"bolt = models.BoltonModel(n_outputs) # tell the model how many outputs we have."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now, we will pick our optimizer and Strongly Convex Loss function. The loss must extend from StrongConvexMixin and implement the associated methods. Some existing loss functions are pre-implemented in bolton.loss"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"pycharm": {
"is_executing": false
}
},
"outputs": [],
"source": [
"optimizer = tf.optimizers.SGD()\n",
"reg_lambda = 1\n",
"C = 1\n",
"radius_constant = 1\n",
"loss = losses.StrongConvexBinaryCrossentropy(reg_lambda, C, radius_constant)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For simplicity, we pick all parameters of the StrongConvexBinaryCrossentropy to be 1; these are all tunable and their impact can be read in losses.StrongConvexBinaryCrossentropy. We then compile the model with the chosen optimizer and loss, which will automatically wrap the chosen optimizer with the Bolton Optimizer, ensuring the required components function as required for privacy guarantees."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"pycharm": {
"is_executing": false
}
},
"outputs": [],
"source": [
"bolt.compile(optimizer, loss)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To fit the model, the optimizer will require additional information about the dataset and model. These parameters are:\n",
"1. the class_weights used\n",
"2. the number of samples in the dataset\n",
"3. the batch size\n",
"which the model will try to infer, if possible. If not, you will be required to pass these explicitly to the fit method.\n",
"As well, there are two privacy parameters than can be altered: \n",
"1. epsilon, a float\n",
"2. noise_distribution, a valid string indicating the distriution to use (must be implemented)\n",
"\n",
"The BoltonModel offers a helper method, .calculate_class_weight to aid in class_weight calculation."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING: Logging before flag parsing goes to stderr.\n",
"W0619 11:00:32.392859 4467058112 deprecation.py:323] From /Users/christopherchoo/PycharmProjects/privacy/venv/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:182: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Train on 20 samples\n",
"Epoch 1/2\n",
"20/20 [==============================] - 0s 4ms/sample - loss: 0.8146\n",
"Epoch 2/2\n",
"20/20 [==============================] - 0s 94us/sample - loss: 0.5699\n"
]
},
{
"data": {
"text/plain": [
"<tensorflow.python.keras.callbacks.History at 0x10543d0f0>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# required parameters\n",
"class_weight = None # default, use .calculate_class_weight to specify other values\n",
"batch_size = None # default, if it cannot be inferred, specify this\n",
"n_samples = None # default, if it cannot be iferred, specify this\n",
"# privacy parameters\n",
"epsilon = 2\n",
"noise_distribution = 'laplace'\n",
"\n",
"bolt.fit(x, \n",
" y, \n",
" epsilon=epsilon, \n",
" class_weight=class_weight, \n",
" batch_size=batch_size, \n",
" n_samples=n_samples,\n",
" noise_distribution=noise_distribution,\n",
" epochs=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We may also train a generator object, or try different optimizers and loss functions. Below, we will see that we must pass the number of samples as the fit method is unable to infer it for a generator."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"optimizer2 = tf.optimizers.Adam()\n",
"bolt.compile(optimizer2, loss)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Could not infer the number of samples. Please pass this in using n_samples.\n"
]
}
],
"source": [
"# required parameters\n",
"class_weight = None # default, use .calculate_class_weight to specify other values\n",
"batch_size = None # default, if it cannot be inferred, specify this\n",
"n_samples = None # default, if it cannot be iferred, specify this\n",
"# privacy parameters\n",
"epsilon = 2\n",
"noise_distribution = 'laplace'\n",
"try:\n",
" bolt.fit(generator,\n",
" epsilon=epsilon, \n",
" class_weight=class_weight, \n",
" batch_size=batch_size, \n",
" n_samples=n_samples,\n",
" noise_distribution=noise_distribution,\n",
" verbose=0\n",
" )\n",
"except ValueError as e:\n",
" print(e)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"And now, re running with the parameter set."
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<tensorflow.python.keras.callbacks.History at 0x1267db4a8>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"n_samples = 20\n",
"bolt.fit(generator,\n",
" epsilon=epsilon, \n",
" class_weight=class_weight, \n",
" batch_size=batch_size, \n",
" n_samples=n_samples,\n",
" noise_distribution=noise_distribution,\n",
" verbose=0\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You don't have to use the bolton model to use the Bolton method. There are only a few requirements:\n",
"1. make sure any requirements from the loss are implemented in the model.\n",
"2. instantiate the optimizer and use it as a context around your fit operation."
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"from privacy.bolton.optimizers import Bolton"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here, we create our own model and setup the Bolton optimizer."
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"class TestModel(tf.keras.Model):\n",
" def __init__(self, reg_layer, n_outputs=1):\n",
" super(TestModel, self).__init__(name='test')\n",
" self.output_layer = tf.keras.layers.Dense(n_outputs,\n",
" kernel_regularizer=reg_layer\n",
" )\n",
" \n",
" def call(self, inputs, training=False):\n",
" return self.output_layer(inputs)\n",
"\n",
"optimizer = tf.optimizers.SGD()\n",
"loss = losses.StrongConvexBinaryCrossentropy(reg_lambda, C, radius_constant)\n",
"optimizer = Bolton(optimizer, loss)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now, we instantiate our model and check for 1. Since our loss requires L2 regularization over the kernel, we will pass it to the model."
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"n_outputs = 1 # parameter for model and optimizer context.\n",
"test_model = TestModel(loss.kernel_regularizer(), n_outputs)\n",
"test_model.compile(optimizer, loss)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We comply with 2., and use the Bolton Optimizer as a context around the fit method."
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Train on 20 samples\n",
"Epoch 1/2\n",
"20/20 [==============================] - 0s 3ms/sample - loss: 0.9096\n",
"Epoch 2/2\n",
"20/20 [==============================] - 0s 430us/sample - loss: 0.5275\n"
]
}
],
"source": [
"# parameters for context\n",
"noise_distribution = 'laplace'\n",
"epsilon = 2\n",
"class_weights = 1 # Previosuly, the fit method auto-detected the class_weights.\n",
"# Here, we need to pass the class_weights explicitly. 1 is the equivalent of None.\n",
"n_samples = 20\n",
"batch_size = 5\n",
"\n",
"with optimizer(\n",
" noise_distribution=noise_distribution,\n",
" epsilon=epsilon,\n",
" layers=test_model.layers,\n",
" class_weights=class_weights, \n",
" n_samples=n_samples,\n",
" batch_size=batch_size\n",
") as _:\n",
" test_model.fit(x, y, batch_size=batch_size, epochs=2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
},
"pycharm": {
"stem_cell": {
"cell_type": "raw",
"metadata": {
"collapsed": false
},
"source": []
}
}
},
"nbformat": 4,
"nbformat_minor": 1
}