From 95b87270d99b6d296709153f4ebb205dcaed96a0 Mon Sep 17 00:00:00 2001 From: Steve Chien Date: Wed, 7 Feb 2024 16:48:13 -0800 Subject: [PATCH] Split TensorFlow Privacy pypi package into two parts-- one for DP training, and one for privacy tests. PiperOrigin-RevId: 605137291 --- pip_tools/build_empirical_pip_package.sh | 39 ++++++++++++++ pip_tools/build_pip_package.sh | 39 ++++++++++++++ pip_tools/publish_empirical_pip_package.sh | 41 ++++++++++++++ pip_tools/publish_pip_package.sh | 41 ++++++++++++++ pip_tools/test_empirical_pip_package.sh | 41 ++++++++++++++ pip_tools/test_pip_package.sh | 41 ++++++++++++++ requirements.txt | 1 - setup.py | 10 +--- setup_empirical.py | 53 +++++++++++++++++++ .../privacy/optimizers/dp_optimizer_test.py | 5 -- .../privacy/privacy_tests/BUILD | 9 ++++ .../privacy/privacy_tests/__init__.py | 3 ++ .../privacy/privacy_tests/version.py | 16 ++++++ 13 files changed, 325 insertions(+), 14 deletions(-) create mode 100755 pip_tools/build_empirical_pip_package.sh create mode 100755 pip_tools/build_pip_package.sh create mode 100755 pip_tools/publish_empirical_pip_package.sh create mode 100755 pip_tools/publish_pip_package.sh create mode 100755 pip_tools/test_empirical_pip_package.sh create mode 100755 pip_tools/test_pip_package.sh create mode 100644 setup_empirical.py create mode 100644 tensorflow_privacy/privacy/privacy_tests/version.py diff --git a/pip_tools/build_empirical_pip_package.sh b/pip_tools/build_empirical_pip_package.sh new file mode 100755 index 0000000..d355b43 --- /dev/null +++ b/pip_tools/build_empirical_pip_package.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright 2020, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tool to build the TensorFlow Privacy/Privacy Tests pip package. +set -e + +main() { + # Create a working directory. + local temp_dir="$(mktemp -d)" + trap "rm -rf ${temp_dir}" EXIT + + # Create a virtual environment + python3.11 -m venv "${temp_dir}/venv" + source "${temp_dir}/venv/bin/activate" + python --version + pip install --upgrade pip + pip --version + + # Build the pip package + pip install --upgrade setuptools wheel + python "setup_empirical.py" sdist bdist_wheel + + # Cleanup. + deactivate +} + +main "$@" diff --git a/pip_tools/build_pip_package.sh b/pip_tools/build_pip_package.sh new file mode 100755 index 0000000..decb50c --- /dev/null +++ b/pip_tools/build_pip_package.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright 2020, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tool to build the TensorFlow Privacy pip package. +set -e + +main() { + # Create a working directory. + local temp_dir="$(mktemp -d)" + trap "rm -rf ${temp_dir}" EXIT + + # Create a virtual environment + python3.11 -m venv "${temp_dir}/venv" + source "${temp_dir}/venv/bin/activate" + python --version + pip install --upgrade pip + pip --version + + # Build the pip package + pip install --upgrade setuptools wheel + python "setup.py" sdist bdist_wheel + + # Cleanup. + deactivate +} + +main "$@" diff --git a/pip_tools/publish_empirical_pip_package.sh b/pip_tools/publish_empirical_pip_package.sh new file mode 100755 index 0000000..42dcf76 --- /dev/null +++ b/pip_tools/publish_empirical_pip_package.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2020, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tool to publish the TensorFlow Privacy pip package. +set -e + +main() { + # Create a working directory. + local temp_dir="$(mktemp -d)" + trap "rm -rf ${temp_dir}" EXIT + + # Create a virtual environment + python3.11 -m venv "${temp_dir}/venv" + source "${temp_dir}/venv/bin/activate" + python --version + pip install --upgrade pip + pip --version + + # Publish the pip package. + package="$(ls "dist/"*".whl" | head -n1)" + pip install --upgrade twine + twine check "${package}" + twine upload "${package}" + + # Cleanup. + deactivate +} + +main "$@" diff --git a/pip_tools/publish_pip_package.sh b/pip_tools/publish_pip_package.sh new file mode 100755 index 0000000..42dcf76 --- /dev/null +++ b/pip_tools/publish_pip_package.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2020, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tool to publish the TensorFlow Privacy pip package. +set -e + +main() { + # Create a working directory. + local temp_dir="$(mktemp -d)" + trap "rm -rf ${temp_dir}" EXIT + + # Create a virtual environment + python3.11 -m venv "${temp_dir}/venv" + source "${temp_dir}/venv/bin/activate" + python --version + pip install --upgrade pip + pip --version + + # Publish the pip package. + package="$(ls "dist/"*".whl" | head -n1)" + pip install --upgrade twine + twine check "${package}" + twine upload "${package}" + + # Cleanup. + deactivate +} + +main "$@" diff --git a/pip_tools/test_empirical_pip_package.sh b/pip_tools/test_empirical_pip_package.sh new file mode 100755 index 0000000..d93ccf7 --- /dev/null +++ b/pip_tools/test_empirical_pip_package.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2020, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tool to build the TensorFlow Privacy pip package. +set -e + +main() { + # Create a working directory. + local temp_dir="$(mktemp -d)" + trap "rm -rf ${temp_dir}" EXIT + + # Create a virtual environment + python3.11 -m venv "${temp_dir}/venv" + source "${temp_dir}/venv/bin/activate" + python --version + pip install --upgrade pip + pip --version + + # Test the pip package. + package="$(ls "dist/"*".whl" | head -n1)" + pip install --upgrade "${package}" + pip freeze + python -c "import tensorflow_privacy.privacy.privacy_tests as pt; print(pt.__version__)" + + # Cleanup. + deactivate +} + +main "$@" diff --git a/pip_tools/test_pip_package.sh b/pip_tools/test_pip_package.sh new file mode 100755 index 0000000..0f6a772 --- /dev/null +++ b/pip_tools/test_pip_package.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2020, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tool to build the TensorFlow Privacy pip package. +set -e + +main() { + # Create a working directory. + local temp_dir="$(mktemp -d)" + trap "rm -rf ${temp_dir}" EXIT + + # Create a virtual environment + python3.11 -m venv "${temp_dir}/venv" + source "${temp_dir}/venv/bin/activate" + python --version + pip install --upgrade pip + pip --version + + # Test the pip package. + package="$(ls "dist/"*".whl" | head -n1)" + pip install --upgrade "${package}" + pip freeze + python -c "import tensorflow_privacy as tfp; print(tfp.__version__)" + + # Cleanup. + deactivate +} + +main "$@" diff --git a/requirements.txt b/requirements.txt index 41cf69e..eb5c6bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,6 @@ # `~=x.0`. absl-py>=1.0,==1.* -attrs>=21.4 dm-tree==0.1.8 dp-accounting==0.4.3 immutabledict~=2.2 diff --git a/setup.py b/setup.py index 80bfe5d..34a958f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""TensorFlow Privacy library setup file for pip.""" +"""TensorFlow Privacy/DP Training library setup file for pip.""" import setuptools @@ -33,24 +33,18 @@ setuptools.setup( long_description_content_type='text/plain', url='https://github.com/tensorflow/privacy', license='Apache-2.0', - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=['*privacy.privacy_tests*']), install_requires=[ 'absl-py>=1.0,==1.*', - 'attrs>=21.4', 'dm-tree==0.1.8', 'dp-accounting==0.4.3', - 'immutabledict~=2.2', - 'matplotlib~=3.3', 'numpy~=1.21', 'packaging~=22.0', - 'pandas~=1.4', 'scikit-learn>=1.0,==1.*', 'scipy~=1.9', - 'statsmodels==0.14.0', 'tensorflow-estimator~=2.4', 'tensorflow-probability~=0.22.0', 'tensorflow~=2.4', - 'tf-models-official~=2.13', ], python_requires='>=3.9.0,<3.12', ) diff --git a/setup_empirical.py b/setup_empirical.py new file mode 100644 index 0000000..5fdccb5 --- /dev/null +++ b/setup_empirical.py @@ -0,0 +1,53 @@ +# Copyright 2024, The TensorFlow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TensorFlow Privacy/Privacy Tests library setup file for pip.""" + +import setuptools + +with open('tensorflow_privacy/privacy/privacy_tests/version.py') as file: + globals_dict = {} + exec(file.read(), globals_dict) # pylint: disable=exec-used + VERSION = globals_dict['__version__'] + +README = ( + 'A Python library that includes implementations of tests for empirical ' + 'privacy.' +) + +setuptools.setup( + name='tensorflow_empirical_privacy', + version=VERSION, + description='Tests for empirical privacy.', + long_description=README, + long_description_content_type='text/plain', + url='https://github.com/tensorflow/privacy', + license='Apache-2.0', + packages=setuptools.find_packages( + where='privacy', include=['privacy_tests*'] + ), + install_requires=[ + 'absl-py>=1.0,==1.*', + 'immutabledict~=2.2', + 'matplotlib~=3.3', + 'numpy~=1.21', + 'pandas~=1.4', + 'scikit-learn>=1.0,==1.*', + 'scipy~=1.9', + 'statsmodels==0.14.0', + 'tensorflow~=2.4', + 'tensorflow-privacy>=0.8.12', + 'tf-models-official~=2.13', + ], + python_requires='>=3.9.0,<3.12', +) diff --git a/tensorflow_privacy/privacy/optimizers/dp_optimizer_test.py b/tensorflow_privacy/privacy/optimizers/dp_optimizer_test.py index a22181b..af526a7 100644 --- a/tensorflow_privacy/privacy/optimizers/dp_optimizer_test.py +++ b/tensorflow_privacy/privacy/optimizers/dp_optimizer_test.py @@ -412,11 +412,6 @@ class DPOptimizerTest(tf.test.TestCase, parameterized.TestCase): tf.keras.optimizers.legacy.Adam) self._test_write_out_and_reload(optimizer_class) - def testWriteOutAndReloadSGD(self): - optimizer_class = dp_optimizer.make_gaussian_optimizer_class( - tf.keras.optimizers.legacy.SGD) - self._test_write_out_and_reload(optimizer_class) - if __name__ == '__main__': tf.test.main() diff --git a/tensorflow_privacy/privacy/privacy_tests/BUILD b/tensorflow_privacy/privacy/privacy_tests/BUILD index 779b878..f29c69e 100644 --- a/tensorflow_privacy/privacy/privacy_tests/BUILD +++ b/tensorflow_privacy/privacy/privacy_tests/BUILD @@ -5,6 +5,9 @@ licenses(["notice"]) py_library( name = "privacy_tests", srcs = ["__init__.py"], + deps = [ + ":version", + ], ) py_test( @@ -32,3 +35,9 @@ py_library( name = "epsilon_lower_bound", srcs = ["epsilon_lower_bound.py"], ) + +py_library( + name = "version", + srcs = ["version.py"], + srcs_version = "PY3", +) diff --git a/tensorflow_privacy/privacy/privacy_tests/__init__.py b/tensorflow_privacy/privacy/privacy_tests/__init__.py index 2e950c8..ce0b905 100644 --- a/tensorflow_privacy/privacy/privacy_tests/__init__.py +++ b/tensorflow_privacy/privacy/privacy_tests/__init__.py @@ -11,3 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""TensorFlow Empirical Privacy.""" + +from tensorflow_privacy.privacy.privacy_tests.version import __version__ # pylint: disable=g-bad-import-order diff --git a/tensorflow_privacy/privacy/privacy_tests/version.py b/tensorflow_privacy/privacy/privacy_tests/version.py new file mode 100644 index 0000000..81b3103 --- /dev/null +++ b/tensorflow_privacy/privacy/privacy_tests/version.py @@ -0,0 +1,16 @@ +# Copyright 2024, The TensorFlow Privacy Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TensorFlow Privacy/Privacy Tests version.""" + +__version__ = '0.1.0'