diff --git a/.bazelrc b/.bazelrc index 638bb39fafd66e..7beb93e42072ae 100644 --- a/.bazelrc +++ b/.bazelrc @@ -137,8 +137,19 @@ build --announce_rc # Other build flags. build --define=grpc_no_ares=true -# Prevent regression of https://github.com/bazelbuild/bazel/issues/7362 -build --incompatible_remove_legacy_whole_archive +# See https://github.com/bazelbuild/bazel/issues/7362 for information on what +# --incompatible_remove_legacy_whole_archive flag does. +# This flag is set to true in Bazel 1.0 and newer versions. We tried to migrate +# Tensorflow to the default, however test coverage wasn't enough to catch the +# errors. +# There is ongoing work on Bazel team's side to provide support for transitive +# shared libraries. As part of migrating to transitive shared libraries, we +# hope to provide a better mechanism for control over symbol exporting, and +# then tackle this issue again. +# +# TODO: Remove this line once TF doesn't depend on Bazel wrapping all library +# archives in -whole_archive -no_whole_archive. +build --noincompatible_remove_legacy_whole_archive # Modular TF build options build:dynamic_kernels --define=dynamic_loaded_kernels=true @@ -149,6 +160,10 @@ build:c++17 --cxxopt=-std=c++1z build:c++17 --cxxopt=-stdlib=libc++ build:c++1z --config=c++17 +# Tensorflow uses M_* math constants that only get defined by MSVC headers if +# _USE_MATH_DEFINES is defined. +build:windows --copt=/D_USE_MATH_DEFINES + # Default paths for TF_SYSTEM_LIBS build --define=PREFIX=/usr build --define=LIBDIR=$(PREFIX)/lib diff --git a/.bazelversion b/.bazelversion index 30f6cf8d98ce36..25939d35c738f0 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -0.26.1 +0.29.1 diff --git a/RELEASE.md b/RELEASE.md index c415315f88270d..3468c459f4242b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,109 @@ +# Release 2.1.0 + +TensorFlow 2.1 will be the last TF release supporting Python 2. Python 2 support [officially ends an January 1, 2020](https://www.python.org/dev/peps/pep-0373/#update). [As announced earlier](https://groups.google.com/a/tensorflow.org/d/msg/announce/gVwS5RC8mds/dCt1ka2XAAAJ), TensorFlow will also stop supporting Python 2 starting January 1, 2020, and no more releases are expected in 2019. + +## Major Features and Improvements +* The `tensorflow` pip package now includes GPU support by default (same as `tensorflow-gpu`) for both Linux and Windows. This runs on machines with and without NVIDIA GPUs. `tensorflow-gpu` is still available, and CPU-only packages can be downloaded at `tensorflow-cpu` for users who are concerned about package size. +* **Windows users:** Officially-released `tensorflow` Pip packages are now built with Visual Studio 2019 version 16.4 in order to take advantage of the new `/d2ReducedOptimizeHugeFunctions` compiler flag. To use these new packages, you must install "Microsoft Visual C++ Redistributable for Visual Studio 2015, 2017 and 2019", available from Microsoft's website [here](https://support.microsoft.com/help/2977003/the-latest-supported-visual-c-downloads). + * This does not change the minimum required version for building TensorFlow from source on Windows, but builds enabling `EIGEN_STRONG_INLINE` can take over 48 hours to compile without this flag. Refer to `configure.py` for more information about `EIGEN_STRONG_INLINE` and `/d2ReducedOptimizeHugeFunctions`. + * If either of the required DLLs, `msvcp140.dll` (old) or `msvcp140_1.dll` (new), are missing on your machine, `import tensorflow` will print a warning message. +* The `tensorflow` pip package is built with CUDA 10.1 and cuDNN 7.6. +* `tf.keras` + * Experimental support for mixed precision is available on GPUs and Cloud TPUs. See [usage guide](https://www.tensorflow.org/guide/keras/mixed_precision). + * Introduced the `TextVectorization` layer, which takes as input raw strings and takes care of text standardization, tokenization, n-gram generation, and vocabulary indexing. See this [end-to-end text classification example](https://colab.research.google.com/drive/1RvCnR7h0_l4Ekn5vINWToI9TNJdpUZB3). + * Keras `.compile` `.fit` `.evaluate` and `.predict` are allowed to be outside of the DistributionStrategy scope, as long as the model was constructed inside of a scope. + * Experimental support for Keras `.compile`, `.fit`, `.evaluate`, and `.predict` is available for Cloud TPUs, Cloud TPU, for all types of Keras models (sequential, functional and subclassing models). + * Automatic outside compilation is now enabled for Cloud TPUs. This allows `tf.summary` to be used more conveniently with Cloud TPUs. + * Dynamic batch sizes with DistributionStrategy and Keras are supported on Cloud TPUs. + * Support for `.fit`, `.evaluate`, `.predict` on TPU using numpy data, in addition to `tf.data.Dataset`. + * Keras reference implementations for many popular models are available in the TensorFlow [Model Garden](https://github.com/tensorflow/models/tree/master/official). +* `tf.data` + * Changes rebatching for `tf.data datasets` + DistributionStrategy for better performance. Note that the dataset also behaves slightly differently, in that the rebatched dataset cardinality will always be a multiple of the number of replicas. + * `tf.data.Dataset` now supports automatic data distribution and sharding in distributed environments, including on TPU pods. + * Distribution policies for `tf.data.Dataset` can now be tuned with 1. `tf.data.experimental.AutoShardPolicy(OFF, AUTO, FILE, DATA)` 2. `tf.data.experimental.ExternalStatePolicy(WARN, IGNORE, FAIL)` +* `tf.debugging` + * Add `tf.debugging.enable_check_numerics()` and `tf.debugging.disable_check_numerics()` to help debugging the root causes of issues involving infinities and `NaN`s. +* `tf.distribute` + * Custom training loop support on TPUs and TPU pods is avaiable through `strategy.experimental_distribute_dataset`, `strategy.experimental_distribute_datasets_from_function`, `strategy.experimental_run_v2`, `strategy.reduce`. + * Support for a global distribution strategy through `tf.distribute.experimental_set_strategy(),` in addition to `strategy.scope()`. +* `TensorRT` + * [TensorRT 6.0](https://developer.nvidia.com/tensorrt#tensorrt-whats-new) is now supported and enabled by default. This adds support for more TensorFlow ops including Conv3D, Conv3DBackpropInputV2, AvgPool3D, MaxPool3D, ResizeBilinear, and ResizeNearestNeighbor. In addition, the TensorFlow-TensorRT python conversion API is exported as `tf.experimental.tensorrt.Converter`. +* Environment variable `TF_DETERMINISTIC_OPS` has been added. When set to "true" or "1", this environment variable makes `tf.nn.bias_add` operate deterministically (i.e. reproducibly), but currently only when XLA JIT compilation is *not* enabled. Setting `TF_DETERMINISTIC_OPS` to "true" or "1" also makes cuDNN convolution and max-pooling operate deterministically. This makes Keras Conv\*D and MaxPool\*D layers operate deterministically in both the forward and backward directions when running on a CUDA-enabled GPU. + +## Known issues +Because of [issues with building on windows](https://github.com/tensorflow/tensorflow/issues/10521), we turned off eigen strong inlining for the Windows builds. Windows binaries are expected to be slightly slower until the build issues are resolved. + +## Breaking Changes +* Deletes `Operation.traceback_with_start_lines` for which we know of no usages. +* Removed `id` from `tf.Tensor.__repr__()` as `id` is not useful other than internal debugging. +* Some `tf.assert_*` methods now raise assertions at operation creation time if the input tensors' values are known at that time, not during the `session.run()`. This only changes behavior when the graph execution would have resulted in an error. When this happens, a noop is returned and the input tensors are marked non-feedable. In other words, if they are used as keys in `feed_dict` argument to `session.run()`, an error will be raised. Also, because some assert ops don't make it into the graph, the graph structure changes. A different graph can result in different per-op random seeds when they are not given explicitly (most often). +* The following APIs are not longer experimental: `tf.config.list_logical_devices`, `tf.config.list_physical_devices`, `tf.config.get_visible_devices`, `tf.config.set_visible_devices`, `tf.config.get_logical_device_configuration`, `tf.config.set_logical_device_configuration`. +* `tf.config.experimentalVirtualDeviceConfiguration` has been renamed to `tf.config.LogicalDeviceConfiguration`. +* `tf.config.experimental_list_devices` has been removed, please use +`tf.config.list_logical_devices`. + +## Bug Fixes and Other Changes +* `tf.data` + * Fixes concurrency issue with `tf.data.experimental.parallel_interleave` with `sloppy=True`. + * Add `tf.data.experimental.dense_to_ragged_batch()`. + * Extend `tf.data` parsing ops to support `RaggedTensors`. +* `tf.distribute` + * Fix issue where GRU would crash or give incorrect output when a `tf.distribute.Strategy` was used. +* `tf.estimator` + * Added option in `tf.estimator.CheckpointSaverHook` to not save the `GraphDef`. + * Moving the checkpoint reader from swig to pybind11. +* `tf.keras` + * Export `depthwise_conv2d` in `tf.keras.backend`. + * In Keras Layers and Models, Variables in `trainable_weights`, `non_trainable_weights`, and `weights` are explicitly deduplicated. + * Keras `model.load_weights` now accepts `skip_mismatch` as an argument. This was available in external Keras, and has now been copied over to `tf.keras`. + * Fix the input shape caching behavior of Keras convolutional layers. + * `Model.fit_generator`, `Model.evaluate_generator`, `Model.predict_generator`, `Model.train_on_batch`, `Model.test_on_batch`, and `Model.predict_on_batch` methods now respect the `run_eagerly` property, and will correctly run using `tf.function` by default. Note that `Model.fit_generator`, `Model.evaluate_generator`, and `Model.predict_generator` are deprecated endpoints. They are subsumed by `Model.fit`, `Model.evaluate`, and `Model.predict` which now support generators and Sequences. +* `tf.lite` + * Legalization for `NMS` ops in TFLite. + * add `narrow_range` and `axis` to `quantize_v2` and `dequantize` ops. + * Added support for `FusedBatchNormV3` in converter. + * Add an `errno`-like field to `NNAPI` delegate for detecting `NNAPI` errors for fallback behaviour. + * Refactors `NNAPI` Delegate to support detailed reason why an operation is not accelerated. + * Converts hardswish subgraphs into atomic ops. +* Other + * Critical stability updates for TPUs, especially in cases where the XLA compiler produces compilation errors. + * TPUs can now be re-initialized multiple times, using `tf.tpu.experimental.initialize_tpu_system`. + * Add `RaggedTensor.merge_dims()`. + * Added new `uniform_row_length` row-partitioning tensor to `RaggedTensor`. + * Add `shape` arg to `RaggedTensor.to_tensor`; Improve speed of `RaggedTensor.to_tensor`. + * `tf.io.parse_sequence_example` and `tf.io.parse_single_sequence_example` now support ragged features. + * Fix `while_v2` with variables in custom gradient. + * Support taking gradients of V2 `tf.cond` and `tf.while_loop` using `LookupTable`. + * Fix bug where `vectorized_map` failed on inputs with unknown static shape. + * Add preliminary support for sparse CSR matrices. + * Tensor equality with `None` now behaves as expected. + * Make calls to `tf.function(f)()`, `tf.function(f).get_concrete_function` and `tf.function(f).get_initialization_function` thread-safe. + * Extend `tf.identity` to work with CompositeTensors (such as SparseTensor) + * Added more `dtypes` and zero-sized inputs to `Einsum` Op and improved its performance + * Enable multi-worker `NCCL` `all-reduce` inside functions executing eagerly. + * Added complex128 support to `RFFT`, `RFFT2D`, `RFFT3D`, `IRFFT`, `IRFFT2D`, and `IRFFT3D`. + * Add `pfor` converter for `SelfAdjointEigV2`. + * Add `tf.math.ndtri` and `tf.math.erfinv`. + * Add `tf.config.experimental.enable_mlir_bridge` to allow using MLIR compiler bridge in eager model. + * Added support for MatrixSolve on Cloud TPU / XLA. + * Added `tf.autodiff.ForwardAccumulator` for forward-mode autodiff + * Add `LinearOperatorPermutation`. + * A few performance optimizations on `tf.reduce_logsumexp`. + * Added multilabel handling to `AUC` metric + * Optimization on `zeros_like`. + * Dimension constructor now requires `None` or types with an `__index__` method. + * Add `tf.random.uniform` microbenchmark. + * Use `_protogen` suffix for proto library targets instead of `_cc_protogen` suffix. + * Moving the checkpoint reader from `swig` to `pybind11`. + * `tf.device` & `MirroredStrategy` now supports passing in a `tf.config.LogicalDevice` + * If you're building Tensorflow from source, consider using [bazelisk](https://github.com/bazelbuild/bazelisk) to automatically download and use the correct Bazel version. Bazelisk reads the `.bazelversion` file at the root of the project directory. + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +8bitmp3, Aaron Ma, AbdüLhamit Yilmaz, Abhai Kollara, aflc, Ag Ramesh, Albert Z. Guo, Alex Torres, amoitra, Andrii Prymostka, angeliand, Anshuman Tripathy, Anthony Barbier, Anton Kachatkou, Anubh-V, Anuja Jakhade, Artem Ryabov, autoih, Bairen Yi, Bas Aarts, Basit Ayantunde, Ben Barsdell, Bhavani Subramanian, Brett Koonce, candy.dc, Captain-Pool, caster, cathy, Chong Yan, Choong Yin Thong, Clayne Robison, Colle, Dan Ganea, David Norman, David Refaeli, dengziming, Diego Caballero, Divyanshu, djshen, Douman, Duncan Riach, EFanZh, Elena Zhelezina, Eric Schweitz, Evgenii Zheltonozhskii, Fei Hu, fo40225, Fred Reiss, Frederic Bastien, Fredrik Knutsson, fsx950223, fwcore, George Grzegorz Pawelczak, George Sterpu, Gian Marco Iodice, Giorgio Arena, giuros01, Gomathi Ramamurthy, Guozhong Zhuang, Haifeng Jin, Haoyu Wu, HarikrishnanBalagopal, HJYOO, Huang Chen-Yi, Ilham Firdausi Putra, Imran Salam, Jared Nielsen, Jason Zaman, Jasper Vicenti, Jeff Daily, Jeff Poznanovic, Jens Elofsson, Jerry Shih, jerryyin, Jesper Dramsch, jim.meyer, Jongwon Lee, Jun Wan, Junyuan Xie, Kaixi Hou, kamalkraj, Kan Chen, Karthik Muthuraman, Keiji Ariyama, Kevin Rose, Kevin Wang, Koan-Sin Tan, kstuedem, Kwabena W. Agyeman, Lakshay Tokas, latyas, Leslie-Fang-Intel, Li, Guizi, Luciano Resende, Lukas Folle, Lukas Geiger, Mahmoud Abuzaina, Manuel Freiberger, Mark Ryan, Martin Mlostek, Masaki Kozuki, Matthew Bentham, Matthew Denton, mbhuiyan, mdfaijul, Muhwan Kim, Nagy Mostafa, nammbash, Nathan Luehr, Nathan Wells, Niranjan Hasabnis, Oleksii Volkovskyi, Olivier Moindrot, olramde, Ouyang Jin, OverLordGoldDragon, Pallavi G, Paul Andrey, Paul Wais, pkanwar23, Pooya Davoodi, Prabindh Sundareson, Rajeshwar Reddy T, Ralovich, Kristof, Refraction-Ray, Richard Barnes, richardbrks, Robert Herbig, Romeo Kienzler, Ryan Mccormick, saishruthi, Saket Khandelwal, Sami Kama, Sana Damani, Satoshi Tanaka, Sergey Mironov, Sergii Khomenko, Shahid, Shawn Presser, ShengYang1, Siddhartha Bagaria, Simon Plovyt, skeydan, srinivasan.narayanamoorthy, Stephen Mugisha, sunway513, Takeshi Watanabe, Taylor Jakobson, TengLu, TheMindVirus, ThisIsIsaac, Tim Gates, Timothy Liu, Tomer Gafner, Trent Lo, Trevor Hickey, Trevor Morris, vcarpani, Wei Wang, Wen-Heng (Jack) Chung, wenshuai, Wenshuai-Xiaomi, wenxizhu, william, William D. Irons, Xinan Jiang, Yannic, Yasir Modak, Yasuhiro Matsumoto, Yong Tang, Yongfeng Gu, Youwei Song, Zaccharie Ramzi, Zhang, Zhenyu Guo, 王振华 (Zhenhua Wang), 韩董, 이중건 Isaac Lee + # Release 1.15.0 This is the last 1.x release for TensorFlow. We do not expect to update the 1.x branch with features, although we will issue patch releases to fix vulnerabilities for at least one year. @@ -83,7 +189,7 @@ This enables writing forward compatible code: by explicitly importing either `te This release contains contributions from many people at Google, as well as: -a6802739, Aaron Ma, Abdullah Selek, Abolfazl Shahbazi, Ag Ramesh, Albert Z. Guo, Albin Joy, Alex Itkes, Alex Sergeev, Alexander Pivovarov, Alexey Romanov, alhkad, Amit Srivastava, amoitra, Andrew Lihonosov, Andrii Prymostka, Anuj Rawat, Astropeak, Ayush Agrawal, Bairen Yi, Bas Aarts, Bastian Eichenberger, Ben Barsdell, Benjamin Peterson, bhack, Bharat Raghunathan, Bhavani Subramanian, Bryan Cutler, candy.dc, Cao Zongyan, Captain-Pool, Casper Da Costa-Luis, Chen Guoyin, Cheng Chang, chengchingwen, Chong Yan, Choong Yin Thong, Christopher Yeh, Clayne Robison, Coady, Patrick, Dan Ganea, David Norman, Denis Khalikov, Deven Desai, Diego Caballero, Duncan Dean, Duncan Riach, Dwight J Lyle, Eamon Ito-Fisher, eashtian3, EFanZh, ejot, Elroy Ashtian Jr, Eric Schweitz, Fangjun Kuang, Fei Hu, fo40225, formath, Fred Reiss, Frederic Bastien, Fredrik Knutsson, G. Hussain Chinoy, Gabriel, gehring, George Grzegorz Pawelczak, Gianluca Varisco, Gleb Popov, Greg Peatfield, Guillaume Klein, Gurpreet Singh, Gustavo Lima Chaves, haison, Haraldur TóMas HallgríMsson, HarikrishnanBalagopal, HåKon Sandsmark, I-Hong, Ilham Firdausi Putra, Imran Salam, Jason Zaman, Jason Zavaglia, jayhpark530, jefby, Jeff Daily, Jeffrey Poznanovic, Jekyll Lai, Jeroen BéDorf, Jerry Shih, jerryyin, jiakai, JiangXIAO, Joe Bowser, Joel Shapiro, Johan Gunnarsson, Jojimon Varghese, Joon, Josh Beal, Julian Niedermeier, Jun Wan, Junqin Zhang, Junyuan Xie, Justin Tunis, Kaixi Hou, Karl Lessard, Karthik Muthuraman, Kbhute-Ibm, khanhlvg, Koock Yoon, kstuedem, Kyuwon Kim, Lakshay Tokas, leike666666, leonard951, Leslie-Fang, Leslie-Fang-Intel, Li, Guizi, Lukas Folle, Lukas Geiger, Mahmoud Abuzaina, Manraj Singh Grover, Margaret Maynard-Reid, Mark Ryan, Matt Conley, Matthew Bentham, Matthew Denton, mbhuiyan, mdfaijul, Mei Jie, merturl, MichaelKonobeev, Michal W. Tarnowski, minds, mpppk, musikisomorphie, Nagy Mostafa, Nayana Thorat, Neil, Niels Ole Salscheider, Niklas SilfverströM, Niranjan Hasabnis, ocjosen, olramde, Pariksheet Pinjari, Patrick J. Lopresti, Patrik Gustavsson, per1234, PeterLee, Phan Van Nguyen Duc, Phillip Kravtsov, Pooya Davoodi, Pranav Marathe, Putra Manggala, Qingqing Cao, Rajeshwar Reddy T, Ramon ViñAs, Rasmus Diederichsen, Reuben Morais, richardbrks, robert, RonLek, Ryan Jiang, saishruthi, Saket Khandelwal, Saleem Abdulrasool, Sami Kama, Sana-Damani, Sergii Khomenko, Severen Redwood, Shubham Goyal, Sigrid Keydana, Siju Samuel, sleighsoft, smilu97, Son Tran, Srini511, srinivasan.narayanamoorthy, Sumesh Udayakumaran, Sungmann Cho, Tae-Hwan Jung, Taehoon Lee, Takeshi Watanabe, TengLu, terryky, TheMindVirus, ThisIsIsaac, Till Hoffmann, Timothy Liu, Tomer Gafner, Tongxuan Liu, Trent Lo, Trevor Morris, Uday Bondhugula, Vasileios Lioutas, vbvg2008, Vishnuvardhan Janapati, Vivek Suryamurthy, Wei Wang, Wen-Heng (Jack) Chung, wenxizhu, William D. Irons, winstonq, wyzhao, Xiaoming (Jason) Cui, Xinan Jiang, Xinping Wang, Yann-Yy, Yasir Modak, Yong Tang, Yongfeng Gu, Yuchen Ying, Yuxin Wu, zyeric, 王振华 (Zhenhua Wang) +a6802739, Aaron Ma, Abdullah Selek, Abolfazl Shahbazi, Ag Ramesh, Albert Z. Guo, Albin Joy, Alex Itkes, Alex Sergeev, Alexander Pivovarov, Alexey Romanov, alhkad, Amit Srivastava, amoitra, Andrew Lihonosov, Andrii Prymostka, Anuj Rawat, Astropeak, Ayush Agrawal, Bairen Yi, Bas Aarts, Bastian Eichenberger, Ben Barsdell, Benjamin Peterson, bhack, Bharat Raghunathan, Bhavani Subramanian, Bryan Cutler, candy.dc, Cao Zongyan, Captain-Pool, Casper Da Costa-Luis, Chen Guoyin, Cheng Chang, chengchingwen, Chong Yan, Choong Yin Thong, Christopher Yeh, Clayne Robison, Coady, Patrick, Dan Ganea, David Norman, Denis Khalikov, Deven Desai, Diego Caballero, Duncan Dean, Duncan Riach, Dwight J Lyle, Eamon Ito-Fisher, eashtian3, EFanZh, ejot, Elroy Ashtian Jr, Eric Schweitz, Fangjun Kuang, Fei Hu, fo40225, formath, Fred Reiss, Frederic Bastien, Fredrik Knutsson, G. Hussain Chinoy, Gabriel, gehring, George Grzegorz Pawelczak, Gianluca Varisco, Gleb Popov, Greg Peatfield, Guillaume Klein, Gurpreet Singh, Gustavo Lima Chaves, haison, Haraldur TóMas HallgríMsson, HarikrishnanBalagopal, HåKon Sandsmark, I-Hong, Ilham Firdausi Putra, Imran Salam, Jason Zaman, Jason Zavaglia, jayhpark530, jefby, Jeff Daily, Jeffrey Poznanovic, Jekyll Lai, Jeroen BéDorf, Jerry Shih, jerryyin, jiakai, JiangXIAO, Joe Bowser, Joel Shapiro, Johan Gunnarsson, Jojimon Varghese, Joon, Josh Beal, Julian Niedermeier, Jun Wan, Junqin Zhang, Junyuan Xie, Justin Tunis, Kaixi Hou, Karl Lessard, Karthik Muthuraman, Kbhute-Ibm, khanhlvg, Koock Yoon, kstuedem, Kyuwon Kim, Lakshay Tokas, leike666666, leonard951, Leslie-Fang, Leslie-Fang-Intel, Li, Guizi, Lukas Folle, Lukas Geiger, Mahmoud Abuzaina, Manraj Singh Grover, Margaret Maynard-Reid, Mark Ryan, Matt Conley, Matthew Bentham, Matthew Denton, mbhuiyan, mdfaijul, Mei Jie, merturl, MichaelKonobeev, Michal W. Tarnowski, Milan Straka, minds, mpppk, musikisomorphie, Nagy Mostafa, Nayana Thorat, Neil, Niels Ole Salscheider, Niklas SilfverströM, Niranjan Hasabnis, ocjosen, olramde, Pariksheet Pinjari, Patrick J. Lopresti, Patrik Gustavsson, per1234, PeterLee, Phan Van Nguyen Duc, Phillip Kravtsov, Pooya Davoodi, Pranav Marathe, Putra Manggala, Qingqing Cao, Rajeshwar Reddy T, Ramon ViñAs, Rasmus Diederichsen, Reuben Morais, richardbrks, robert, RonLek, Ryan Jiang, saishruthi, Saket Khandelwal, Saleem Abdulrasool, Sami Kama, Sana-Damani, Sergii Khomenko, Severen Redwood, Shubham Goyal, Sigrid Keydana, Siju Samuel, sleighsoft, smilu97, Son Tran, Srini511, srinivasan.narayanamoorthy, Sumesh Udayakumaran, Sungmann Cho, Tae-Hwan Jung, Taehoon Lee, Takeshi Watanabe, TengLu, terryky, TheMindVirus, ThisIsIsaac, Till Hoffmann, Timothy Liu, Tomer Gafner, Tongxuan Liu, Trent Lo, Trevor Morris, Uday Bondhugula, Vasileios Lioutas, vbvg2008, Vishnuvardhan Janapati, Vivek Suryamurthy, Wei Wang, Wen-Heng (Jack) Chung, wenxizhu, William D. Irons, winstonq, wyzhao, Xiaoming (Jason) Cui, Xinan Jiang, Xinping Wang, Yann-Yy, Yasir Modak, Yong Tang, Yongfeng Gu, Yuchen Ying, Yuxin Wu, zyeric, 王振华 (Zhenhua Wang) # Release 2.0.0 diff --git a/configure.py b/configure.py index ff615a739acd6a..8ec47294b47e88 100644 --- a/configure.py +++ b/configure.py @@ -33,7 +33,7 @@ from distutils.spawn import find_executable as which # pylint: enable=g-import-not-at-top -_DEFAULT_CUDA_VERSION = '10' +_DEFAULT_CUDA_VERSION = '10.1' _DEFAULT_CUDNN_VERSION = '7' _DEFAULT_TENSORRT_VERSION = '6' _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' @@ -817,7 +817,7 @@ def valid_api_level(api_level): android_ndk_api_level = prompt_loop_or_load_from_env( environ_cp, var_name='ANDROID_NDK_API_LEVEL', - var_default='18', # 18 is required for GPU acceleration. + var_default='21', # 21 is required for ARM64 support. ask_for_var=('Please specify the (min) Android NDK API level to use. ' '[Available levels: %s]') % api_levels, check_success=valid_api_level, @@ -1196,6 +1196,34 @@ def set_system_libs_flag(environ_cp): write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) +def is_reduced_optimize_huge_functions_available(environ_cp): + """Check to see if the system supports /d2ReducedOptimizeHugeFunctions. + + The above compiler flag is a new compiler flag introduced to the Visual Studio + compiler in version 16.4 (available in Visual Studio 2019, Preview edition + only, as of 2019-11-19). TensorFlow needs this flag to massively reduce + compile times, but until 16.4 is officially released, we can't depend on it. + + See also https://groups.google.com/a/tensorflow.org/g/build/c/SsW98Eo7l3o + + Because it's very annoying to check this manually (to check the MSVC installed + versions, you need to use the registry, and it's not clear if Bazel will be + using that install version anyway), we expect enviroments who know they may + use this flag to export TF_VC_VERSION=16.4 + + TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS + version is upgraded to 16.4. + + Arguments: + environ_cp: Environment of the current execution + + Returns: + boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this + machine. + """ + return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4 + + def set_windows_build_flags(environ_cp): """Set Windows specific build options.""" # The non-monolithic build is not supported yet @@ -1205,13 +1233,18 @@ def set_windows_build_flags(environ_cp): # Fix winsock2.h conflicts write_to_bazelrc( 'build --copt=-DWIN32_LEAN_AND_MEAN --host_copt=-DWIN32_LEAN_AND_MEAN ' - '--copt=-DNOGDI --host_copt=-DNOGDI') + '--copt=-DNOGDI --host_copt=-DNOGDI --copt=-D_USE_MATH_DEFINES') # Output more verbose information when something goes wrong write_to_bazelrc('build --verbose_failures') # The host and target platforms are the same in Windows build. So we don't # have to distinct them. This avoids building the same targets twice. write_to_bazelrc('build --distinct_host_configuration=false') + if is_reduced_optimize_huge_functions_available(environ_cp): + write_to_bazelrc( + 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions' + ) + if get_var( environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', True, ('Would you like to override eigen strong inline for some C++ ' diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 43585f0ed3e4e1..2ccb9854622282 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -607,6 +607,18 @@ tf_cc_shared_object( ] + tf_additional_binary_deps(), ) +# This is intended to be the same as tf_binary_additional_srcs: +# https://github.com/tensorflow/tensorflow/blob/cd67f4f3723f9165aabedd0171aaadc6290636e5/tensorflow/tensorflow.bzl#L396-L425 +# And is usable in the "deps" attribute instead of the "srcs" attribute +# as a workaround for https://github.com/tensorflow/tensorflow/issues/34117 +cc_import( + name = "libtensorflow_framework_import_lib", + shared_library = select({ + "//tensorflow:macos": ":libtensorflow_framework.dylib", + "//conditions:default": ":libtensorflow_framework.so", + }), +) + # ------------------------------------------- # New rules should be added above this target. # ------------------------------------------- diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py index 56d65d45faf0b1..c515cc76b9aacd 100644 --- a/tensorflow/api_template.__init__.py +++ b/tensorflow/api_template.__init__.py @@ -119,11 +119,11 @@ def _running_from_pip_package(): _current_file_location.startswith(dir_) for dir_ in _site_packages_dirs) if _running_from_pip_package(): - for s in _site_packages_dirs: + for _s in _site_packages_dirs: # TODO(gunan): Add sanity checks to loaded modules here. - plugin_dir = _os.path.join(s, 'tensorflow-plugins') - if _fi.file_exists(plugin_dir): - _ll.load_library(plugin_dir) + _plugin_dir = _os.path.join(_s, 'tensorflow-plugins') + if _fi.file_exists(_plugin_dir): + _ll.load_library(_plugin_dir) # Add module aliases if hasattr(_current_module, 'keras'): @@ -136,3 +136,5 @@ def _running_from_pip_package(): setattr(_current_module, "optimizers", optimizers) setattr(_current_module, "initializers", initializers) # pylint: enable=undefined-variable + +# __all__ PLACEHOLDER diff --git a/tensorflow/api_template_v1.__init__.py b/tensorflow/api_template_v1.__init__.py index 97478a18b8a20a..2b2899c3fe031e 100644 --- a/tensorflow/api_template_v1.__init__.py +++ b/tensorflow/api_template_v1.__init__.py @@ -132,9 +132,10 @@ def _running_from_pip_package(): _current_file_location.startswith(dir_) for dir_ in _site_packages_dirs) if _running_from_pip_package(): - for s in _site_packages_dirs: + for _s in _site_packages_dirs: # TODO(gunan): Add sanity checks to loaded modules here. - plugin_dir = _os.path.join(s, 'tensorflow-plugins') - if _fi.file_exists(plugin_dir): - _ll.load_library(plugin_dir) + _plugin_dir = _os.path.join(_s, 'tensorflow-plugins') + if _fi.file_exists(_plugin_dir): + _ll.load_library(_plugin_dir) +# __all__ PLACEHOLDER diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index e6dfb20166bb2f..bffceab1bc770b 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -264,6 +264,7 @@ tensorflow::Status CreateRemoteContexts( tensorflow::uint64 context_view_id, int keep_alive_secs, const tensorflow::ServerDef& server_def, tensorflow::eager::EagerClientCache* remote_eager_workers, bool async, + const bool lazy_copy_remote_function_inputs, const tensorflow::eager::CreateContextRequest& base_request) { int num_remote_workers = remote_workers.size(); tensorflow::BlockingCounter counter(num_remote_workers); @@ -300,6 +301,8 @@ tensorflow::Status CreateRemoteContexts( request.mutable_server_def()->set_task_index(parsed_name.task); request.set_async(async); request.set_keep_alive_secs(keep_alive_secs); + request.set_lazy_copy_remote_function_inputs( + lazy_copy_remote_function_inputs); eager_client->CreateContextAsync( &request, response, @@ -319,7 +322,7 @@ tensorflow::Status CreateRemoteContexts( tensorflow::Status UpdateRemoteContexts( const std::vector& remote_workers, tensorflow::uint64 context_id, tensorflow::uint64 context_view_id, const tensorflow::ServerDef& server_def, - tensorflow::eager::EagerClientCache* remote_eager_workers, bool async, + tensorflow::eager::EagerClientCache* remote_eager_workers, const tensorflow::eager::CreateContextRequest& base_request) { int num_remote_workers = remote_workers.size(); tensorflow::BlockingCounter counter(num_remote_workers); @@ -527,7 +530,8 @@ tensorflow::Status UpdateTFE_ContextWithServerDef( LOG_AND_RETURN_IF_ERROR(CreateRemoteContexts( remote_workers, context_id, context_view_id, keep_alive_secs, server_def, remote_eager_workers.get(), - ctx->context->Executor().Async(), base_request)); + ctx->context->Executor().Async(), + ctx->context->LazyCopyFunctionRemoteInputs(), base_request)); } else { // The master's context_view_id will be incremented by one // the UpdateRemoteMaster call later. We want all new workers and @@ -537,7 +541,8 @@ tensorflow::Status UpdateTFE_ContextWithServerDef( LOG_AND_RETURN_IF_ERROR(CreateRemoteContexts( added_workers, context_id, context_view_id + 1, keep_alive_secs, server_def, remote_eager_workers.get(), - ctx->context->Executor().Async(), base_request)); + ctx->context->Executor().Async(), + ctx->context->LazyCopyFunctionRemoteInputs(), base_request)); if (!existing_workers.empty()) { if (VLOG_IS_ON(1)) { for (const string& w : existing_workers) { @@ -546,8 +551,7 @@ tensorflow::Status UpdateTFE_ContextWithServerDef( } LOG_AND_RETURN_IF_ERROR(UpdateRemoteContexts( existing_workers, context_id, context_view_id + 1, server_def, - remote_eager_workers.get(), ctx->context->Executor().Async(), - base_request)); + remote_eager_workers.get(), base_request)); } } @@ -713,7 +717,8 @@ TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) { return new TFE_Context(opts->session_options.options, opts->device_placement_policy, opts->mirroring_policy, - opts->async, device_mgr.release(), + opts->async, opts->lazy_remote_inputs_copy, + device_mgr.release(), /*device_mgr_owned*/ true, r, tensorflow::GetDefaultCustomKernelCreator()); } @@ -728,7 +733,8 @@ TFE_Context* TFE_NewContextFromSession(const TFE_ContextOptions* opts, return new TFE_Context(opts->session_options.options, opts->device_placement_policy, opts->mirroring_policy, - opts->async, device_mgr, /*device_mgr_owned*/ false, r, + opts->async, opts->lazy_remote_inputs_copy, device_mgr, + /*device_mgr_owned*/ false, r, tensorflow::GetDefaultCustomKernelCreator()); } diff --git a/tensorflow/c/eager/c_api_experimental.cc b/tensorflow/c/eager/c_api_experimental.cc index a40a435065f0cd..b513fcedc5978c 100644 --- a/tensorflow/c/eager/c_api_experimental.cc +++ b/tensorflow/c/eager/c_api_experimental.cc @@ -557,6 +557,11 @@ extern TFE_ContextMirroringPolicy TFE_ContextGetMirroringPolicy( ctx->context->GetMirroringPolicy()); } +void TFE_ContextOptionsSetLazyRemoteInputsCopy(TFE_ContextOptions* options, + bool lazy_copy) { + options->lazy_remote_inputs_copy = lazy_copy; +} + TFE_CancellationManager* TFE_NewCancellationManager() { return new TFE_CancellationManager; } diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h index 4da08641907815..055f9f9d60240d 100644 --- a/tensorflow/c/eager/c_api_experimental.h +++ b/tensorflow/c/eager/c_api_experimental.h @@ -336,6 +336,10 @@ TF_CAPI_EXPORT extern void TFE_ContextSetThreadLocalMirroringPolicy( TF_CAPI_EXPORT extern TFE_ContextMirroringPolicy TFE_ContextGetMirroringPolicy( TFE_Context*); +// Sets whether to copy the remote inputs of a function lazily. +TF_CAPI_EXPORT extern void TFE_ContextOptionsSetLazyRemoteInputsCopy( + TFE_ContextOptions*, bool lazy_copy); + // ----------------------------------------------------------------------------- // Cancellation APIs. diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h index 1841d4846abf1b..56ee1e01cc9688 100644 --- a/tensorflow/c/eager/c_api_internal.h +++ b/tensorflow/c/eager/c_api_internal.h @@ -57,12 +57,15 @@ struct TFE_ContextOptions { TFE_ContextDevicePlacementPolicy device_placement_policy{ TFE_DEVICE_PLACEMENT_SILENT}; TFE_ContextMirroringPolicy mirroring_policy{TFE_MIRRORING_NONE}; + // If true, lazily copy the remote inputs of a function to the target devices. + bool lazy_remote_inputs_copy = false; }; struct TFE_Context { TFE_Context(const tensorflow::SessionOptions& opts, TFE_ContextDevicePlacementPolicy default_device_placement_policy, TFE_ContextMirroringPolicy default_mirroring_policy, bool async, + const bool lazy_remote_inputs_copy, const tensorflow::DeviceMgr* device_mgr, bool device_mgr_owned, tensorflow::Rendezvous* rendezvous, const tensorflow::CustomKernelCreator* custom_kernel_creator) @@ -72,8 +75,8 @@ struct TFE_Context { default_device_placement_policy), static_cast( default_mirroring_policy), - async, device_mgr, device_mgr_owned, rendezvous, - custom_kernel_creator)) {} + async, lazy_remote_inputs_copy, device_mgr, device_mgr_owned, + rendezvous, custom_kernel_creator)) {} ~TFE_Context() { // TODO(iga): Add a separate API method to shutdown TFE_Context so that we diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 7cac7d78235feb..770352c62c1585 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -1,8 +1,8 @@ load( "//tensorflow:tensorflow.bzl", "tf_cc_test", - "tf_kernel_library", "tf_gen_op_libs", + "tf_kernel_library", ) package( diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index b3c1e6a913a4fa..f67c6f91d6c9e7 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#define _USE_MATH_DEFINES #include #include "tensorflow/cc/ops/array_ops_internal.h" diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 20da141aa20e49..7ebba30b46e518 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -1,4 +1,4 @@ -load("//tensorflow:tensorflow.bzl", "if_mlir", "tf_cc_test", "cc_header_only_library") +load("//tensorflow:tensorflow.bzl", "cc_header_only_library", "if_mlir", "tf_cc_test") load("//tensorflow/stream_executor:build_defs.bzl", "if_cuda_or_rocm") load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library", "tf_jit_compilation_passes_extra_deps") load("//tensorflow/core/platform:default/build_config.bzl", "tf_additional_all_protos", "tf_proto_library") diff --git a/tensorflow/compiler/mlir/tensorflow/BUILD b/tensorflow/compiler/mlir/tensorflow/BUILD index b8a27639e7527b..887e2aca0f10d3 100644 --- a/tensorflow/compiler/mlir/tensorflow/BUILD +++ b/tensorflow/compiler/mlir/tensorflow/BUILD @@ -198,6 +198,7 @@ cc_library( "transforms/bridge_pass.cc", "transforms/cluster_formation.cc", "transforms/cluster_outlining.cc", + "transforms/delete_unused_funcs.cc", "transforms/executor_island_coarsening.cc", "transforms/fold_switch.cc", "transforms/functional_control_flow_to_cfg.cc", @@ -771,6 +772,7 @@ cc_library( ":convert_type", ":error_util", ":tensorflow_passes", + ":tf_dialect_passes", "//tensorflow/compiler/mlir/xla:hlo", "//tensorflow/compiler/mlir/xla:mlir_hlo_to_hlo", "//tensorflow/compiler/mlir/xla:type_to_shape", diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc index 7ce2f39d784c3a..34682175324784 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc @@ -234,5 +234,11 @@ SmallVector GetExportedNames(Operation *op) { return ret; } +bool IsExported(Operation *op) { return !GetExportedNames(op).empty(); } + +bool HasTfSavedModelSemantics(ModuleOp module) { + return module.getAttr("tf_saved_model.semantics") != nullptr; +} + } // namespace tf_saved_model } // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h index 7c67f833191117..b53616f6a13076 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_IR_TF_SAVED_MODEL_H_ #include "mlir/IR/Dialect.h" // TF:local_config_mlir +#include "mlir/IR/Module.h" // TF:local_config_mlir #include "mlir/IR/OpDefinition.h" // TF:local_config_mlir namespace mlir { @@ -41,8 +42,15 @@ class TensorFlowSavedModelDialect : public Dialect { #include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h.inc" // Returns the list of exported names for `op`. +// An empty list means `op` is not exported. SmallVector GetExportedNames(Operation *op); +// Returns true if `op` is exported. +bool IsExported(Operation *op); + +// Returns true if `module` has tf_saved_model linkage semantics. +bool HasTfSavedModelSemantics(ModuleOp module); + } // namespace tf_saved_model } // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_delete_unused_funcs.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_delete_unused_funcs.mlir new file mode 100644 index 00000000000000..9aea3847549fe4 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_delete_unused_funcs.mlir @@ -0,0 +1,96 @@ +// RUN: tf-opt -tf-saved-model-delete-unused-funcs -split-input-file %s | FileCheck %s --dump-input=fail + +module attributes {tf_saved_model.semantics} { + + // Test case: Unused function should be deleted. + + // CHECK-NOT: func @unused + func @unused() { + return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Root calls child. Child should not be deleted. + + // CHECK: func @root + func @root() attributes {tf_saved_model.exported_names = ["root"]} { + "some_dialect.call"() { callee = @child } : () -> () + return + } + + // CHECK: func @child + func @child() { + return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Don't crash if attribute that doesn't reference a func. + + "some_dialect.global_variable"() { sym_name = "some_global" } : () -> () + + func @root2() attributes {tf_saved_model.exported_names = ["root2"]} { + "some_dialect.do_something_with_a_global"() { global = @some_global } : () -> () + return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Delete recursively dead cycle. + + // CHECK-NOT func @recursively_dead0 + func @recursively_dead0() { + "some_dialect.call"() { callee = @recursively_dead1 } : () -> () + return + } + // CHECK-NOT func @recursively_dead1 + func @recursively_dead1() { + "some_dialect.call"() { callee = @recursively_dead0 } : () -> () + return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Root calls child with a deeply nested symbol reference. + // Child should not be deleted. + + // CHECK: func @root + func @root() attributes {tf_saved_model.exported_names = ["root"]} { + "some_dialect.call"() {callee = {callee = {callee = @child}}} : () -> () + return + } + + // CHECK: func @child + func @child() { + return + } + +} + +// ----- + +// Test case: If the module doesn't have tf_saved_model semantics, then this +// pass shouldn't do anything. +module { + // CHECK: func @not_dead() + func @not_dead() { + return + } +} diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/delete_unused_funcs.cc b/tensorflow/compiler/mlir/tensorflow/transforms/delete_unused_funcs.cc new file mode 100644 index 00000000000000..1b76af15dfa991 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/transforms/delete_unused_funcs.cc @@ -0,0 +1,99 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass uses tf_saved_model dialect linkage information to delete +// unused func's. + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "mlir/IR/Module.h" // TF:local_config_mlir +#include "mlir/Pass/Pass.h" // TF:local_config_mlir +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" + +namespace mlir { +namespace tf_saved_model { + +namespace { +struct DeleteUnusedFuncsPass : public ModulePass { + void runOnModule() override; +}; +} // namespace + +void DeleteUnusedFuncsPass::runOnModule() { + // If the model doesn't have tf_saved_model semantics, we can't do anything. + if (!HasTfSavedModelSemantics(getModule())) { + return; + } + + // TODO(silvasean): Use more generic MLIR functionality when available. + // This is just a basic call graph reachability pass (which in the case of TF + // functional control flow also implies handling tf.If/tf.While). + // The only thing specific to tf_saved_model is the set of roots. + + auto module = getModule(); + SymbolTable symbol_table(module); + + // Calculate func reachability with a DFS on the symbol reference graph. + SmallPtrSet dfs_visited_set; + SmallVector dfs_stack; + + // Initialize the roots of the DFS search. + for (auto func : module.getOps()) { + if (IsExported(func)) { + dfs_stack.push_back(func); + } + } + + // Do the DFS. + while (!dfs_stack.empty()) { + FuncOp func = dfs_stack.pop_back_val(); + if (!dfs_visited_set.insert(func).second) { + // If we already visited this node, skip it. + continue; + } + + SmallPtrSet callees; + auto uses = SymbolTable::getSymbolUses(func); + assert(uses.hasValue() && "malformed module"); + for (auto use : *uses) { + auto func = symbol_table.lookup(use.getSymbolRef().getValue()); + if (func) { + callees.insert(func); + } + } + + for (auto callee : callees) { + dfs_stack.push_back(callee); + } + } + + // Erase all unreachable func's. + for (auto func : llvm::make_early_inc_range(module.getOps())) { + if (dfs_visited_set.find(func) == dfs_visited_set.end()) { + func.erase(); + } + } +} + +std::unique_ptr> CreateDeleteUnusedFuncsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "tf-saved-model-delete-unused-funcs", + "Use tf_saved_model linkage information to delete unused func's."); + +} // namespace tf_saved_model +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h index d6fa49e24718af..dbfc7b3c98fa69 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h +++ b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h @@ -110,6 +110,14 @@ void createTPUBridge(OpPassManager& pm); } // namespace TFTPU +namespace tf_saved_model { + +// Creates a pass that uses tf_saved_model dialect linkage information +// to delete unused func's. +std::unique_ptr> CreateDeleteUnusedFuncsPass(); + +} // namespace tf_saved_model + } // namespace mlir #endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_PASSES_H_ diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index 1723e60891d401..7315824964d866 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -1,4 +1,4 @@ -load("//tensorflow:tensorflow.bzl", "tf_cc_test", "cc_header_only_library") +load("//tensorflow:tensorflow.bzl", "cc_header_only_library", "tf_cc_test") load("//tensorflow/compiler/xla:xla.bzl", "xla_proto_library") load( "//tensorflow/core/platform:default/build_config.bzl", diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc index 989968b5cbc889..81c3a874f4ea4d 100644 --- a/tensorflow/compiler/xla/client/lib/math.cc +++ b/tensorflow/compiler/xla/client/lib/math.cc @@ -15,9 +15,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/lib/math.h" -// This macro is required to make MSVC defines math constants in math.h -#define _USE_MATH_DEFINES -#include +#include #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/lib/constants.h" diff --git a/tensorflow/compiler/xla/client/lib/tridiagonal.cc b/tensorflow/compiler/xla/client/lib/tridiagonal.cc index d2ea6d57069fc7..13cc3630137fd9 100644 --- a/tensorflow/compiler/xla/client/lib/tridiagonal.cc +++ b/tensorflow/compiler/xla/client/lib/tridiagonal.cc @@ -36,6 +36,8 @@ namespace { struct TridiagonalSystemShape { const int64 rank; const int64 num_equations; + TridiagonalSystemShape(int64 rk, int64 num_eqs) + : rank(rk), num_equations(num_eqs) {} }; Status CheckSecondToLastDimension(const Shape& op_shape, int64 rank, @@ -109,9 +111,7 @@ StatusOr CheckSystemAndReturnShape(XlaOp lower_diagonal, TF_RETURN_IF_ERROR(CheckSecondToLastDimension(upper_diagonal_shape, rank, 1, "upper diagonal")); - TridiagonalSystemShape result = {.rank = rank, - .num_equations = num_equations}; - return result; + return TridiagonalSystemShape(rank, num_equations); } XlaOp Coefficient(XlaOp operand, int64 i) { diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc index e93e234f3db538..d58c9ce0e47a0b 100644 --- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc +++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc @@ -469,7 +469,7 @@ Status DynamicDimensionInferenceVisitor::HandleSetDimensionSize( // Propagate dynamic dimension indicated by this set dimension size // instruction. parent_->SetDynamicSize(hlo, {}, hlo->dimension(), hlo->mutable_operand(1), - {.stride = 1, .multiple_of = 1}); + DimensionConstraint(1, 1)); } // Also Propagate dynamic dimension already set by operands. @@ -813,7 +813,7 @@ Status DynamicDimensionInferenceVisitor::HandleReshape(HloInstruction* hlo) { parent_->SetDynamicSize( reshape, {}, output_dynamic_dimension, new_dynamic_size, - {.stride = 1, .multiple_of = constraint.multiple_of / divisor}); + DimensionConstraint(1, constraint.multiple_of / divisor)); } if (input_dim_size < output_dim_size) { @@ -850,12 +850,12 @@ Status DynamicDimensionInferenceVisitor::HandleReshape(HloInstruction* hlo) { hlo->parent()->AddInstruction(HloInstruction::CreateBinary( output_dynamic_size->shape(), HloOpcode::kMultiply, new_dynamic_size, operand_dynamic_size)); + int64 new_multiple_of_constraint = + constraint.multiple_of * output_dim_size / + operand->shape().dimensions(input_dynamic_dimension); parent_->SetDynamicSize( reshape, {}, output_dynamic_dimension, new_dynamic_size, - {.stride = 1, - .multiple_of = - constraint.multiple_of * output_dim_size / - operand->shape().dimensions(input_dynamic_dimension)}); + DimensionConstraint(1, new_multiple_of_constraint)); } return Status::OK(); @@ -1227,7 +1227,7 @@ Status DynamicDimensionInferenceVisitor::HandleParameter(HloInstruction* hlo) { parent_->SetDynamicSize(target_parameter, dynamic_dimension.parameter_index, dynamic_dimension.dimension, dynamic_size, - {.stride = 1, .multiple_of = 1}); + DimensionConstraint(1, 1)); return Status::OK(); }); } diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.h b/tensorflow/compiler/xla/service/dynamic_dimension_inference.h index 21808385ec21a8..070127796d6760 100644 --- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.h +++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.h @@ -149,6 +149,9 @@ class DynamicDimensionInference { // // struct DimensionConstraint { + explicit DimensionConstraint(int64 s, int64 m) + : stride(s), multiple_of(m) {} + DimensionConstraint() : stride(1), multiple_of(1) {} // Stride represents the distance of a newly placed element and the previous // placed element on this dynamic dimension. int64 stride; diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index ef58b37b469d7a..46fa47eaf1a970 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -2611,18 +2611,37 @@ struct MinMaxFiniteValue { static double min() { return -max(); } }; +// MSVC's standard C++ library does not define isnan/isfinite for integer types. +// To work around that we will need to provide our own. +template +std::enable_if_t::value, bool> IsFinite(T val) { + return std::isfinite(val); +} +template +std::enable_if_t::value, bool> IsNaN(T val) { + return std::isnan(val); +} +template +std::enable_if_t::value, bool> IsFinite(T val) { + return std::isfinite(static_cast(val)); +} +template +std::enable_if_t::value, bool> IsNaN(T val) { + return std::isnan(static_cast(val)); +} + template bool HloParserImpl::CheckParsedValueIsInRange(LocTy loc, ParsedElemT value) { if (std::is_floating_point::value) { auto value_as_native_t = static_cast(value); auto value_double_converted = static_cast(value_as_native_t); - if (!std::isfinite(value) || std::isfinite(value_double_converted)) { + if (!IsFinite(value) || IsFinite(value_double_converted)) { value = value_double_converted; } } PrimitiveType literal_ty = primitive_util::NativeToPrimitiveType(); - if (std::isnan(value) || + if (IsNaN(value) || (std::numeric_limits::has_infinity && (std::numeric_limits::infinity() == value || -std::numeric_limits::infinity() == value))) { diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 2d61ec49a0bf9a..8369046aa85b57 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -4468,11 +4468,18 @@ tf_cc_test( srcs = ["framework/run_handler_test.cc"], linkstatic = tf_kernel_tests_linkstatic(), deps = [ + ":core_cpu", + ":direct_session_internal", ":framework_internal", ":lib", ":lib_internal", + ":protos_all_cc", + ":tensor_testutil", ":test", ":test_main", + ":testlib", + "//tensorflow/core/kernels:cwise_op", + "//tensorflow/core/kernels:matmul_op", "//third_party/eigen3", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", diff --git a/tensorflow/core/api_def/python_api/api_def_Erfinv.pbtxt b/tensorflow/core/api_def/python_api/api_def_Erfinv.pbtxt new file mode 100644 index 00000000000000..fae017dde2edca --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_Erfinv.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "Erfinv" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_Ndtri.pbtxt b/tensorflow/core/api_def/python_api/api_def_Ndtri.pbtxt new file mode 100644 index 00000000000000..7e6e68ed45da74 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_Ndtri.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "Ndtri" + visibility: HIDDEN +} diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc index 3a0b24f0b2208c..083bcf8d85b21b 100644 --- a/tensorflow/core/common_runtime/eager/context.cc +++ b/tensorflow/core/common_runtime/eager/context.cc @@ -68,13 +68,12 @@ auto* eager_context_created = } // namespace -// TODO(b/134094971): Make lazily_copy_function_remote_inputs_ configurable once -// it's ready to enable. EagerContext::EagerContext( const SessionOptions& opts, ContextDevicePlacementPolicy default_device_placement_policy, ContextMirroringPolicy default_mirroring_policy, bool async, - const DeviceMgr* device_mgr, bool device_mgr_owned, Rendezvous* rendezvous, + const bool lazy_copy_function_remote_inputs, const DeviceMgr* device_mgr, + bool device_mgr_owned, Rendezvous* rendezvous, const CustomKernelCreator* custom_kernel_creator, DistributedFunctionLibraryRuntime* cluster_flr) : default_device_placement_policy_(default_device_placement_policy), @@ -91,7 +90,7 @@ EagerContext::EagerContext( default_executor_(async), log_memory_(LogMemory::IsEnabled()), env_(opts.env), - lazily_copy_function_remote_inputs_(false), + lazy_copy_function_remote_inputs_(lazy_copy_function_remote_inputs), use_send_tensor_rpc_(false), pin_small_ops_to_cpu_(ReadBoolFromEnvVar( "TF_EAGER_ENABLE_SMALL_TENSOR_CPU_PINNING", false)) { @@ -130,7 +129,7 @@ void EagerContext::ResetPFLR(const DeviceMgr* device_mgr, Env* env, thread::ThreadPool* thread_pool, DistributedFunctionLibraryRuntime* cluster_flr, const CustomKernelCreator* custom_kernel_creator) { - if (lazily_copy_function_remote_inputs_) { + if (lazy_copy_function_remote_inputs_) { pflr_.reset(new eager::EagerProcessFunctionLibraryRuntime( device_mgr, env, config, graph_def_version, lib_def, optimizer_options, thread_pool, cluster_flr, custom_kernel_creator)); @@ -164,7 +163,7 @@ void EagerContext::InitDeviceMapAndAsync() { void EagerContext::ResetClusterFLR( DistributedFunctionLibraryRuntime* cluster_flr) { - cluster_flr_.Reset(cluster_flr, lazily_copy_function_remote_inputs_); + cluster_flr_.Reset(cluster_flr, lazy_copy_function_remote_inputs_); } EagerExecutor& EagerContext::Executor() { @@ -239,8 +238,8 @@ bool EagerContext::MirrorTensors() const { return GetMirroringPolicy() == MIRRORING_ALL; } -bool EagerContext::LazilyCopyFunctionRemoteInputs() const { - return lazily_copy_function_remote_inputs_; +bool EagerContext::LazyCopyFunctionRemoteInputs() const { + return lazy_copy_function_remote_inputs_; } #if !defined(IS_MOBILE_PLATFORM) diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h index 1749188ac173af..116c6685c27b7e 100644 --- a/tensorflow/core/common_runtime/eager/context.h +++ b/tensorflow/core/common_runtime/eager/context.h @@ -121,6 +121,7 @@ class EagerContext : public core::RefCounted { EagerContext(const SessionOptions& opts, ContextDevicePlacementPolicy default_device_placement_policy, ContextMirroringPolicy default_mirroring_policy, bool async, + const bool lazy_copy_function_remote_inputs, const DeviceMgr* device_mgr, bool device_mgr_owned, Rendezvous* rendezvous, const CustomKernelCreator* custom_kernel_creator, @@ -168,7 +169,7 @@ class EagerContext : public core::RefCounted { bool MirrorTensors() const; - bool LazilyCopyFunctionRemoteInputs() const; + bool LazyCopyFunctionRemoteInputs() const; bool FindFunctionByName(const string& name); @@ -461,7 +462,7 @@ class EagerContext : public core::RefCounted { // EagerContext owns the DistributedFunctionLibraryRuntime( // EagerClusterFunctionLibraryRuntime) if using EagerService for remote - // function execution (lazily_copy_function_remote_inputs_=true). + // function execution (lazy_copy_function_remote_inputs_=true). OwnedOrUnownedHelper cluster_flr_; // One FunctionLibraryRuntime per device. // func_libs[i] is the FunctionLibraryRuntime corresponding to @@ -553,7 +554,12 @@ class EagerContext : public core::RefCounted { bool is_master_ GUARDED_BY(remote_state_mu_); #endif // IS_MOBILE_PLATFORM - bool lazily_copy_function_remote_inputs_; + // For a multi device function, the target device of each input is unknown + // until the function is instantiated on the default function device. + // If false, eagerly copy all remote inputs to the default function device; + // if true, lazily copy remote inputs to their target devices to avoid + // redundant copies. + bool lazy_copy_function_remote_inputs_ = false; bool use_send_tensor_rpc_; const bool pin_small_ops_to_cpu_; diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index f11a37f204de56..e783aaefdc6133 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -212,7 +212,7 @@ Status ValidateInputTypeAndPlacement( " inputs, got ", n_inputs); } const bool skip_remote_copy = - ctx->LazilyCopyFunctionRemoteInputs() && kernel->IsFunction(); + ctx->LazyCopyFunctionRemoteInputs() && kernel->IsFunction(); for (int i = 0; i < n_inputs; ++i) { TensorHandle* handle = op->Inputs()[i]; Device* expected_device = kernel->InputDevice(i); @@ -499,14 +499,12 @@ Status EagerLocalExecute(EagerOperation* op, TensorHandle** retvals, profiler::TraceMe activity("EagerCopyToDeviceAndAddCacheKey", profiler::TraceMeLevel::kInfo); input_dev_ptrs.reserve(op->Inputs().size()); - // When LazilyCopyFunctionRemoteInputs is disabled, all inputs need to be on + // When LazyCopyFunctionRemoteInputs is disabled, all inputs need to be on // local devices, since we execute a remote function through worker service, // which doesn't accept remote inputs. - // TODO(b/134094971): Make resource_dtypes_and_shapes avaliable without - // remote tensor copy. for (int i = 0; i < op->Inputs().size(); i++) { TensorHandle* input = op->Inputs()[i]; - if (!ctx->LazilyCopyFunctionRemoteInputs() && input->IsRemote()) { + if (!ctx->LazyCopyFunctionRemoteInputs() && input->IsRemote()) { TensorHandle* handle = nullptr; TF_RETURN_IF_ERROR(EagerCopyToDevice( input, ctx, &executor, device == nullptr ? ctx->HostCPU() : device, @@ -603,7 +601,7 @@ Status EagerLocalExecute(EagerOperation* op, TensorHandle** retvals, << ". Full node_def=" << ndef.DebugString(); std::function get_op_id = nullptr; #if !defined(IS_MOBILE_PLATFORM) - if (ctx->LazilyCopyFunctionRemoteInputs()) { + if (ctx->LazyCopyFunctionRemoteInputs()) { get_op_id = [ctx]() { return ctx->RemoteMgr()->NextOpId(); }; } #endif // IS_MOBILE_PLATFORM @@ -750,7 +748,7 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, profiler::TraceMe activity("CopyInputToExpectedDevice", profiler::TraceMeLevel::kInfo); const bool eagerly_copy_function_remote_inputs = - !ctx->LazilyCopyFunctionRemoteInputs() || !op->is_function(); + !ctx->LazyCopyFunctionRemoteInputs() || !op->is_function(); for (int i = 0; i < op->Inputs().size(); i++) { tensorflow::TensorHandle* input = op->Inputs()[i]; tensorflow::Device* input_device = input->device(); @@ -834,12 +832,12 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, } } - if (ctx->LazilyCopyFunctionRemoteInputs()) { + if (ctx->LazyCopyFunctionRemoteInputs()) { // Store the data type and shape of a remote resource variable on the // corresponding remote TensorHandle (output of 'VarHandleOp'). // If the variable is an input of a remote function, the function may need // the type and shape during function instantiation. When - // LazilyCopyFunctionRemoteInputs is enabled, we no longer copy the resource + // LazyCopyFunctionRemoteInputs is enabled, we no longer copy the resource // handle (contains the type and shape) of the variable to the default // function device. Instead, we store the type and shape on eager master // and sent them to the default function device along with the diff --git a/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc b/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc index b9db52c023d4e7..42494a1f2e5745 100644 --- a/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc +++ b/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc @@ -87,7 +87,7 @@ REGISTER_REWRITE(EagerOpRewriteRegistry::PRE_EXECUTION, MklEagerOpRewrite); // Constructor MklEagerOpRewrite::MklEagerOpRewrite(string name, string file, string line) - : EagerOpRewrite(name, file, line) { + : EagerOpRewrite(name, file, line), registered_kernels_map_() { InsertMKLEagerOps({"BatchMatMul", AlwaysRewrite, CreateGenericMklOp}); InsertMKLEagerOps({"BatchMatMulV2", AlwaysRewrite, CreateGenericMklOp}); InsertMKLEagerOps({"Conv2D", RewriteConv2D, CreateMklConv2DOp}); diff --git a/tensorflow/core/debug/BUILD b/tensorflow/core/debug/BUILD index ccdf478fabfb59..10e7c17abdc062 100644 --- a/tensorflow/core/debug/BUILD +++ b/tensorflow/core/debug/BUILD @@ -15,10 +15,10 @@ load( "//tensorflow:tensorflow.bzl", "check_deps", + "tf_cc_binary", "tf_cc_test", "tf_copts", "tf_cuda_library", - "tf_cc_binary", ) # For platform specific build config diff --git a/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc b/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc index c221d76aafab67..a1cfe5813f14cf 100644 --- a/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc +++ b/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc @@ -216,7 +216,7 @@ void EagerClusterFunctionLibraryRuntime::CleanUp( DistributedFunctionLibraryRuntime* CreateClusterFLR( const uint64 context_id, EagerContext* ctx, WorkerSession* worker_session) { - if (ctx->LazilyCopyFunctionRemoteInputs()) { + if (ctx->LazyCopyFunctionRemoteInputs()) { return new EagerClusterFunctionLibraryRuntime( context_id, ctx, worker_session->remote_device_mgr()); } else { diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc index 083aeefbf7b21c..92e3d2fb3cf860 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc @@ -162,8 +162,8 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request, tensorflow::EagerContext* ctx = new tensorflow::EagerContext( opts, tensorflow::ContextDevicePlacementPolicy::DEVICE_PLACEMENT_SILENT, tensorflow::ContextMirroringPolicy::MIRRORING_NONE, request->async(), - device_mgr, false, r, GetDefaultCustomKernelCreator(), - worker_session->cluster_flr()); + request->lazy_copy_remote_function_inputs(), device_mgr, false, r, + GetDefaultCustomKernelCreator(), worker_session->cluster_flr()); // Ownership will be transferred to the ServerContext, or else in an error // case ctx will be deleted by this unref. core::ScopedUnref unref_ctx(ctx); diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc index 8b8fe42502acdc..dbf3c6370bce2b 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc @@ -710,8 +710,9 @@ TEST_F(EagerServiceImplTest, RequestsToMasterTest) { tensorflow::EagerContext* ctx = new tensorflow::EagerContext( SessionOptions(), tensorflow::ContextDevicePlacementPolicy::DEVICE_PLACEMENT_SILENT, - tensorflow::ContextMirroringPolicy::MIRRORING_NONE, false, - device_mgr_.get(), false, rendezvous, GetDefaultCustomKernelCreator()); + tensorflow::ContextMirroringPolicy::MIRRORING_NONE, /*async=*/false, + /*lazy_copy_function_remote_inputs=*/false, device_mgr_.get(), false, + rendezvous, GetDefaultCustomKernelCreator()); const uint64 context_id = random::New64(); // Set RemoteMgr to ctx. diff --git a/tensorflow/core/distributed_runtime/eager/remote_mgr_test.cc b/tensorflow/core/distributed_runtime/eager/remote_mgr_test.cc index 7b68100b54366a..6bb4943ffeeb53 100644 --- a/tensorflow/core/distributed_runtime/eager/remote_mgr_test.cc +++ b/tensorflow/core/distributed_runtime/eager/remote_mgr_test.cc @@ -55,9 +55,9 @@ class RemoteMgrTest : public ::testing::Test { ctx_ = new tensorflow::EagerContext( SessionOptions(), tensorflow::ContextDevicePlacementPolicy::DEVICE_PLACEMENT_SILENT, - tensorflow::ContextMirroringPolicy::MIRRORING_NONE, false, - device_mgr.release(), true, rendezvous, GetDefaultCustomKernelCreator(), - nullptr); + tensorflow::ContextMirroringPolicy::MIRRORING_NONE, /*async=*/false, + /*lazy_copy_function_remote_inputs=*/false, device_mgr.release(), true, + rendezvous, GetDefaultCustomKernelCreator(), nullptr); } ~RemoteMgrTest() override { ctx_->Unref(); } diff --git a/tensorflow/core/framework/dataset.cc b/tensorflow/core/framework/dataset.cc index fc6f8fdbb9012a..f27fa75eb7db7f 100644 --- a/tensorflow/core/framework/dataset.cc +++ b/tensorflow/core/framework/dataset.cc @@ -403,6 +403,7 @@ Status DatasetBaseIterator::GetNext(IteratorContext* ctx, bool* end_of_sequence) { profiler::TraceMe activity([&] { return BuildTraceMeName(); }, profiler::TraceMeLevel::kInfo); + DVLOG(3) << prefix() << " GetNext enter"; RecordStart(ctx, /*stop_output=*/true); Status s = GetNextInternal(ctx, out_tensors, end_of_sequence); if (s.ok() && !*end_of_sequence) RecordElement(ctx); @@ -415,6 +416,7 @@ Status DatasetBaseIterator::GetNext(IteratorContext* ctx, s.error_message()); LOG(ERROR) << s; } + DVLOG(3) << prefix() << " GetNext exit"; return s; } diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h index 5a1fe974094a8e..ecb85a85ac139d 100644 --- a/tensorflow/core/framework/dataset.h +++ b/tensorflow/core/framework/dataset.h @@ -815,9 +815,13 @@ class DatasetBaseIterator : public IteratorBase { explicit DatasetBaseIterator(const BaseParams& params) : params_(params) { params_.dataset->Ref(); + VLOG(2) << prefix() << " constructor"; } - ~DatasetBaseIterator() override { params_.dataset->Unref(); } + ~DatasetBaseIterator() override { + VLOG(2) << prefix() << " destructor"; + params_.dataset->Unref(); + } const DataTypeVector& output_dtypes() const override { return params_.dataset->output_dtypes(); diff --git a/tensorflow/core/framework/run_handler.cc b/tensorflow/core/framework/run_handler.cc index 2fcbf3807ab171..448082b99e175b 100644 --- a/tensorflow/core/framework/run_handler.cc +++ b/tensorflow/core/framework/run_handler.cc @@ -98,6 +98,55 @@ class RunHandlerEnvironment { typedef typename RunHandlerEnvironment::Task Task; typedef Eigen::RunQueue Queue; +// To reduce cache misses, we use a doubly-linked list of Waiter structs and +// queue them in LIFO order rather than the FIFO order used by a single +// condition variable. +struct Waiter { + Waiter() { + next = this; + prev = this; + } + condition_variable cv; + mutex mu; + Waiter* next; + Waiter* prev; +}; + +void WaitOnWaiter(Waiter* waiter, Waiter* queue_head, mutex* mutex, + int max_sleep_micros) { + { + mutex_lock l(*mutex); + CHECK_EQ(waiter->next, waiter); // Crash OK. + CHECK_EQ(waiter->prev, waiter); // Crash OK. + + // Add waiter to the LIFO queue + waiter->prev = queue_head; + waiter->next = queue_head->next; + waiter->next->prev = waiter; + waiter->prev->next = waiter; + } + { + mutex_lock l(waiter->mu); + // Wait on the condition variable + waiter->cv.wait_for(l, std::chrono::microseconds(max_sleep_micros)); + } + + mutex_lock l(*mutex); + // Remove waiter from the LIFO queue. Note even when a waiter wakes up due + // to a notification we cannot conclude the waiter is not in the queue. + // This is due to the fact that a thread preempted right before notifying + // may resume after a waiter got re-added. + if (waiter->next != waiter) { + CHECK(waiter->prev != waiter); // Crash OK. + waiter->next->prev = waiter->prev; + waiter->prev->next = waiter->next; + waiter->next = waiter; + waiter->prev = waiter; + } else { + CHECK_EQ(waiter->prev, waiter); // Crash OK. + } +} + class ThreadWorkSource { public: ThreadWorkSource() @@ -155,11 +204,32 @@ class ThreadWorkSource { if (max_rank_to_wakeup > 0 && rank_.load(std::memory_order_relaxed) <= max_rank_to_wakeup) { Waiter* w = nullptr; + bool use_sub_thread_pool = ParamFromEnvBoolWithDefault( + "TF_RUN_HANDLER_USE_SUB_THREAD_POOL", false); + + Waiter* waiter_queue; + mutex* waiter_queue_mu; + if (use_sub_thread_pool) { + // When we use multiple sub thread pools, free threads wait on sub + // thread pool waiting queues. Wake up threads from sub thread waiting + // queues. + // The waiting queues are defined at RunHandlerPool. + // Get the waiter_queue and coresponding mutex. Note, the thread work + // source may change afterwards if a new request comes or an old request + // finishes. + tf_shared_lock lock(run_handler_waiter_mu_); + waiter_queue = sub_thread_pool_waiter_; + waiter_queue_mu = sub_thread_pool_waiter_mu_; + } else { + waiter_queue = &queue_waiters_; + waiter_queue_mu = &waiters_mu_; + } + { - mutex_lock l(waiters_mu_); - if (queue_waiters_.next != &queue_waiters_) { + mutex_lock l(*waiter_queue_mu); + if (waiter_queue->next != waiter_queue) { // Remove waiter from the LIFO queue - w = queue_waiters_.next; + w = waiter_queue->next; CHECK(w->prev != w); CHECK(w->next != w); @@ -187,43 +257,25 @@ class ThreadWorkSource { Task PopBlockingTask() { return blocking_work_queue_.PopBack(); } - Task PopNonBlockingTask(int index) { - return non_blocking_work_queues_[index]->queue.PopBack(); + Task PopNonBlockingTask(int start_index, bool search_from_all_queue) { + Task t; + unsigned sharding_factor = NonBlockingWorkShardingFactor(); + for (unsigned j = 0; j < sharding_factor; ++j) { + t = non_blocking_work_queues_[(start_index + j) % sharding_factor] + ->queue.PopBack(); + if (t.f) { + return t; + } + if (!search_from_all_queue) { + break; + } + } + return t; } void WaitForWork(int max_sleep_micros) { thread_local Waiter waiter; - { - mutex_lock l(waiters_mu_); - CHECK_EQ(waiter.next, &waiter); - CHECK_EQ(waiter.prev, &waiter); - - // Add waiter to the LIFO queue - waiter.prev = &queue_waiters_; - waiter.next = queue_waiters_.next; - waiter.next->prev = &waiter; - waiter.prev->next = &waiter; - } - { - mutex_lock l(waiter.mu); - // Wait on the condition variable - waiter.cv.wait_for(l, std::chrono::microseconds(max_sleep_micros)); - } - - mutex_lock l(waiters_mu_); - // Remove waiter from the LIFO queue. Note even when a waiter wakes up due - // to a notification we cannot conclude the waiter is not in the queue. - // This is due to the fact that a thread preempted right before notifying - // may resume after a waiter got re-added. - if (waiter.next != &waiter) { - CHECK(waiter.prev != &waiter); - waiter.next->prev = waiter.prev; - waiter.prev->next = waiter.next; - waiter.next = &waiter; - waiter.prev = &waiter; - } else { - CHECK_EQ(waiter.prev, &waiter); - } + WaitOnWaiter(&waiter, &queue_waiters_, &waiters_mu_, max_sleep_micros); } int TaskQueueSize(bool is_blocking) { @@ -243,6 +295,12 @@ class ThreadWorkSource { void SetTracemeId(int64 value) { traceme_id_ = value; } void SetRank(int64 value) { rank_ = value; } + void SetWaiter(Waiter* waiter, mutex* mutex) { + mutex_lock l(run_handler_waiter_mu_); + sub_thread_pool_waiter_ = waiter; + sub_thread_pool_waiter_mu_ = mutex; + } + int64 GetInflightTaskCount(bool is_blocking) { std::atomic* counter = is_blocking ? &blocking_inflight_ : &non_blocking_inflight_; @@ -274,20 +332,6 @@ class ThreadWorkSource { } private: - // To reduce cache misses, we use a doubly-linked list of Waiter structs and - // queue them in LIFO order rather than the FIFO order used by a single - // condition variable. - struct Waiter { - Waiter() { - next = this; - prev = this; - } - condition_variable cv; - mutex mu; - Waiter* next; - Waiter* prev; - }; - struct NonBlockingQueue { mutex queue_op_mu; char pad[128]; @@ -307,6 +351,10 @@ class ThreadWorkSource { Waiter queue_waiters_ GUARDED_BY(waiters_mu_); std::atomic traceme_id_; std::atomic rank_; + + mutex run_handler_waiter_mu_; + mutex* sub_thread_pool_waiter_mu_ GUARDED_BY(run_handler_waiter_mu_); + Waiter* sub_thread_pool_waiter_ GUARDED_BY(run_handler_waiter_mu_); }; class RunHandlerThreadPool { @@ -319,25 +367,33 @@ class RunHandlerThreadPool { RunHandlerThreadPool(int num_blocking_threads, int num_non_blocking_threads, Env* env, const ThreadOptions& thread_options, - const string& name) + const string& name, + Eigen::MaxSizeVector* waiters_mu, + Eigen::MaxSizeVector* queue_waiters) : num_threads_(num_blocking_threads + num_non_blocking_threads), num_blocking_threads_(num_blocking_threads), num_non_blocking_threads_(num_non_blocking_threads), thread_data_(num_threads_), env_(env, thread_options, name), - name_(name) { + name_(name), + waiters_mu_(waiters_mu), + queue_waiters_(queue_waiters), + use_sub_thread_pool_(ParamFromEnvBoolWithDefault( + "TF_RUN_HANDLER_USE_SUB_THREAD_POOL", false)), + num_threads_in_sub_thread_pool_(ParamFromEnvWithDefault( + "TF_RUN_HANDLER_NUM_THREADS_IN_SUB_THREAD_POOL", + std::vector( + {num_blocking_threads / 2, + num_blocking_threads - num_blocking_threads / 2}))), + sub_thread_pool_start_request_percentage_(ParamFromEnvWithDefault( + "TF_RUN_HANDLER_SUB_THREAD_POOL_START_REQUEST_PERCENTAGE", + std::vector({0, 0.4}))), + sub_thread_pool_end_request_percentage_(ParamFromEnvWithDefault( + "TF_RUN_HANDLER_SUB_THREAD_POOL_END_REQUEST_PERCENTAGE", + std::vector({0.4, 1}))) { VLOG(1) << "Creating RunHandlerThreadPool " << name << " with " << num_blocking_threads_ << " blocking threads and " << num_non_blocking_threads_ << " non-blocking threads."; - cancelled_ = false; - - thread_data_.resize(num_threads_); - for (int i = 0; i < num_threads_; i++) { - thread_data_[i].thread.reset( - env_.CreateThread([this, i, num_blocking_threads]() { - WorkerLoop(i, i < num_blocking_threads); - })); - } } ~RunHandlerThreadPool() { @@ -353,6 +409,26 @@ class RunHandlerThreadPool { } } + void Start() { + cancelled_ = false; + thread_data_.resize(num_threads_); + int num_blocking_threads = num_blocking_threads_; + for (int i = 0; i < num_threads_; i++) { + int sub_thread_pool_id = num_threads_in_sub_thread_pool_.size() - 1; + for (int j = 0; j < num_threads_in_sub_thread_pool_.size(); ++j) { + if (i < num_threads_in_sub_thread_pool_[j]) { + sub_thread_pool_id = j; + break; + } + } + thread_data_[i].sub_thread_pool_id = sub_thread_pool_id; + thread_data_[i].thread.reset( + env_.CreateThread([this, i, num_blocking_threads]() { + WorkerLoop(i, i < num_blocking_threads); + })); + } + } + void AddWorkToQueue(ThreadWorkSource* tws, bool is_blocking, std::function fn) { Task t = env_.CreateTask(std::move(fn)); @@ -384,30 +460,37 @@ class RunHandlerThreadPool { return; } thread_data_[tid].thread_work_sources.resize(0); - thread_data_[tid].thread_work_sources.emplace_back( - thread_work_sources[start_request_idx]); - // The number of shards for the queue. Threads in each shard will prioritize - // different thread_work_sources. Increase the number of shards could - // decrease the contention in the queue. - // For example, when num_shards == 1: - // thread_work_sources are ordered as start_request_idx, 0, 1, 2, 3, 4 ... - // for all threads. - // When num_shards == 2: - // thread_work_sources are order as start_request_idx, 0, 2, 4 ... 1, 3, - // 5... for half of the threads and start_request_idx, 1, 3, 5 ... 0, 2, - // 4... for the other half of the threads. - int num_shards = ParamFromEnvWithDefault("TF_RUN_HANDLER_QUEUE_SHARDS", 1); - int token = tid % num_shards; - for (int i = 0; i < num_shards; ++i) { - for (int j = token; j < thread_work_sources.size(); j += num_shards) { - if (j != start_request_idx) { - thread_data_[tid].thread_work_sources.emplace_back( - thread_work_sources[j]); + + if (use_sub_thread_pool_) { + for (int i = 0; i < thread_work_sources.size(); ++i) { + thread_data_[tid].thread_work_sources.emplace_back( + thread_work_sources[i]); + } + } else { + thread_data_[tid].thread_work_sources.emplace_back( + thread_work_sources[start_request_idx]); + // The number of shards for the queue. Threads in each shard will + // prioritize different thread_work_sources. Increase the number of shards + // could decrease the contention in the queue. For example, when + // num_shards == 1: thread_work_sources are ordered as start_request_idx, + // 0, 1, 2, 3, 4 ... for all threads. When num_shards == 2: + // thread_work_sources are order as start_request_idx, 0, 2, 4 ... 1, 3, + // 5... for half of the threads and start_request_idx, 1, 3, 5 ... 0, 2, + // 4... for the other half of the threads. + int num_shards = + ParamFromEnvWithDefault("TF_RUN_HANDLER_QUEUE_SHARDS", 1); + int token = tid % num_shards; + for (int i = 0; i < num_shards; ++i) { + for (int j = token; j < thread_work_sources.size(); j += num_shards) { + if (j != start_request_idx) { + thread_data_[tid].thread_work_sources.emplace_back( + thread_work_sources[j]); + } } + token = (token + 1) % num_shards; } - token = (token + 1) % num_shards; + thread_data_[tid].sources_not_empty.notify_all(); } - thread_data_[tid].sources_not_empty.notify_all(); } PerThread* GetPerThread() { @@ -434,13 +517,26 @@ class RunHandlerThreadPool { void WorkerLoop(int thread_id, bool may_steal_blocking_work); + // Search tasks from Requets range searching_range_start to + // searching_range_end. If there is no tasks in the search range and + // may_steal_blocking_work is true, then search from all reuqests. + Task FindTask( + int searching_range_start, int searching_range_end, int thread_id, + int sub_thread_pool_id, int max_blocking_inflight, + bool may_steal_blocking_work, + const Eigen::MaxSizeVector& thread_work_sources, + bool* task_from_blocking_queue, ThreadWorkSource** tws); + void WaitForWork(bool is_blocking, int thread_id, int32 max_blocking_inflight); + void WaitForWorkInSubThreadPool(bool is_blocking, int sub_thread_pool_id); + private: struct ThreadData { ThreadData() : version(0), + current_index(0), thread_work_sources(static_cast( ParamFromEnvWithDefault("TF_RUN_HANDLER_MAX_CONCURRENT_HANDLERS", kMaxConcurrentHandlers))) {} @@ -448,7 +544,9 @@ class RunHandlerThreadPool { uint64 version; condition_variable sources_not_empty; std::unique_ptr thread; + int current_index; Eigen::MaxSizeVector thread_work_sources GUARDED_BY(mu); + int sub_thread_pool_id; }; const int num_threads_; @@ -458,8 +556,58 @@ class RunHandlerThreadPool { RunHandlerEnvironment env_; std::atomic cancelled_; string name_; + Eigen::MaxSizeVector* waiters_mu_; + Eigen::MaxSizeVector* queue_waiters_; + + bool use_sub_thread_pool_; + std::vector num_threads_in_sub_thread_pool_; + + // Threads in each sub thread pool will search tasks from the given + // start_request_percentage to end_request_percentage in a round robin + // fashion. + std::vector sub_thread_pool_start_request_percentage_; + std::vector sub_thread_pool_end_request_percentage_; }; +Task RunHandlerThreadPool::FindTask( + int searching_range_start, int searching_range_end, int thread_id, + int sub_thread_pool_id, int max_blocking_inflight, + bool may_steal_blocking_work, + const Eigen::MaxSizeVector& thread_work_sources, + bool* task_from_blocking_queue, ThreadWorkSource** tws) { + Task t; + int current_index = thread_data_[thread_id].current_index; + *task_from_blocking_queue = false; + + // TODO(chaox): Chagne the search algorithm from round robin to random + // walk. + for (int i = 0; i < searching_range_end - searching_range_start; ++i) { + if (current_index >= searching_range_end) { + current_index = searching_range_start; + } + *tws = thread_work_sources[current_index]; + ++current_index; + + // For blocking thread, search for blocking tasks first. + if (may_steal_blocking_work && + (*tws)->GetInflightTaskCount(true) < max_blocking_inflight) { + t = (*tws)->PopBlockingTask(); + if (t.f) { + *task_from_blocking_queue = true; + break; + } + } + + // Search for non-blocking tasks. + t = (*tws)->PopNonBlockingTask(thread_id, true); + if (t.f) { + break; + } + } + thread_data_[thread_id].current_index = current_index; + return t; +} + // Main worker thread loop. void RunHandlerThreadPool::WorkerLoop(int thread_id, bool may_steal_blocking_work) { @@ -474,11 +622,47 @@ void RunHandlerThreadPool::WorkerLoop(int thread_id, bool task_from_blocking_queue = true; Eigen::MaxSizeVector* thread_work_sources = &thread_data_[thread_id].thread_work_sources; - { + int sub_thread_pool_id; + if (use_sub_thread_pool_) { + // The mutex is not hot since its per thread and can only be held + // by some other thread when a session run starts/finishes. + mutex_lock l(thread_data_[thread_id].mu); + sub_thread_pool_id = thread_data_[thread_id].sub_thread_pool_id; + int active_requests = thread_work_sources->size(); + if (may_steal_blocking_work) { + // Each thread will first look for tasks from requests that belongs to + // its sub thread pool. + t = FindTask( + active_requests * + sub_thread_pool_start_request_percentage_[sub_thread_pool_id], + active_requests * + sub_thread_pool_end_request_percentage_[sub_thread_pool_id], + thread_id, sub_thread_pool_id, kMaxBlockingInflight, + /*may_steal_blocking_work=*/true, *thread_work_sources, + &task_from_blocking_queue, &tws); + if (!t.f) { + // Search from all requests if the thread cannot find tasks from + // requests that belong to its own sub thread pool. + t = FindTask(0, active_requests, thread_id, sub_thread_pool_id, + kMaxBlockingInflight, + /*may_steal_blocking_work=*/true, *thread_work_sources, + &task_from_blocking_queue, &tws); + } + } else { + // For non-blocking threads, it will always search from all pending + // requests. + t = FindTask(0, active_requests, thread_id, sub_thread_pool_id, + kMaxBlockingInflight, + /*may_steal_blocking_work=*/false, *thread_work_sources, + &task_from_blocking_queue, &tws); + } + } else { // The mutex is not hot since its per thread and can only be held // by some other thread when a session run starts/finishes. mutex_lock l(thread_data_[thread_id].mu); + // TODO(chaox): Refactor the following code to share the logic with + // FindTask. for (int i = 0; i < thread_work_sources->size(); ++i) { tws = (*thread_work_sources)[i]; // We want a smallish numbers of inter threads since @@ -495,20 +679,16 @@ void RunHandlerThreadPool::WorkerLoop(int thread_id, // Always look for any work from the "primary" work source. // This way when we wake up a thread for a new closure we are // guaranteed it can be worked on. - for (int j = 0; j < tws->NonBlockingWorkShardingFactor(); ++j) { - t = tws->PopNonBlockingTask((j + thread_id) % - tws->NonBlockingWorkShardingFactor()); - if (t.f) { - task_from_blocking_queue = false; - break; - } + t = tws->PopNonBlockingTask(thread_id, true); + if (t.f) { + task_from_blocking_queue = false; + break; } if (t.f) { break; } } else { - t = tws->PopNonBlockingTask(thread_id % - tws->NonBlockingWorkShardingFactor()); + t = tws->PopNonBlockingTask(thread_id, false); if (t.f) { task_from_blocking_queue = false; break; @@ -542,12 +722,30 @@ void RunHandlerThreadPool::WorkerLoop(int thread_id, << (*thread_work_sources)[i]->ToString(); } } - - WaitForWork(may_steal_blocking_work, thread_id, kMaxBlockingInflight); + if (use_sub_thread_pool_) { + WaitForWorkInSubThreadPool(may_steal_blocking_work, sub_thread_pool_id); + } else { + WaitForWork(may_steal_blocking_work, thread_id, kMaxBlockingInflight); + } } } } +void RunHandlerThreadPool::WaitForWorkInSubThreadPool(bool is_blocking, + int sub_thread_pool_id) { + const int kMaxSleepMicros = 250; + + // The non-blocking thread will just sleep. + if (!is_blocking) { + Env::Default()->SleepForMicroseconds(kMaxSleepMicros); + return; + } + + thread_local Waiter waiter; + WaitOnWaiter(&waiter, &(*queue_waiters_)[sub_thread_pool_id], + &(*waiters_mu_)[sub_thread_pool_id], kMaxSleepMicros); +} + void RunHandlerThreadPool::WaitForWork(bool is_blocking, int thread_id, int32 max_blocking_inflight) { const int kMaxSleepMicros = 250; @@ -636,16 +834,33 @@ class RunHandlerPool::Impl { explicit Impl(int num_inter_op_threads, int num_intra_op_threads) : max_handlers_(static_cast(ParamFromEnvWithDefault( "TF_RUN_HANDLER_MAX_CONCURRENT_HANDLERS", kMaxConcurrentHandlers))), + waiters_mu_( + ParamFromEnvWithDefault("TF_RUN_HANDLER_NUM_SUB_THREAD_POOL", 2)), + queue_waiters_( + ParamFromEnvWithDefault("TF_RUN_HANDLER_NUM_SUB_THREAD_POOL", 2)), run_handler_thread_pool_(new RunHandlerThreadPool( num_inter_op_threads, num_intra_op_threads, Env::Default(), - ThreadOptions(), "tf_run_handler_pool")), + ThreadOptions(), "tf_run_handler_pool", &waiters_mu_, + &queue_waiters_)), iterations_(0), - version_(0) { + version_(0), + sub_thread_pool_end_request_percentage_(ParamFromEnvWithDefault( + "TF_RUN_HANDLER_SUB_THREAD_POOL_END_REQUEST_PERCENTAGE", + std::vector({1}))) { VLOG(1) << "Creating a RunHandlerPool with max handlers: " << max_handlers_; for (int i = 0; i < max_handlers_; ++i) { handlers_.emplace_back(new RunHandler::Impl(this)); free_handlers_.push_back(handlers_.back().get()); } + queue_waiters_.resize( + ParamFromEnvWithDefault("TF_RUN_HANDLER_NUM_SUB_THREAD_POOL", 2)); + waiters_mu_.resize( + ParamFromEnvWithDefault("TF_RUN_HANDLER_NUM_SUB_THREAD_POOL", 2)); + for (auto& queue_waiter : queue_waiters_) { + queue_waiter.next = &queue_waiter; + queue_waiter.prev = &queue_waiter; + } + run_handler_thread_pool_->Start(); } ~Impl() { @@ -693,6 +908,19 @@ class RunHandlerPool::Impl { for (int i = 0; i < num_active_requests; ++i) { (*thread_work_sources)[i] = sorted_active_handlers_[i]->tws(); (*thread_work_sources)[i]->SetRank(i); + int sub_thread_pool_id = + sub_thread_pool_end_request_percentage_.size() - 1; + for (int j = 0; j < sub_thread_pool_end_request_percentage_.size(); + ++j) { + if (i < num_active_requests * + sub_thread_pool_end_request_percentage_[j]) { + sub_thread_pool_id = j; + break; + } + } + (*thread_work_sources)[i]->SetWaiter( + &queue_waiters_[sub_thread_pool_id], + &waiters_mu_[sub_thread_pool_id]); } version = ++version_; } @@ -738,6 +966,19 @@ class RunHandlerPool::Impl { for (int i = 0; i < num_active_requests; ++i) { (*thread_work_sources)[i] = sorted_active_handlers_[i]->tws(); (*thread_work_sources)[i]->SetRank(i); + int sub_thread_pool_id = + sub_thread_pool_end_request_percentage_.size() - 1; + for (int j = 0; j < sub_thread_pool_end_request_percentage_.size(); + ++j) { + if (i < num_active_requests * + sub_thread_pool_end_request_percentage_[j]) { + sub_thread_pool_id = j; + break; + } + } + (*thread_work_sources)[i]->SetWaiter( + &queue_waiters_[sub_thread_pool_id], + &waiters_mu_[sub_thread_pool_id]); } version = ++version_; LogInfo(); @@ -759,6 +1000,9 @@ class RunHandlerPool::Impl { // inference). const int max_handlers_; + Eigen::MaxSizeVector waiters_mu_; + Eigen::MaxSizeVector queue_waiters_; + std::unique_ptr run_handler_thread_pool_; // Thread compatible part used only by lock under RunHandlerPool. // Handlers are sorted by start time. @@ -773,6 +1017,7 @@ class RunHandlerPool::Impl { condition_variable one_handler_free_; mutex mu_; int64 version_ GUARDED_BY(mu_); + const std::vector sub_thread_pool_end_request_percentage_; }; void RunHandlerPool::Impl::RecomputePoolStats( diff --git a/tensorflow/core/framework/run_handler_test.cc b/tensorflow/core/framework/run_handler_test.cc index 263ef16796ff6a..71b1fbc8d8d00d 100644 --- a/tensorflow/core/framework/run_handler_test.cc +++ b/tensorflow/core/framework/run_handler_test.cc @@ -24,11 +24,17 @@ limitations under the License. #include "absl/memory/memory.h" #include "absl/synchronization/barrier.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/testlib.h" #include "tensorflow/core/lib/core/blocking_counter.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow/core/public/session_options.h" namespace tensorflow { namespace { @@ -72,5 +78,132 @@ TEST(RunHandlerUtilTest, TestBasicScheduling) { counter.Wait(); } +SessionOptions DefaultSessionOptions() { + SessionOptions options; + (*options.config.mutable_device_count())["CPU"] = 2; + return options; +} + +std::unique_ptr CreateSession() { + return std::unique_ptr(NewSession(DefaultSessionOptions())); +} + +class RunHandlerTest : public ::testing::Test { + public: + void Initialize(std::initializer_list a_values) { + Graph graph(OpRegistry::Global()); + + Tensor a_tensor(DT_FLOAT, TensorShape({2, 2})); + test::FillValues(&a_tensor, a_values); + Node* a = test::graph::Constant(&graph, a_tensor); + a->set_assigned_device_name("/job:localhost/replica:0/task:0/cpu:0"); + a_ = a->name(); + + Tensor x_tensor(DT_FLOAT, TensorShape({2, 1})); + test::FillValues(&x_tensor, {1, 1}); + Node* x = test::graph::Constant(&graph, x_tensor); + x->set_assigned_device_name("/job:localhost/replica:0/task:0/cpu:1"); + x_ = x->name(); + + // y = A * x + Node* y = test::graph::Matmul(&graph, a, x, false, false); + y->set_assigned_device_name("/job:localhost/replica:0/task:0/cpu:0"); + y_ = y->name(); + + Node* y_neg = test::graph::Unary(&graph, "Neg", y); + y_neg_ = y_neg->name(); + y_neg->set_assigned_device_name("/job:localhost/replica:0/task:0/cpu:1"); + + Node* z = test::graph::Unary(&graph, "Identity", y_neg); + z_ = z->name(); + z->set_assigned_device_name("/job:localhost/replica:0/task:0/cpu:1"); + + graph.ToGraphDef(&def_); + + ASSERT_EQ(setenv("TF_RUN_HANDLER_NUM_SUB_THREAD_POOL", "2", true), 0); + ASSERT_EQ( + setenv("TF_RUN_HANDLER_NUM_THREADS_IN_SUB_THREAD_POOL", "8,8", true), + 0); + ASSERT_EQ(setenv("TF_RUN_HANDLER_SUB_THREAD_POOL_START_REQUEST_PERCENTAGE", + "0,0.4", true), + 0); + ASSERT_EQ(setenv("TF_RUN_HANDLER_SUB_THREAD_POOL_END_REQUEST_PERCENTAGE", + "0.4,1", true), + 0); + ASSERT_EQ(setenv("TF_NUM_INTEROP_THREADS", "16", true), 0); + } + + string a_; + string x_; + string y_; + string y_neg_; + string z_; + GraphDef def_; +}; + +TEST_F(RunHandlerTest, UseRunHandlerPoolEnableSubPool) { + Initialize({3, 2, -1, 0}); + auto session = CreateSession(); + ASSERT_TRUE(session != nullptr); + EXPECT_EQ(::tensorflow::Status::OK(), session->Create(def_)); + std::vector> inputs; + + // Request two targets: one fetch output and one non-fetched output. + std::vector output_names = {y_ + ":0"}; + std::vector target_nodes = {y_neg_}; + std::vector outputs; + + // Prepares RunOptions and RunMetadata + RunOptions run_options; + run_options.mutable_experimental()->set_use_run_handler_pool(true); + + Status s = session->Run(run_options, inputs, output_names, target_nodes, + &outputs, nullptr); + EXPECT_EQ(::tensorflow::Status::OK(), s); + + ASSERT_EQ(1, outputs.size()); + // The first output should be initialized and have the correct + // output. + auto mat = outputs[0].matrix(); + ASSERT_TRUE(outputs[0].IsInitialized()); + EXPECT_FLOAT_EQ(5.0, mat(0, 0)); +} + +TEST_F(RunHandlerTest, TestConcurrencyUseRunHandlerPool) { + Initialize({1, 2, 3, 4}); + auto session = CreateSession(); + ASSERT_TRUE(session != nullptr); + EXPECT_EQ(::tensorflow::Status::OK(), session->Create(def_)); + + RunOptions run_options; + run_options.mutable_experimental()->set_use_run_handler_pool(true); + + // Fill in the input and ask for the output + thread::ThreadPool* tp = new thread::ThreadPool(Env::Default(), "test", 4); + + // Run the graph 1000 times in 4 different threads concurrently. + std::vector output_names = {y_ + ":0"}; + auto fn = [&session, output_names, run_options]() { + for (int i = 0; i < 1000; ++i) { + std::vector> inputs; + std::vector outputs; + // Run the graph + Status s = session->Run(run_options, inputs, output_names, {}, &outputs, + nullptr); + EXPECT_EQ(::tensorflow::Status::OK(), s); + ASSERT_EQ(1, outputs.size()); + auto mat = outputs[0].matrix(); + EXPECT_FLOAT_EQ(3.0, mat(0, 0)); + } + }; + + for (int i = 0; i < 4; ++i) { + tp->Schedule(fn); + } + + // Wait for the functions to finish. + delete tp; +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/framework/run_handler_util.cc b/tensorflow/core/framework/run_handler_util.cc index ebdc670a92514c..c832a64338527d 100644 --- a/tensorflow/core/framework/run_handler_util.cc +++ b/tensorflow/core/framework/run_handler_util.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/str_util.h" namespace tensorflow { @@ -29,6 +30,54 @@ double ParamFromEnvWithDefault(const std::string& var_name, return (val && strings::safe_strtod(val, &num)) ? num : default_value; } +std::vector ParamFromEnvWithDefault(const std::string& var_name, + std::vector default_value) { + const char* val = std::getenv(var_name.c_str()); + if (!val) { + return default_value; + } + std::vector splits = str_util::Split(val, ","); + std::vector result; + result.reserve(splits.size()); + for (auto& split : splits) { + double num; + if (strings::safe_strtod(split, &num)) { + result.push_back(num); + } else { + LOG(ERROR) << "Wrong format for " << var_name << ". Use default value."; + return default_value; + } + } + return result; +} + +std::vector ParamFromEnvWithDefault(const std::string& var_name, + std::vector default_value) { + const char* val = std::getenv(var_name.c_str()); + if (!val) { + return default_value; + } + std::vector splits = str_util::Split(val, ","); + std::vector result; + result.reserve(splits.size()); + for (auto& split : splits) { + int num; + if (strings::safe_strto32(split, &num)) { + result.push_back(num); + } else { + LOG(ERROR) << "Wrong format for " << var_name << ". Use default value."; + return default_value; + } + } + return result; +} + +bool ParamFromEnvBoolWithDefault(const std::string& var_name, + bool default_value) { + const char* val = std::getenv(var_name.c_str()); + return (val) ? str_util::Lowercase(val) == "true" : default_value; +} + void ComputeInterOpSchedulingRanges(int num_active_requests, int num_threads, int min_threads_per_request, std::vector* start_vec, diff --git a/tensorflow/core/framework/run_handler_util.h b/tensorflow/core/framework/run_handler_util.h index 864e6e698fc9a8..982f06fb7e0686 100644 --- a/tensorflow/core/framework/run_handler_util.h +++ b/tensorflow/core/framework/run_handler_util.h @@ -54,10 +54,27 @@ void ComputeInterOpStealingRanges(int num_threads, int min_threads_per_domain, std::vector ChooseRequestsWithExponentialDistribution( int num_active_requests, int num_threads); -// Loop environment variable named 'var_name' and return the value if it exist -// and can be parsed. Return 'default_value' otherwise. +// Look up environment variable named 'var_name' and return the value if it +// exist and can be parsed. Return 'default_value' otherwise. double ParamFromEnvWithDefault(const std::string& var_name, double default_value); +// Look up environment variable named 'var_name' and return the value if it +// exist and can be parsed. The value must be in format val1,val2... Return +// 'default_value' otherwise. +std::vector ParamFromEnvWithDefault(const std::string& var_name, + std::vector default_value); + +// Look up environment variable named 'var_name' and return the value if it +// exist and can be parsed. The value must be in format val1,val2... Return +// 'default_value' otherwise. +std::vector ParamFromEnvWithDefault(const std::string& var_name, + std::vector default_value); + +// Look up environment variable named 'var_name' and return the value if it +// exist and can be parsed. Return 'default_value' otherwise. +bool ParamFromEnvBoolWithDefault(const std::string& var_name, + bool default_value); + } // end namespace tensorflow #endif // TENSORFLOW_CORE_FRAMEWORK_RUN_HANDLER_UTIL_H_ diff --git a/tensorflow/core/framework/run_handler_util_test.cc b/tensorflow/core/framework/run_handler_util_test.cc index 7f85118671cde4..769991920d14b3 100644 --- a/tensorflow/core/framework/run_handler_util_test.cc +++ b/tensorflow/core/framework/run_handler_util_test.cc @@ -124,5 +124,44 @@ TEST(RunHandlerUtilTest, TestExponentialRequestDistribution) { ASSERT_EQ(actual_distribution, expected_distribution); } +TEST(RunHandlerUtilTest, TestParamFromEnvWithDefault) { + std::vector result = ParamFromEnvWithDefault( + "RUN_HANDLER_TEST_ENV", std::vector{0, 0, 0}); + EXPECT_EQ(result.size(), 3); + EXPECT_EQ(result[0], 0); + EXPECT_EQ(result[1], 0); + EXPECT_EQ(result[2], 0); + + std::vector result2 = ParamFromEnvWithDefault("RUN_HANDLER_TEST_ENV", + std::vector{0, 0, 0}); + EXPECT_EQ(result2.size(), 3); + EXPECT_EQ(result2[0], 0); + EXPECT_EQ(result2[1], 0); + EXPECT_EQ(result2[2], 0); + + bool result3 = + ParamFromEnvBoolWithDefault("RUN_HANDLER_TEST_ENV_BOOL", false); + EXPECT_EQ(result3, false); + + // Set environment variable. + EXPECT_EQ(setenv("RUN_HANDLER_TEST_ENV", "1,2,3", true), 0); + result = ParamFromEnvWithDefault("RUN_HANDLER_TEST_ENV", + std::vector{0, 0, 0}); + EXPECT_EQ(result.size(), 3); + EXPECT_EQ(result[0], 1); + EXPECT_EQ(result[1], 2); + EXPECT_EQ(result[2], 3); + result2 = ParamFromEnvWithDefault("RUN_HANDLER_TEST_ENV", + std::vector{0, 0, 0}); + EXPECT_EQ(result.size(), 3); + EXPECT_EQ(result2[0], 1); + EXPECT_EQ(result2[1], 2); + EXPECT_EQ(result2[2], 3); + + EXPECT_EQ(setenv("RUN_HANDLER_TEST_ENV_BOOL", "true", true), 0); + result3 = ParamFromEnvBoolWithDefault("RUN_HANDLER_TEST_ENV_BOOL", false); + EXPECT_EQ(result3, true); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc index d1e7b9b2d5dbe6..579be7302ccdb0 100644 --- a/tensorflow/core/kernels/data/dataset_utils.cc +++ b/tensorflow/core/kernels/data/dataset_utils.cc @@ -18,6 +18,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "tensorflow/core/framework/dataset.h" #include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/op_def_builder.h" #include "tensorflow/core/framework/op_def_util.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" @@ -33,6 +34,26 @@ namespace { constexpr char kDelimiter[] = "@@"; +// clang-format off +constexpr std::array kOpsWithSeed = { + "AnonymousRandomSeedGenerator", + "ShuffleDataset", + "ShuffleAndRepeatDataset" +}; +// clang-format on + +constexpr char kSeedInputName[] = "seed"; +constexpr char kSeed2InputName[] = "seed2"; + +template +bool IsNodeOfType(const NodeDef& node, + const std::array& op_types) { + for (const auto& type : op_types) { + if (node.op() == type) return true; + } + return false; +} + Status FindNode(const GraphDef& graph, const string& name, const NodeDef** result) { for (const auto& node : graph.node()) { @@ -134,6 +155,30 @@ Status HashNodeImpl(const GraphDef& graph, const NodeDef& node, uint64* hash, for (int i = 0; i < node.input_size(); ++i) { DCHECK_GT(node.input(i).length(), 0); + + // We skip trying to take the hash of the seeds of any ops, as they + // are irrelevant to the hash of the graph and may vary from run to run. + if (IsNodeOfType(node, kOpsWithSeed)) { + const OpRegistrationData* reg; + auto status = OpRegistry::Global()->LookUp(node.op(), ®); + + if (status.ok()) { + if (reg->op_def.input_arg_size() > i) { + const std::string input_arg_name = reg->op_def.input_arg(i).name(); + if (input_arg_name == kSeedInputName || + input_arg_name == kSeed2InputName) { + continue; + } + } + } else if (errors::IsNotFound(status)) { + LOG(WARNING) << "Cannot find " << node.op() + << " in global op registry, so cannot determine which " + "inputs are seeds."; + } else { + return status; + } + } + if (node.input(i)[0] == '^') { // TODO(frankchn): Investigate if control dependencies are necessary // inputs to the hash. Control dependency node names start with '^', and diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc index 0582b4239a5048..f016f72be6b110 100644 --- a/tensorflow/core/kernels/data/dataset_utils_test.cc +++ b/tensorflow/core/kernels/data/dataset_utils_test.cc @@ -328,6 +328,63 @@ TEST_F(DatasetHashUtilsTest, HashNodeDifferentGraphs) { EXPECT_NE(hash1, hash2); } +TEST_F(DatasetHashUtilsTest, HashSameGraphDifferentSeeds) { + GraphDef gd; + + NodeDef* n1 = gd.add_node(); + TF_CHECK_OK(NodeDefBuilder("graph_1/node_1", "Const") + .Attr("value", 1) + .Device("CPU:0") + .Finalize(n1)); + + NodeDef* seed = gd.add_node(); + TF_CHECK_OK(NodeDefBuilder("graph_1/seed", "Const") + .Attr("value", 123) + .Device("CPU:0") + .Finalize(seed)); + + NodeDef* seed2 = gd.add_node(); + TF_CHECK_OK(NodeDefBuilder("graph_1/seed2", "Const") + .Attr("value", 456) + .Device("CPU:0") + .Finalize(seed2)); + + NodeDef* range_ds = gd.add_node(); + TF_CHECK_OK(NodeDefBuilder("graph_1/range", "RangeDataset") + .Input(n1->name(), 0, DT_INT64) + .Input(n1->name(), 0, DT_INT64) + .Input(n1->name(), 0, DT_INT64) + .Device("CPU:0") + .Finalize(range_ds)); + + NodeDef* shuffle_ds = gd.add_node(); + TF_CHECK_OK(NodeDefBuilder("graph_1/shuffle", "ShuffleDataset") + .Input(range_ds->name(), 0, DT_VARIANT) + .Input(n1->name(), 0, DT_INT64) + .Input(seed->name(), 0, DT_INT64) + .Input(seed2->name(), 0, DT_INT64) + .Device("CPU:0") + .Finalize(shuffle_ds)); + + uint64 hash1 = GetHash(gd, *shuffle_ds); + + seed->Clear(); + seed2->Clear(); + + TF_CHECK_OK(NodeDefBuilder("graph_1/seed", "Const") + .Attr("value", 789) + .Device("CPU:0") + .Finalize(seed)); + TF_CHECK_OK(NodeDefBuilder("graph_1/seed2", "Const") + .Attr("value", 654) + .Device("CPU:0") + .Finalize(seed2)); + + uint64 hash2 = GetHash(gd, *shuffle_ds); + + EXPECT_EQ(hash1, hash2); +} + TEST_F(DatasetHashUtilsTest, HashNodeReversedOrder) { GraphDef gd; diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD index 9cd0e926ea06ed..961d6d52cf1312 100644 --- a/tensorflow/core/kernels/data/experimental/BUILD +++ b/tensorflow/core/kernels/data/experimental/BUILD @@ -184,6 +184,7 @@ tf_kernel_library( "//tensorflow/core:core_cpu_internal", "//tensorflow/core:dataset_ops_op_lib", "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "//tensorflow/core:nn_ops_op_lib", diff --git a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc index b9fb85ce7bf772..1dffff217a0ff6 100644 --- a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc @@ -87,14 +87,15 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel { Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, bool* end_of_sequence) override { + Status s; { tf_shared_lock l(mu_); if (!input_impl_) { *end_of_sequence = true; return Status::OK(); } - Status s = input_impl_->GetNext(ctx, out_tensors, end_of_sequence); - while (!s.ok()) { + s = input_impl_->GetNext(ctx, out_tensors, end_of_sequence); + while (!s.ok() && !errors::IsCancelled(s)) { out_tensors->clear(); s = input_impl_->GetNext(ctx, out_tensors, end_of_sequence); } @@ -103,7 +104,7 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel { mutex_lock l(mu_); input_impl_.reset(); } - return Status::OK(); + return s; } protected: diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc index f765cffcd90d14..6fbf153e9d1a67 100644 --- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/common_runtime/input_colocation_exemption_registry.h" #include "tensorflow/core/common_runtime/metrics.h" +#include "tensorflow/core/framework/model.h" #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/stats_aggregator.h" #include "tensorflow/core/framework/tensor.h" @@ -170,9 +171,12 @@ class MapAndBatchDatasetOp::Dataset : public DatasetBase { num_parallel_calls_(std::make_shared( params.dataset->num_parallel_calls_, mu_, cond_var_)), max_batch_results_( - std::min(kMaxBatchResults, (params.dataset->num_parallel_calls_ + - params.dataset->batch_size_ - 1) / - params.dataset->batch_size_)) {} + params.dataset->num_parallel_calls_ == model::kAutotune + ? kMaxBatchResults + : std::min(kMaxBatchResults, + (params.dataset->num_parallel_calls_ + + params.dataset->batch_size_ - 1) / + params.dataset->batch_size_)) {} ~Iterator() override { mutex_lock l(*mu_); diff --git a/tensorflow/core/kernels/data/iterator_ops.h b/tensorflow/core/kernels/data/iterator_ops.h index a24132e97ccb86..4f90b79265d11f 100644 --- a/tensorflow/core/kernels/data/iterator_ops.h +++ b/tensorflow/core/kernels/data/iterator_ops.h @@ -43,7 +43,11 @@ class IteratorResource : public ResourceBase { iterator_state_(std::make_shared( std::move(flib_def), std::move(pflr), flr, /*iterator=*/nullptr)), output_dtypes_(output_dtypes), - output_shapes_(output_shapes) {} + output_shapes_(output_shapes) { + VLOG(2) << "constructor"; + } + + ~IteratorResource() override { VLOG(2) << "destructor"; } Status GetNext(OpKernelContext* ctx, std::vector* out_tensors, bool* end_of_sequence); diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc index 9e1237c133b362..d64c623565e356 100644 --- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc +++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc @@ -261,7 +261,6 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { if (num_parallel_calls_->value == model::kAutotune) { num_parallel_calls_->value = dataset()->cycle_length_; } - last_valid_current_element_ = dataset()->cycle_length_ - 1; ctx_ = std::make_unique(*ctx); TF_RETURN_IF_ERROR( dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_)); @@ -437,10 +436,12 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { if (!initial_elements_created_) { for (int i = 0; i < dataset()->cycle_length_; ++i) { current_elements_[i] = MakeElement(); - if (current_elements_[i]) { - current_elements_[i]->cycle_index = i; - elements_to_process_.push_back(i); + if (!current_elements_[i]) { + break; } + current_elements_[i]->cycle_index = i; + elements_to_process_.push_back(i); + last_valid_current_element_ = i; } initial_elements_created_ = true; } @@ -457,8 +458,9 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { // Advances the position in the interleave cycle to the next cycle // element. void AdvanceToNextInCycle() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + DCHECK_NE(last_valid_current_element_, -1); block_index_ = 0; - cycle_index_ = (cycle_index_ + 1) % dataset()->cycle_length_; + cycle_index_ = (cycle_index_ + 1) % (last_valid_current_element_ + 1); } // Advances the position in the interleave cycle by one. @@ -494,6 +496,10 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { bool ConsumeHelper(std::shared_ptr* result) EXCLUSIVE_LOCKS_REQUIRED(mu_) { while (true) { + if (last_valid_current_element_ == -1) { + // Reached end of input. + return true; + } for (int64 i = 0; i < (last_valid_current_element_ + 1); ++i) { int64 index = (cycle_index_ + i) % (last_valid_current_element_ + 1); if (current_elements_[index]) { @@ -504,10 +510,7 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { break; } } - if (!current_elements_[cycle_index_]) { - // Reached end of input. - return true; - } + DCHECK(current_elements_[cycle_index_]); std::shared_ptr element = current_elements_[cycle_index_]; if (!element->results.empty()) { // We found a result. @@ -551,9 +554,16 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { while (last_valid_current_element_ >= 0 && !current_elements_[last_valid_current_element_]) { last_valid_current_element_--; + if (cycle_index_ > last_valid_current_element_) { + // We are about to move the cycle index below in + // AdvanceToNextInCycle(). + cycle_index_ = last_valid_current_element_; + } } } - AdvanceToNextInCycle(); + if (last_valid_current_element_ != -1) { + AdvanceToNextInCycle(); + } } } @@ -1152,7 +1162,7 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { // TODO(aaudibert): Generalize this optimization by removing null elements // from `current_elements_`, e.g. by compacting the vector when x% of // its elements are null. - int64 last_valid_current_element_ GUARDED_BY(mu_); + int64 last_valid_current_element_ GUARDED_BY(mu_) = -1; const int per_iterator_prefetch_; const int future_elements_prefetch_; @@ -1208,6 +1218,8 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { // Identifies position in the interleave cycle. int64 block_index_ GUARDED_BY(mu_) = 0; + // It is an invariant that either `last_valid_current_element_ == -1` or + // `cycle_index_ <= last_valid_current_element_`. int64 cycle_index_ GUARDED_BY(mu_) = 0; // Elements of the current interleave cycle. diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op_test.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op_test.cc index 8b0bd0ce316d92..6517cac7799f94 100644 --- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op_test.cc +++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op_test.cc @@ -364,6 +364,29 @@ ParallelInterleaveDatasetParams ParallelInterleaveDatasetParams10() { /*node_name=*/kNodeName); } +ParallelInterleaveDatasetParams LongCycleDeteriministicParams() { + auto tensor_slice_dataset_params = TensorSliceDatasetParams( + /*components=*/{CreateTensor( + TensorShape{3, 3, 1}, {"a", "b", "c", "d", "e", "f", "g", "h", "i"})}, + /*node_name=*/"tensor_slice"); + return ParallelInterleaveDatasetParams( + tensor_slice_dataset_params, + /*other_arguments=*/{}, + /*cycle_length=*/11, + /*block_length=*/1, + /*num_parallel_calls=*/2, + /*func=*/ + MakeTensorSliceDatasetFunc( + DataTypeVector({DT_STRING}), + std::vector({PartialTensorShape({1})})), + /*func_lib=*/{test::function::MakeTensorSliceDataset()}, + /*type_arguments=*/{}, + /*output_dtypes=*/{DT_STRING}, + /*output_shapes=*/{PartialTensorShape({1})}, + /*sloppy=*/false, + /*node_name=*/kNodeName); +} + // test case 11: cycle_length = 0, block_length = 1, num_parallel_calls = 2, // sloppy = true ParallelInterleaveDatasetParams @@ -504,7 +527,14 @@ GetNextTestCases() { CreateTensors( TensorShape{1}, {{"a"}, {"b"}, {"c"}, {"d"}, {"e"}, {"f"}, {"g"}, {"h"}, {"i"}}), - /*compare_order=*/false}}; + /*compare_order=*/false}, + {/*dataset_params=*/ + LongCycleDeteriministicParams(), + /*expected_outputs=*/ + CreateTensors( + TensorShape{1}, + {{"a"}, {"d"}, {"g"}, {"b"}, {"e"}, {"h"}, {"c"}, {"f"}, {"i"}}), + /*compare_order=*/true}}; } ITERATOR_GET_NEXT_TEST_P(ParallelInterleaveDatasetOpTest, diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc index 2a2c33296a2486..1658f0a63f2925 100644 --- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc +++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc @@ -318,6 +318,9 @@ class ParallelMapIterator : public DatasetBaseIterator { cond_var_->wait(l); RecordStart(ctx); } + if (cancelled_) { + return errors::Cancelled("Iterator was cancelled"); + } } RecordStop(ctx); result->notification.WaitForNotification(); @@ -555,6 +558,9 @@ class ParallelMapIterator : public DatasetBaseIterator { // false, `result` will point to the result. bool ShouldWait(std::shared_ptr* result) EXCLUSIVE_LOCKS_REQUIRED(*mu_) { + if (cancelled_) { + return false; + } if (sloppy_) { for (auto it = invocation_results_.begin(); it != invocation_results_.end(); ++it) { diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc index 389b1cb856e738..097f3cdc688e87 100644 --- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc +++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc @@ -192,8 +192,7 @@ class PrefetchDatasetOp::Dataset : public DatasetBase { } if (cancelled_) { - return errors::Cancelled( - "PrefetchDatasetOp::Dataset::Iterator::GetNext"); + return errors::Cancelled("Iterator was cancelled"); } if (!buffer_.empty()) { @@ -209,14 +208,17 @@ class PrefetchDatasetOp::Dataset : public DatasetBase { } mutex_lock parent_l(*parent_mu_); - mutex_lock l(*mu_); - if (stats_aggregator) { - stats_aggregator->AddScalar( - stats_utils::BufferSizeScalarName(dataset()->node_name()), - static_cast(buffer_.size()), num_elements()); - stats_aggregator->AddScalar( - stats_utils::BufferCapacityScalarName(dataset()->node_name()), - static_cast(buffer_limit()), num_elements()); + { + mutex_lock l(*mu_); + if (stats_aggregator) { + stats_aggregator->AddScalar( + stats_utils::BufferSizeScalarName(dataset()->node_name()), + static_cast(buffer_.size()), num_elements()); + stats_aggregator->AddScalar( + stats_utils::BufferCapacityScalarName(dataset()->node_name()), + static_cast(buffer_limit()), num_elements()); + } + // Release mu_ } return input_impl_->GetNext(ctx, out_tensors, end_of_sequence); } @@ -478,6 +480,7 @@ class PrefetchDatasetOp::Dataset : public DatasetBase { // This mutex is used to ensure exclusivity between multiple threads // reading/writing this iterator's local state. + // Note: We should never call GetNext on the input while holding this. const std::shared_ptr mu_; // This mutex is used to ensure exclusivity between multiple threads // accessing the parent iterator. We keep this separate from `mu_` to diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc index 6f3b939bac5db4..674467abedfa5b 100644 --- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc +++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc @@ -54,6 +54,7 @@ const int64 kLogIntervalMicros = 10 * 1000000; // 10 seconds. const int64 kMaxEpochsInBuffer = 3; constexpr char kNumRandomSamples[] = "num_random_samples"; +constexpr char kDataProduced[] = "data_produced"; constexpr char kEndOfInputSequence[] = "end_of_input_sequence"; constexpr char kEpoch[] = "epoch"; constexpr char kNumElements[] = "num_elements"; @@ -138,9 +139,7 @@ class ShuffleDatasetOpBase::ShuffleDatasetBase : public DatasetBase { mutex_lock l(mu_); int64 start_micros = ctx->env()->NowMicros(); int64 num_log_entries = 0; - bool first_call = false; if (!input_impl_ && epoch_ == 0) { - first_call = true; TF_RETURN_IF_ERROR(this->dataset()->input_->MakeIterator( ctx, this->prefix(), &input_impl_)); } @@ -158,13 +157,12 @@ class ShuffleDatasetOpBase::ShuffleDatasetBase : public DatasetBase { TF_RETURN_IF_ERROR(input_impl_->GetNext(ctx, &input_element, &end_of_input_sequence)); if (!end_of_input_sequence) { - first_call = false; + data_produced_ = true; break; } - if (first_call && this->dataset()->count_ == -1) { - // If the first call to GetNext() fails because the end - // of sequence has been reached, we terminate the - // iteration immediately. (Otherwise, this iterator + if (!data_produced_ && this->dataset()->count_ == -1) { + // If we encounter the end of sequence without producing data, we + // terminate the iteration immediately. (Otherwise, this iterator // would loop infinitely and never produce a value.) *end_of_sequence = true; return Status::OK(); @@ -289,6 +287,10 @@ class ShuffleDatasetOpBase::ShuffleDatasetBase : public DatasetBase { } } } + if (data_produced_) { + TF_RETURN_IF_ERROR( + writer->WriteScalar(this->full_name(kDataProduced), "")); + } return Status::OK(); } @@ -353,6 +355,7 @@ class ShuffleDatasetOpBase::ShuffleDatasetBase : public DatasetBase { } } } + data_produced_ = reader->Contains(this->full_name(kDataProduced)); return Status::OK(); } @@ -394,6 +397,7 @@ class ShuffleDatasetOpBase::ShuffleDatasetBase : public DatasetBase { random::SingleSampleAdapter generator_ GUARDED_BY(mu_); int64 num_random_samples_ GUARDED_BY(mu_) = 0; + bool data_produced_ GUARDED_BY(mu_) = false; }; const DatasetBase* const input_; diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index f5a037f8f2988f..d23f84fc059b61 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -24,8 +24,8 @@ limitations under the License. #include #include -#include "mkldnn.hpp" #include "absl/strings/str_join.h" +#include "mkldnn.hpp" #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" @@ -570,17 +570,15 @@ class MklConvOp : public OpKernel { OP_REQUIRES(context, dilations_.size() == 5, errors::InvalidArgument("Dilation rates field must " "specify 5 dimensions")); - OP_REQUIRES(context, - (GetTensorDim(dilations_, data_format_, 'N') == 1 && - GetTensorDim(dilations_, data_format_, 'C') == 1), + OP_REQUIRES(context, (GetTensorDim(dilations_, data_format_, 'N') == 1 && + GetTensorDim(dilations_, data_format_, 'C') == 1), errors::InvalidArgument( "Current implementation does not yet support " "dilations rates in the batch and depth dimensions.")); OP_REQUIRES( - context, - (GetTensorDim(dilations_, data_format_, '0') > 0 && - GetTensorDim(dilations_, data_format_, '1') > 0 && - GetTensorDim(dilations_, data_format_, '2') > 0), + context, (GetTensorDim(dilations_, data_format_, '0') > 0 && + GetTensorDim(dilations_, data_format_, '1') > 0 && + GetTensorDim(dilations_, data_format_, '2') > 0), errors::InvalidArgument("Dilated rates should be larger than 0.")); } } @@ -1350,7 +1348,7 @@ class MklFusedConvOp } else if (fused_ops == std::vector{"Relu6"}) { this->set_fuse_activation(true, ALGORITHM::eltwise_bounded_relu, 6.0); } else if (fused_ops == std::vector{"Elu"}) { - this->set_fuse_activation(true, ALGORITHM::eltwise_elu); + this->set_fuse_activation(true, ALGORITHM::eltwise_elu, 1.0); } else if (fused_ops == std::vector{"BiasAdd", "Relu"}) { this->set_fuse_biasadd(true); this->set_fuse_activation(true, ALGORITHM::eltwise_relu); @@ -1365,7 +1363,7 @@ class MklFusedConvOp "Fused Conv2D must have one extra argument: bias.")); } else if (fused_ops == std::vector{"BiasAdd", "Elu"}) { this->set_fuse_biasadd(true); - this->set_fuse_activation(true, ALGORITHM::eltwise_elu); + this->set_fuse_activation(true, ALGORITHM::eltwise_elu, 1.0); OP_REQUIRES(context, num_args == 1, errors::InvalidArgument( "Fused Conv2D must have one extra argument: bias.")); @@ -1395,7 +1393,7 @@ class MklFusedConvOp } else if (fused_ops == std::vector{"BiasAdd", "Add", "Elu"}) { this->set_fuse_biasadd(true); this->set_fuse_add(true); - this->set_fuse_activation(true, ALGORITHM::eltwise_elu); + this->set_fuse_activation(true, ALGORITHM::eltwise_elu, 1.0); OP_REQUIRES( context, num_args == 2, errors::InvalidArgument( diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc index 863e4ea0d3ab99..a92a4f38f54dae 100644 --- a/tensorflow/core/kernels/mkl_fused_ops_test.cc +++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc @@ -110,14 +110,14 @@ class CommonTestUtilities : public OpsTestBase { DataType dtype = DataTypeToEnum::v(); Tensor image(dtype, {image_batch_count, image_height, image_width, depth}); - image.flat() = image.flat().setRandom(); + image.flat() = image.flat().template setRandom(); Tensor filter(dtype, {filter_size, filter_size, depth, filter_count}); - filter.flat() = filter.flat().setRandom(); + filter.flat() = filter.flat().template setRandom(); const int bias_size = filter_count; Tensor bias(dtype, {bias_size}); - bias.flat() = bias.flat().setRandom(); + bias.flat() = bias.flat().template setRandom(); Tensor conv_2d; Tensor fused_conv_2d; @@ -140,14 +140,14 @@ class CommonTestUtilities : public OpsTestBase { DataType dtype = DataTypeToEnum::v(); Tensor image(dtype, {image_batch_count, image_height, image_width, depth}); - image.flat() = image.flat().setRandom(); + image.flat() = image.flat().template setRandom(); Tensor filter(dtype, {filter_size, filter_size, depth, filter_count}); - filter.flat() = filter.flat().setRandom(); + filter.flat() = filter.flat().template setRandom(); const int bias_size = filter_count; Tensor bias(dtype, {bias_size}); - bias.flat() = bias.flat().setRandom(); + bias.flat() = bias.flat().template setRandom(); Tensor conv_2d; Tensor fused_conv_2d; @@ -168,13 +168,13 @@ class CommonTestUtilities : public OpsTestBase { DataType dtype = DataTypeToEnum::v(); Tensor input(dtype, {batch, depth}); - input.flat() = input.flat().setRandom(); + input.flat() = input.flat().template setRandom(); Tensor weight(dtype, {depth, weight_count}); - weight.flat() = weight.flat().setRandom(); + weight.flat() = weight.flat().template setRandom(); Tensor bias(dtype, {weight_count}); - bias.flat() = bias.flat().setRandom(); + bias.flat() = bias.flat().template setRandom(); Tensor output; Tensor fused_output; @@ -187,6 +187,9 @@ class CommonTestUtilities : public OpsTestBase { test::ExpectClose(output, fused_output, 1e-5); } + + private: + using random_gen_ = Eigen::internal::NormalRandomGenerator; }; // Testing MKL's fused convolution ops @@ -242,7 +245,7 @@ class MklFusedConv2DOpTest : public OpsTestBase { if (std::find(fused_ops.begin(), fused_ops.end(), "Elu") != fused_ops.end()) { last_op = "with_elu"; - next_op = ops::Relu(root.WithOpName(last_op), next_op); + next_op = ops::Elu(root.WithOpName(last_op), next_op); } CommonTestUtilities::RunAndFetch(root, last_op, output); diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index 3490dc1ee80475..467087b786423d 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -670,7 +670,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -682,7 +684,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); @@ -849,7 +853,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -861,7 +867,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); @@ -1340,7 +1348,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -1352,7 +1362,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); @@ -1456,7 +1468,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -1468,7 +1482,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); @@ -2957,7 +2973,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -2969,7 +2987,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); @@ -3195,7 +3215,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -3207,7 +3229,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); @@ -3337,7 +3361,9 @@ DECLARE_GPU_SPEC(float, int32); DECLARE_GPU_SPEC(float, int64); DECLARE_GPU_SPEC(double, int32); DECLARE_GPU_SPEC(double, int64); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64, int32); DECLARE_GPU_SPEC(complex64, int64); @@ -3355,7 +3381,9 @@ DECLARE_GPU_SPEC(complex128, int64); REGISTER_GPU_KERNELS(Eigen::half); REGISTER_GPU_KERNELS(float); REGISTER_GPU_KERNELS(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_GPU_KERNELS(complex64); REGISTER_GPU_KERNELS(complex128); @@ -3622,7 +3650,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -3634,7 +3664,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); @@ -4151,7 +4183,9 @@ namespace functor { DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); DECLARE_GPU_SPEC(double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS DECLARE_GPU_SPEC(complex64); DECLARE_GPU_SPEC(complex128); @@ -4163,7 +4197,9 @@ DECLARE_GPU_SPEC(complex128); REGISTER_KERNELS(GPU, Eigen::half); REGISTER_KERNELS(GPU, float); REGISTER_KERNELS(GPU, double); -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS REGISTER_KERNELS(GPU, complex64); REGISTER_KERNELS(GPU, complex128); diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc index 0995b31e734751..8b7f5dc2e40ef3 100644 --- a/tensorflow/core/kernels/training_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc @@ -524,7 +524,9 @@ struct ApplyPowerSign { template struct functor::ApplyGradientDescent; template struct functor::ApplyGradientDescent; template struct functor::ApplyGradientDescent; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyGradientDescent; template struct functor::ApplyGradientDescent; @@ -534,7 +536,9 @@ template struct functor::ApplyGradientDescent; template struct functor::ApplyAdagrad; template struct functor::ApplyAdagrad; template struct functor::ApplyAdagrad; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyAdagrad; template struct functor::ApplyAdagrad; @@ -544,7 +548,9 @@ template struct functor::ApplyAdagrad; template struct functor::ApplyAdagradV2; template struct functor::ApplyAdagradV2; template struct functor::ApplyAdagradV2; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyAdagradV2; template struct functor::ApplyAdagradV2; @@ -554,7 +560,9 @@ template struct functor::ApplyAdagradV2; template struct functor::ApplyAdadelta; template struct functor::ApplyAdadelta; template struct functor::ApplyAdadelta; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyAdadelta; template struct functor::ApplyAdadelta; @@ -572,7 +580,9 @@ template struct functor::ApplyFtrlV2; template struct functor::ApplyMomentum; template struct functor::ApplyMomentum; template struct functor::ApplyMomentum; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyMomentum; template struct functor::ApplyMomentum; @@ -582,7 +592,9 @@ template struct functor::ApplyMomentum; template struct functor::ApplyKerasMomentum; template struct functor::ApplyKerasMomentum; template struct functor::ApplyKerasMomentum; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyKerasMomentum; template struct functor::ApplyKerasMomentum; @@ -597,7 +609,9 @@ template struct functor::SparseApplyKerasMomentum; template struct functor::SparseApplyKerasMomentum; template struct functor::SparseApplyKerasMomentum; template struct functor::SparseApplyKerasMomentum; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::SparseApplyKerasMomentum; template struct functor::SparseApplyKerasMomentum; @@ -609,7 +623,9 @@ template struct functor::SparseApplyKerasMomentum; template struct functor::ApplyAdam; template struct functor::ApplyAdam; template struct functor::ApplyAdam; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyAdam; template struct functor::ApplyAdam; @@ -627,7 +643,9 @@ template struct functor::ApplyAdaMax; template struct functor::ApplyRMSProp; template struct functor::ApplyRMSProp; template struct functor::ApplyRMSProp; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyRMSProp; template struct functor::ApplyRMSProp; @@ -637,7 +655,9 @@ template struct functor::ApplyRMSProp; template struct functor::ApplyCenteredRMSProp; template struct functor::ApplyCenteredRMSProp; template struct functor::ApplyCenteredRMSProp; -#ifndef TENSORFLOW_USE_NVCC // TODO(b/143684500): Eigen to support complex sqrt +#if !defined(TENSORFLOW_USE_NVCC) && \ + !defined(TENSORFLOW_USE_ROCM) // TODO(b/143684500): Eigen to support + // complex sqrt #ifndef PLATFORM_WINDOWS template struct functor::ApplyCenteredRMSProp; template struct functor::ApplyCenteredRMSProp; diff --git a/tensorflow/core/lib/random/random_distributions.h b/tensorflow/core/lib/random/random_distributions.h index 6fb6babe7ec77a..6f40816aedb9e0 100644 --- a/tensorflow/core/lib/random/random_distributions.h +++ b/tensorflow/core/lib/random/random_distributions.h @@ -18,10 +18,7 @@ limitations under the License. #include -#define _USE_MATH_DEFINES -#include #include -#undef _USE_MATH_DEFINES #include #include diff --git a/tensorflow/core/lib/random/random_distributions_test.cc b/tensorflow/core/lib/random/random_distributions_test.cc index 8868672a10ae02..a49731601828cf 100644 --- a/tensorflow/core/lib/random/random_distributions_test.cc +++ b/tensorflow/core/lib/random/random_distributions_test.cc @@ -15,8 +15,8 @@ limitations under the License. #include "tensorflow/core/lib/random/random_distributions.h" -#include #include +#include #include #include #include diff --git a/tensorflow/core/platform/build_config.bzl b/tensorflow/core/platform/build_config.bzl new file mode 100644 index 00000000000000..03a67e9b789d1b --- /dev/null +++ b/tensorflow/core/platform/build_config.bzl @@ -0,0 +1,70 @@ +"""Provides a redirection point for platform specific implementations of starlark utilities.""" + +load( + "//tensorflow/core/platform:default/build_config.bzl", + _pyx_library = "pyx_library", + _tf_additional_all_protos = "tf_additional_all_protos", + _tf_additional_binary_deps = "tf_additional_binary_deps", + _tf_additional_core_deps = "tf_additional_core_deps", + _tf_additional_cupti_test_flags = "tf_additional_cupti_test_flags", + _tf_additional_cupti_utils_cuda_deps = "tf_additional_cupti_utils_cuda_deps", + _tf_additional_device_tracer_srcs = "tf_additional_device_tracer_srcs", + _tf_additional_lib_deps = "tf_additional_lib_deps", + _tf_additional_lib_hdrs = "tf_additional_lib_hdrs", + _tf_additional_lib_srcs = "tf_additional_lib_srcs", + _tf_additional_monitoring_hdrs = "tf_additional_monitoring_hdrs", + _tf_additional_monitoring_srcs = "tf_additional_monitoring_srcs", + _tf_additional_proto_hdrs = "tf_additional_proto_hdrs", + _tf_additional_rpc_deps = "tf_additional_rpc_deps", + _tf_additional_tensor_coding_deps = "tf_additional_tensor_coding_deps", + _tf_additional_test_deps = "tf_additional_test_deps", + _tf_additional_test_srcs = "tf_additional_test_srcs", + _tf_fingerprint_deps = "tf_fingerprint_deps", + _tf_jspb_proto_library = "tf_jspb_proto_library", + _tf_kernel_tests_linkstatic = "tf_kernel_tests_linkstatic", + _tf_lib_proto_parsing_deps = "tf_lib_proto_parsing_deps", + _tf_proto_library = "tf_proto_library", + _tf_proto_library_cc = "tf_proto_library_cc", + _tf_proto_library_py = "tf_proto_library_py", + _tf_protobuf_compiler_deps = "tf_protobuf_compiler_deps", + _tf_protobuf_deps = "tf_protobuf_deps", + _tf_protos_all = "tf_protos_all", + _tf_protos_all_impl = "tf_protos_all_impl", + _tf_protos_grappler = "tf_protos_grappler", + _tf_protos_grappler_impl = "tf_protos_grappler_impl", + _tf_py_clif_cc = "tf_py_clif_cc", + _tf_pyclif_proto_library = "tf_pyclif_proto_library", +) + +pyx_library = _pyx_library +tf_additional_all_protos = _tf_additional_all_protos +tf_additional_binary_deps = _tf_additional_binary_deps +tf_additional_core_deps = _tf_additional_core_deps +tf_additional_cupti_test_flags = _tf_additional_cupti_test_flags +tf_additional_cupti_utils_cuda_deps = _tf_additional_cupti_utils_cuda_deps +tf_additional_device_tracer_srcs = _tf_additional_device_tracer_srcs +tf_additional_lib_deps = _tf_additional_lib_deps +tf_additional_lib_hdrs = _tf_additional_lib_hdrs +tf_additional_lib_srcs = _tf_additional_lib_srcs +tf_additional_monitoring_hdrs = _tf_additional_monitoring_hdrs +tf_additional_monitoring_srcs = _tf_additional_monitoring_srcs +tf_additional_proto_hdrs = _tf_additional_proto_hdrs +tf_additional_rpc_deps = _tf_additional_rpc_deps +tf_additional_tensor_coding_deps = _tf_additional_tensor_coding_deps +tf_additional_test_deps = _tf_additional_test_deps +tf_additional_test_srcs = _tf_additional_test_srcs +tf_fingerprint_deps = _tf_fingerprint_deps +tf_jspb_proto_library = _tf_jspb_proto_library +tf_kernel_tests_linkstatic = _tf_kernel_tests_linkstatic +tf_lib_proto_parsing_deps = _tf_lib_proto_parsing_deps +tf_proto_library = _tf_proto_library +tf_proto_library_cc = _tf_proto_library_cc +tf_proto_library_py = _tf_proto_library_py +tf_protobuf_compiler_deps = _tf_protobuf_compiler_deps +tf_protobuf_deps = _tf_protobuf_deps +tf_protos_all = _tf_protos_all +tf_protos_all_impl = _tf_protos_all_impl +tf_protos_grappler = _tf_protos_grappler +tf_protos_grappler_impl = _tf_protos_grappler_impl +tf_py_clif_cc = _tf_py_clif_cc +tf_pyclif_proto_library = _tf_pyclif_proto_library diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD index b92d96e539e2c4..ac5c04938b0439 100644 --- a/tensorflow/core/profiler/internal/BUILD +++ b/tensorflow/core/profiler/internal/BUILD @@ -1,4 +1,4 @@ -load("//tensorflow:tensorflow.bzl", "tf_cc_test", "if_not_windows", "tf_cuda_library") +load("//tensorflow:tensorflow.bzl", "if_not_windows", "tf_cc_test", "tf_cuda_library") package( default_visibility = ["//tensorflow:internal"], diff --git a/tensorflow/core/profiler/internal/gpu/BUILD b/tensorflow/core/profiler/internal/gpu/BUILD index 40a0a744b45628..2067acc9a3497b 100644 --- a/tensorflow/core/profiler/internal/gpu/BUILD +++ b/tensorflow/core/profiler/internal/gpu/BUILD @@ -1,8 +1,8 @@ load( "//tensorflow:tensorflow.bzl", + "if_cuda_is_configured_compat", "tf_copts", "tf_cuda_library", - "if_cuda_is_configured_compat", ) load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu") load( diff --git a/tensorflow/core/protobuf/eager_service.proto b/tensorflow/core/protobuf/eager_service.proto index d90ba548e0e7b3..4335d87309a664 100644 --- a/tensorflow/core/protobuf/eager_service.proto +++ b/tensorflow/core/protobuf/eager_service.proto @@ -90,6 +90,11 @@ message CreateContextRequest { // The view ID of the context. fixed64 context_view_id = 8; + // For a multi device function, if false, eagerly copy all remote inputs to + // the default function device; if true, lazily copy remote inputs to their + // target devices after function instantiation to avoid redundant copies. + bool lazy_copy_remote_function_inputs = 9; + reserved 5; } diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 4c35788e5dee7c..10d6b545b2a254 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -21,7 +21,7 @@ limitations under the License. // Also update tensorflow/tensorflow.bzl and // tensorflow/tools/pip_package/setup.py #define TF_MAJOR_VERSION 2 -#define TF_MINOR_VERSION 0 +#define TF_MINOR_VERSION 1 #define TF_PATCH_VERSION 0 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index 08b44eb42c641d..05c83b3001e544 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -218,6 +218,7 @@ cc_library( "//tensorflow/lite/nnapi:nnapi_implementation", "//tensorflow/lite/schema:schema_fbs", ], + alwayslink = 1, ) cc_library( diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl index 9753e9bf35c57a..a6236f98da6f65 100644 --- a/tensorflow/lite/build_def.bzl +++ b/tensorflow/lite/build_def.bzl @@ -24,7 +24,10 @@ def tflite_copts(): "/wd4018", # -Wno-sign-compare ], "//conditions:default": [ + "-Wno-deprecated-declarations", "-Wno-sign-compare", + "-Wno-unused-const-variable", + "-Wno-unused-function", ], }) + select({ clean_dep("//tensorflow:optimized"): ["-O3"], diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h index 55920dfd93ddf9..7a525ef0b063e1 100644 --- a/tensorflow/lite/c/c_api_internal.h +++ b/tensorflow/lite/c/c_api_internal.h @@ -261,39 +261,22 @@ typedef struct { int32_t quantized_dimension; } TfLiteAffineQuantization; -/* Note: can't use TFLITE_DEPRECATED from compatibility.h, it needs C++. */ -#if defined(__clang__) && __cplusplus >= 201103L -#define DEPRECATED_PTRUNION_MEMBER \ - __attribute__( \ - (deprecated("Do not use this field. Use GetTensorData(tensor) or " \ - "cast .raw to the appropriate type."))) -#elif defined(__GNUC__) || defined(__clang__) -#define DEPRECATED_PTRUNION_MEMBER __attribute__((deprecated)) -#else -#define DEPRECATED_PTRUNION_MEMBER -#endif - -/* A union of pointers that points to memory for a given tensor. - * Do not access these members directly, if possible, use GetTensorData instead. - * When accessing directly, only do .data, as other members are deprecated. */ +// A union of pointers that points to memory for a given tensor. typedef union { - int32_t* i32 DEPRECATED_PTRUNION_MEMBER; - int64_t* i64 DEPRECATED_PTRUNION_MEMBER; - float* f DEPRECATED_PTRUNION_MEMBER; - TfLiteFloat16* f16 DEPRECATED_PTRUNION_MEMBER; - char* raw DEPRECATED_PTRUNION_MEMBER; - const char* raw_const DEPRECATED_PTRUNION_MEMBER; - uint8_t* uint8 DEPRECATED_PTRUNION_MEMBER; - bool* b DEPRECATED_PTRUNION_MEMBER; - int16_t* i16 DEPRECATED_PTRUNION_MEMBER; - TfLiteComplex64* c64 DEPRECATED_PTRUNION_MEMBER; - int8_t* int8 DEPRECATED_PTRUNION_MEMBER; - /* Only use this member. */ - void* data; + int32_t* i32; + int64_t* i64; + float* f; + // Placeholder for 16b float type. Use uint16* in the pointer union for now. + TfLiteFloat16* f16; + char* raw; + const char* raw_const; + uint8_t* uint8; + bool* b; + int16_t* i16; + TfLiteComplex64* c64; + int8_t* int8; } TfLitePtrUnion; -#undef DEPRECATED_PTRUNION_MEMBER - // Memory allocation strategies. kTfLiteMmapRo is for read-only memory-mapped // data (or data externally allocated). kTfLiteArenaRw is arena allocated // data. kTfLiteDynamic is for tensors that are allocated during evaluation. diff --git a/tensorflow/lite/delegates/flex/delegate_data.cc b/tensorflow/lite/delegates/flex/delegate_data.cc index bed38bdffdd998..2be928073ffed6 100644 --- a/tensorflow/lite/delegates/flex/delegate_data.cc +++ b/tensorflow/lite/delegates/flex/delegate_data.cc @@ -47,8 +47,8 @@ tensorflow::Status DelegateData::Prepare( session_options, tensorflow::ContextDevicePlacementPolicy::DEVICE_PLACEMENT_SILENT, tensorflow::ContextMirroringPolicy::MIRRORING_NONE, - /*async=*/false, device_mgr.release(), /*device_mgr_owned*/ true, - rendezvous, nullptr); + /*async=*/false, /*lazy_copy_function_remote_inputs=*/false, + device_mgr.release(), /*device_mgr_owned*/ true, rendezvous, nullptr); return tensorflow::Status(); } diff --git a/tensorflow/lite/delegates/gpu/metal/BUILD b/tensorflow/lite/delegates/gpu/metal/BUILD index 3f4497748583f7..4bf443195df4bd 100644 --- a/tensorflow/lite/delegates/gpu/metal/BUILD +++ b/tensorflow/lite/delegates/gpu/metal/BUILD @@ -1,5 +1,5 @@ load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test") -load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite", "tflite_ios_per_kernel_test") +load("//tensorflow/lite:special_rules.bzl", "tflite_ios_per_kernel_test", "tflite_portable_test_suite") package( default_visibility = ["//visibility:public"], diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD index 2befc2d14e708b..84ea6cf2d8a7b2 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD @@ -1,5 +1,5 @@ load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") -load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite", "tflite_ios_per_kernel_test") +load("//tensorflow/lite:special_rules.bzl", "tflite_ios_per_kernel_test", "tflite_portable_test_suite") package( default_visibility = ["//visibility:public"], diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD index aa516543244f1a..180443c8b956c4 100644 --- a/tensorflow/lite/delegates/nnapi/BUILD +++ b/tensorflow/lite/delegates/nnapi/BUILD @@ -26,6 +26,9 @@ cc_library( "nnapi_delegate.h", "nnapi_delegate_kernel.h", ], + copts = [ + "-Wno-unused-private-field", + ], deps = [ "//tensorflow/lite:allocation", "//tensorflow/lite:kernel_api", diff --git a/tensorflow/lite/examples/experimental_new_converter/keras_lstm.ipynb b/tensorflow/lite/examples/experimental_new_converter/keras_lstm.ipynb index 1ab17d03c975d1..7f10e50a48fccf 100644 --- a/tensorflow/lite/examples/experimental_new_converter/keras_lstm.ipynb +++ b/tensorflow/lite/examples/experimental_new_converter/keras_lstm.ipynb @@ -156,13 +156,11 @@ "outputs": [], "source": [ "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n", - "# Note: It will NOT work without enabling the MLIR-based converter!\n", - "# `experimental_enable_mlir_converter` flag was renamed to\n", - "# `experimental_new_converter`. The new code wasn't deployed to the\n", + "# Note: It will NOT work without enabling the experimental converter!\n", + "# `experimental_new_converter` flag. The new code wasn't deployed to the\n", "# \"TensorFlow public guest runtime\" so we're setting both flags in Colab\n", "# for now.\n", "converter.experimental_new_converter = True\n", - "converter.experimental_enable_mlir_converter = True\n", "tflite_model = converter.convert()" ] }, diff --git a/tensorflow/lite/experimental/c/BUILD b/tensorflow/lite/experimental/c/BUILD index 23b64f72d16f61..8f0c29eb755cbc 100644 --- a/tensorflow/lite/experimental/c/BUILD +++ b/tensorflow/lite/experimental/c/BUILD @@ -5,7 +5,7 @@ load( ) package( - default_visibility = [":experimental"], + default_visibility = ["//visibility:public"], licenses = ["notice"], # Apache 2.0 ) @@ -64,7 +64,7 @@ cc_library( ], copts = tflite_copts(), visibility = [ - ":experimental", + "//visibility:public", ], deps = [ ":c_api_internal", @@ -73,6 +73,7 @@ cc_library( "//tensorflow/lite/c:c_api_internal", "//tensorflow/lite/kernels:builtin_ops", ], + alwayslink = 1, ) cc_library( @@ -85,6 +86,7 @@ cc_library( ":c_api_internal", "//tensorflow/lite:kernel_api", ], + alwayslink = 1, ) cc_test( @@ -108,6 +110,9 @@ cc_test( size = "small", srcs = ["c_api_experimental_test.cc"], data = ["//tensorflow/lite:testdata/add.bin"], + visibility = [ + "//visibility:public", + ], deps = [ ":c_api", ":c_api_experimental", diff --git a/tensorflow/lite/experimental/c/c_api_types.h b/tensorflow/lite/experimental/c/c_api_types.h index 55920dfd93ddf9..7a525ef0b063e1 100644 --- a/tensorflow/lite/experimental/c/c_api_types.h +++ b/tensorflow/lite/experimental/c/c_api_types.h @@ -261,39 +261,22 @@ typedef struct { int32_t quantized_dimension; } TfLiteAffineQuantization; -/* Note: can't use TFLITE_DEPRECATED from compatibility.h, it needs C++. */ -#if defined(__clang__) && __cplusplus >= 201103L -#define DEPRECATED_PTRUNION_MEMBER \ - __attribute__( \ - (deprecated("Do not use this field. Use GetTensorData(tensor) or " \ - "cast .raw to the appropriate type."))) -#elif defined(__GNUC__) || defined(__clang__) -#define DEPRECATED_PTRUNION_MEMBER __attribute__((deprecated)) -#else -#define DEPRECATED_PTRUNION_MEMBER -#endif - -/* A union of pointers that points to memory for a given tensor. - * Do not access these members directly, if possible, use GetTensorData instead. - * When accessing directly, only do .data, as other members are deprecated. */ +// A union of pointers that points to memory for a given tensor. typedef union { - int32_t* i32 DEPRECATED_PTRUNION_MEMBER; - int64_t* i64 DEPRECATED_PTRUNION_MEMBER; - float* f DEPRECATED_PTRUNION_MEMBER; - TfLiteFloat16* f16 DEPRECATED_PTRUNION_MEMBER; - char* raw DEPRECATED_PTRUNION_MEMBER; - const char* raw_const DEPRECATED_PTRUNION_MEMBER; - uint8_t* uint8 DEPRECATED_PTRUNION_MEMBER; - bool* b DEPRECATED_PTRUNION_MEMBER; - int16_t* i16 DEPRECATED_PTRUNION_MEMBER; - TfLiteComplex64* c64 DEPRECATED_PTRUNION_MEMBER; - int8_t* int8 DEPRECATED_PTRUNION_MEMBER; - /* Only use this member. */ - void* data; + int32_t* i32; + int64_t* i64; + float* f; + // Placeholder for 16b float type. Use uint16* in the pointer union for now. + TfLiteFloat16* f16; + char* raw; + const char* raw_const; + uint8_t* uint8; + bool* b; + int16_t* i16; + TfLiteComplex64* c64; + int8_t* int8; } TfLitePtrUnion; -#undef DEPRECATED_PTRUNION_MEMBER - // Memory allocation strategies. kTfLiteMmapRo is for read-only memory-mapped // data (or data externally allocated). kTfLiteArenaRw is arena allocated // data. kTfLiteDynamic is for tensors that are allocated during evaluation. diff --git a/tensorflow/lite/experimental/microfrontend/lib/window_util.c b/tensorflow/lite/experimental/microfrontend/lib/window_util.c index 3e544f5dd385e5..eee6e7b56ef340 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/window_util.c +++ b/tensorflow/lite/experimental/microfrontend/lib/window_util.c @@ -14,8 +14,6 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/experimental/microfrontend/lib/window_util.h" -// This macro is required to make MSVC defines math constants in math.h -#define _USE_MATH_DEFINES #include #include #include diff --git a/tensorflow/lite/experimental/ruy/pack_avx2.cc b/tensorflow/lite/experimental/ruy/pack_avx2.cc index 95f39ca5b85fde..ad71a0860ea2f4 100644 --- a/tensorflow/lite/experimental/ruy/pack_avx2.cc +++ b/tensorflow/lite/experimental/ruy/pack_avx2.cc @@ -60,25 +60,6 @@ using PackImplFloatAvx2 = namespace { -inline __m256i MaskLoadu(int available_src_rows, std::int8_t zero_point, - const std::int8_t* addr) { - RUY_DCHECK_LT(available_src_rows, 32); - __m256i padded_data; - - if (available_src_rows >= 16) { - __m128i load_hi = _mm_set1_epi8(zero_point); - __m128i load_lo = _mm_loadu_si128(reinterpret_cast(addr)); - memcpy(&load_hi, addr + 16, available_src_rows - 16); - padded_data = _mm256_set_m128i(load_hi, load_lo); - } else { - __m128i load_hi = _mm_set1_epi8(zero_point); - __m128i load_lo = load_hi; - memcpy(&load_lo, addr, available_src_rows); - padded_data = _mm256_set_m128i(load_hi, load_lo); - } - return padded_data; -} - inline void Pack8bitAvx2Packer(const std::int8_t* src_ptr, std::int8_t input_xor, const std::int8_t* zerobuf, int src_stride, @@ -93,6 +74,8 @@ inline void Pack8bitAvx2Packer(const std::int8_t* src_ptr, constexpr int kNumRowChunks = 8; constexpr int kNumChunkedSrcRows = kNumRowChunks * Layout::kRows; + std::int8_t in_data[Layout::kCols][kNumRowChunks][Layout::kRows]; + const std::int8_t* src_ptr0 = src_ptr; const std::int8_t* src_ptr1 = src_ptr0 + src_stride; const std::int8_t* src_ptr2 = src_ptr1 + src_stride; @@ -153,7 +136,6 @@ inline void Pack8bitAvx2Packer(const std::int8_t* src_ptr, } __m256i sums_8x4_16bit_lo = _mm256_set1_epi16(0); __m256i sums_8x4_16bit_hi = _mm256_set1_epi16(0); - std::int32_t sums_adjustment = 0; // The overall packing effectively pads the source rows to // (src_rows + 63) & ~63. The iteration over k may skip when m=1, and then we @@ -364,132 +346,80 @@ inline void Pack8bitAvx2Packer(const std::int8_t* src_ptr, } } else if (available_src_rows > 0) { RUY_DCHECK_LT(available_src_rows, kNumChunkedSrcRows); - + int i = 0; + // Consume chunks of 4 rows that are complete. + for (; i < (available_src_rows >> 2); ++i) { + for (int s = 0; s < 4; ++s) { + in_data[0][i][s] = src_ptr0[i * 4 + s]; + in_data[1][i][s] = src_ptr1[i * 4 + s]; + in_data[2][i][s] = src_ptr2[i * 4 + s]; + in_data[3][i][s] = src_ptr3[i * 4 + s]; + in_data[4][i][s] = src_ptr4[i * 4 + s]; + in_data[5][i][s] = src_ptr5[i * 4 + s]; + in_data[6][i][s] = src_ptr6[i * 4 + s]; + in_data[7][i][s] = src_ptr7[i * 4 + s]; + } + } + // Consume any incomplete chunk. + if (i < ((available_src_rows + 3) >> 2)) { + int s = 0; + for (; s < (available_src_rows & 3); ++s) { + in_data[0][i][s] = src_ptr0[i * 4 + s]; + in_data[1][i][s] = src_ptr1[i * 4 + s]; + in_data[2][i][s] = src_ptr2[i * 4 + s]; + in_data[3][i][s] = src_ptr3[i * 4 + s]; + in_data[4][i][s] = src_ptr4[i * 4 + s]; + in_data[5][i][s] = src_ptr5[i * 4 + s]; + in_data[6][i][s] = src_ptr6[i * 4 + s]; + in_data[7][i][s] = src_ptr7[i * 4 + s]; + } + RUY_DCHECK_LE(s, 4); + for (; s < 4; ++s) { + // j: Layout::kCols. + for (int j = 0; j < 8; ++j) { + in_data[j][i][s] = zero_point; + } + } + ++i; + } // We do not care what goes into the trailing buffer, but we want // in_data[...] ^ input_xor == 0 for irrelevant values in the summation. // - // We compensate for padding-with-zero_point by initializing the - // summations with the compensating offset, effectively - // ((input_xor ^ input_xor) - (zero_point ^ input_xor)) * + // It might prove better in optimized code to pad uniformly with + // zero_point, and compensate by initializing the summations with the + // compensating offset, effectively + // ((input_xor - zero_point) ^ input_xor) * // 4 * (8 - ((available_src_rows + 3) >> 2)). + for (; i < 8; ++i) { + for (int s = 0; s < 4; ++s) { + for (int j = 0; j < 8; ++j) { + in_data[j][i][s] = input_xor; + } + } + } + // We loop through [0, 8) rather than + // [0, (available_src_rows + 3) >> 2), since that emulates what we might + // do in fully-optimized code. // - // Note that (zero_point ^ input_xor) is performed in 8-bits and then - // cast. - sums_adjustment += - -(zero_point ^ input_xor) * 4 * (8 - ((available_src_rows + 3) >> 2)); - - __m256i t0, t1, t2, t3, t4, t5, t6, t7; - __m256i r0, r1, r2, r3, r4, r5, r6, r7; - const __m256i input_xor_v = _mm256_set1_epi8(input_xor); - - t0 = MaskLoadu(available_src_rows, zero_point, src_ptr0); - t4 = MaskLoadu(available_src_rows, zero_point, src_ptr4); - t1 = MaskLoadu(available_src_rows, zero_point, src_ptr1); - t5 = MaskLoadu(available_src_rows, zero_point, src_ptr5); - t2 = MaskLoadu(available_src_rows, zero_point, src_ptr2); - t6 = MaskLoadu(available_src_rows, zero_point, src_ptr6); - t3 = MaskLoadu(available_src_rows, zero_point, src_ptr3); - t7 = MaskLoadu(available_src_rows, zero_point, src_ptr7); - - r0 = _mm256_unpacklo_epi32(t0, t1); - r4 = _mm256_unpacklo_epi32(t4, t5); - r2 = _mm256_unpackhi_epi32(t0, t1); - r6 = _mm256_unpackhi_epi32(t4, t5); - r1 = _mm256_unpacklo_epi32(t2, t3); - r5 = _mm256_unpacklo_epi32(t6, t7); - r3 = _mm256_unpackhi_epi32(t2, t3); - r7 = _mm256_unpackhi_epi32(t6, t7); - - t0 = _mm256_unpacklo_epi64(r0, r1); - t4 = _mm256_unpacklo_epi64(r4, r5); - t2 = _mm256_unpackhi_epi64(r0, r1); - t6 = _mm256_unpackhi_epi64(r4, r5); - t1 = _mm256_unpacklo_epi64(r2, r3); - t5 = _mm256_unpacklo_epi64(r6, r7); - t3 = _mm256_unpackhi_epi64(r2, r3); - t7 = _mm256_unpackhi_epi64(r6, r7); - - // The preceding sets of rearrangement operations interleaved by 4 bytes - // and then by 8 bytes *within* lanes. The following set interleave by - // 16 bytes (128-bit), operating *between* AVX lanes. For instance (t0, - // t4) are interleaved to create (r0, r1). This complexity follows from - // the way that AVX is centered around MM 128-bit lanes. - r0 = _mm256_permute2x128_si256(t0, t4, 0x20); - r4 = _mm256_permute2x128_si256(t1, t5, 0x20); - r1 = _mm256_permute2x128_si256(t0, t4, 0x31); - r5 = _mm256_permute2x128_si256(t1, t5, 0x31); - r2 = _mm256_permute2x128_si256(t2, t6, 0x20); - r6 = _mm256_permute2x128_si256(t3, t7, 0x20); - r3 = _mm256_permute2x128_si256(t2, t6, 0x31); - r7 = _mm256_permute2x128_si256(t3, t7, 0x31); - - r0 = _mm256_xor_si256(r0, input_xor_v); - r1 = _mm256_xor_si256(r1, input_xor_v); - r2 = _mm256_xor_si256(r2, input_xor_v); - r3 = _mm256_xor_si256(r3, input_xor_v); - r4 = _mm256_xor_si256(r4, input_xor_v); - r5 = _mm256_xor_si256(r5, input_xor_v); - r6 = _mm256_xor_si256(r6, input_xor_v); - r7 = _mm256_xor_si256(r7, input_xor_v); - - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r0))); - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r1))); - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r2))); - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r3))); - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r4))); - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r5))); - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r6))); - sums_8x4_16bit_lo = _mm256_add_epi16( - sums_8x4_16bit_lo, _mm256_cvtepi8_epi16(_mm256_castsi256_si128(r7))); - - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r0, 1))); - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r1, 1))); - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r2, 1))); - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r3, 1))); - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r4, 1))); - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r5, 1))); - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r6, 1))); - sums_8x4_16bit_hi = _mm256_add_epi16( - sums_8x4_16bit_hi, - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(r7, 1))); - - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 0 * 8 * 4), - r0); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 2 * 8 * 4), - r4); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 4 * 8 * 4), - r1); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 6 * 8 * 4), - r5); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 1 * 8 * 4), - r2); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 3 * 8 * 4), - r6); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 5 * 8 * 4), - r3); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(trailing_buf + 7 * 8 * 4), - r7); + // i: chunks, j: Layout::kCols, s: Layout::Rows. + if (sums_ptr) { + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + for (int s = 0; s < 4; ++s) { + trailing_buf[(8 * i + j) * 4 + s] = in_data[j][i][s] ^ input_xor; + sums_ptr[j] = sums_ptr[j] + (in_data[j][i][s] ^ input_xor); + } + } + } + } else { + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + for (int s = 0; s < 4; ++s) { + trailing_buf[(8 * i + j) * 4 + s] = in_data[j][i][s] ^ input_xor; + } + } + } + } } packed_ptr += 8 * kNumChunkedSrcRows; @@ -504,8 +434,6 @@ inline void Pack8bitAvx2Packer(const std::int8_t* src_ptr, } if (sums_ptr) { - const __m256i sums_adjustment_v = _mm256_set1_epi32(sums_adjustment); - __m256i sums = _mm256_loadu_si256(reinterpret_cast(sums_ptr)); const __m256i ones_16bit = _mm256_set1_epi16(1); @@ -527,7 +455,6 @@ inline void Pack8bitAvx2Packer(const std::int8_t* src_ptr, _mm256_permute2x128_si256(sums_2x8_32bit_lo, sums_2x8_32bit_hi, 0x20); const __m256i sums_2x8_32bit_b = _mm256_permute2x128_si256(sums_2x8_32bit_lo, sums_2x8_32bit_hi, 0x31); - sums = _mm256_add_epi32(sums, sums_adjustment_v); sums = _mm256_add_epi32(sums, sums_2x8_32bit_a); sums = _mm256_add_epi32(sums, sums_2x8_32bit_b); diff --git a/tensorflow/lite/experimental/ruy/pack_avx512.cc b/tensorflow/lite/experimental/ruy/pack_avx512.cc index 4c3504724750bd..0c1466048816b8 100644 --- a/tensorflow/lite/experimental/ruy/pack_avx512.cc +++ b/tensorflow/lite/experimental/ruy/pack_avx512.cc @@ -76,22 +76,6 @@ inline void ZeroHalf8bitAvx512(int src_rows, std::int8_t packed_zero_point, } } -inline __m512i LoaduTwo(const std::int8_t* addr_lo, - const std::int8_t* addr_hi) { - __m512i lower_filled = _mm512_castsi256_si512(_mm256_loadu_epi8(addr_lo)); - return _mm512_inserti32x8(lower_filled, _mm256_loadu_epi8(addr_hi), 1); -} - -inline __m512i MaskLoaduTwo(__mmask32 row_mask, const __m256i default_value_v, - const std::int8_t* addr_lo, - const std::int8_t* addr_hi) { - const __m512i lower_filled = _mm512_castsi256_si512( - _mm256_mask_loadu_epi8(default_value_v, row_mask, addr_lo)); - return _mm512_inserti32x8( - lower_filled, _mm256_mask_loadu_epi8(default_value_v, row_mask, addr_hi), - 1); -} - inline void HalfPack8bitAvx512(const std::int8_t* src_ptr, std::int8_t input_xor, const std::int8_t* zerobuf, int src_stride, @@ -99,13 +83,19 @@ inline void HalfPack8bitAvx512(const std::int8_t* src_ptr, std::int8_t* packed_ptr, std::int32_t* sums_ptr, std::int8_t* trailing_buf) { using Layout = PackImpl8bitAvx512::Layout; + static constexpr int kHalfLayoutCols = + PackImpl8bitAvx512::kHalfLayoutCols; // Half the number of cols in a + // block. RUY_DCHECK_EQ(Layout::kCols, 16); RUY_DCHECK_EQ(Layout::kRows, 4); + RUY_DCHECK_EQ(kHalfLayoutCols, 8); // Each Layout::Rows is 4 contiguous input, contiguous packed elements. // We process 8 of these chunks at a time, padding short input chunks. constexpr int kNumRowChunks = 8; constexpr int kNumChunkedSrcRows = kNumRowChunks * Layout::kRows; + std::int8_t in_data[kHalfLayoutCols][kNumRowChunks][Layout::kRows]; + const std::int8_t* src_ptr0 = src_ptr; const std::int8_t* src_ptr1 = src_ptr0 + src_stride; const std::int8_t* src_ptr2 = src_ptr1 + src_stride; @@ -164,8 +154,6 @@ inline void HalfPack8bitAvx512(const std::int8_t* src_ptr, sums_ptr[i] = 0; } } - __m512i sums_8x4_16bit = _mm512_set1_epi16(0); - std::int32_t sums_adjustment = 0; // The overall packing effectively pads the source rows to // (src_rows + 63) & ~63. The iteration over k may skip when m=1, and then we @@ -184,195 +172,111 @@ inline void HalfPack8bitAvx512(const std::int8_t* src_ptr, // treat each case separately. if (available_src_rows >= kNumChunkedSrcRows) { // i: chunks, s: Layout::Rows. - if (sums_ptr) { - __m512i t0, t1, t2, t3; - __m512i r0, r1, r2, r3; - const __m512i input_xor_v = _mm512_set1_epi8(input_xor); - - t0 = LoaduTwo(src_ptr0, src_ptr4); - t1 = LoaduTwo(src_ptr1, src_ptr5); - t2 = LoaduTwo(src_ptr2, src_ptr6); - t3 = LoaduTwo(src_ptr3, src_ptr7); - - r0 = _mm512_unpacklo_epi32(t0, t1); - r2 = _mm512_unpackhi_epi32(t0, t1); - r1 = _mm512_unpacklo_epi32(t2, t3); - r3 = _mm512_unpackhi_epi32(t2, t3); - - t0 = _mm512_unpacklo_epi64(r0, r1); - t2 = _mm512_unpackhi_epi64(r0, r1); - t1 = _mm512_unpacklo_epi64(r2, r3); - t3 = _mm512_unpackhi_epi64(r2, r3); - - r0 = _mm512_shuffle_i32x4(t0, t1, 0x88); - r1 = _mm512_shuffle_i32x4(t0, t1, 0xdd); - r2 = _mm512_shuffle_i32x4(t2, t3, 0x88); - r3 = _mm512_shuffle_i32x4(t2, t3, 0xdd); - - r0 = _mm512_xor_si512(r0, input_xor_v); - r1 = _mm512_xor_si512(r1, input_xor_v); - r2 = _mm512_xor_si512(r2, input_xor_v); - r3 = _mm512_xor_si512(r3, input_xor_v); - - const __m256i r0_0 = _mm512_castsi512_si256(r0); - const __m256i r0_1 = _mm512_extracti32x8_epi32(r0, 1); - const __m256i r1_0 = _mm512_castsi512_si256(r1); - const __m256i r1_1 = _mm512_extracti32x8_epi32(r1, 1); - const __m256i r2_0 = _mm512_castsi512_si256(r2); - const __m256i r2_1 = _mm512_extracti32x8_epi32(r2, 1); - const __m256i r3_0 = _mm512_castsi512_si256(r3); - const __m256i r3_1 = _mm512_extracti32x8_epi32(r3, 1); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r0_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r0_1)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r1_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r1_1)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r2_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r2_1)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r3_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r3_1)); - _mm256_storeu_epi8(packed_ptr + 0 * 16 * 4, r0_0); - _mm256_storeu_epi8(packed_ptr + 2 * 16 * 4, r0_1); - _mm256_storeu_epi8(packed_ptr + 4 * 16 * 4, r1_0); - _mm256_storeu_epi8(packed_ptr + 6 * 16 * 4, r1_1); - _mm256_storeu_epi8(packed_ptr + 1 * 16 * 4, r2_0); - _mm256_storeu_epi8(packed_ptr + 3 * 16 * 4, r2_1); - _mm256_storeu_epi8(packed_ptr + 5 * 16 * 4, r3_0); - _mm256_storeu_epi8(packed_ptr + 7 * 16 * 4, r3_1); - } else { - __m512i t0, t1, t2, t3; - __m512i r0, r1, r2, r3; - const __m512i input_xor_v = _mm512_set1_epi8(input_xor); - - t0 = LoaduTwo(src_ptr0, src_ptr4); - t1 = LoaduTwo(src_ptr1, src_ptr5); - t2 = LoaduTwo(src_ptr2, src_ptr6); - t3 = LoaduTwo(src_ptr3, src_ptr7); - - r0 = _mm512_unpacklo_epi32(t0, t1); - r2 = _mm512_unpackhi_epi32(t0, t1); - r1 = _mm512_unpacklo_epi32(t2, t3); - r3 = _mm512_unpackhi_epi32(t2, t3); - - t0 = _mm512_unpacklo_epi64(r0, r1); - t2 = _mm512_unpackhi_epi64(r0, r1); - t1 = _mm512_unpacklo_epi64(r2, r3); - t3 = _mm512_unpackhi_epi64(r2, r3); - - r0 = _mm512_shuffle_i32x4(t0, t1, 0x88); - r1 = _mm512_shuffle_i32x4(t0, t1, 0xdd); - r2 = _mm512_shuffle_i32x4(t2, t3, 0x88); - r3 = _mm512_shuffle_i32x4(t2, t3, 0xdd); - - r0 = _mm512_xor_si512(r0, input_xor_v); - r1 = _mm512_xor_si512(r1, input_xor_v); - r2 = _mm512_xor_si512(r2, input_xor_v); - r3 = _mm512_xor_si512(r3, input_xor_v); - - const __m256i r0_0 = _mm512_castsi512_si256(r0); - const __m256i r0_1 = _mm512_extracti32x8_epi32(r0, 1); - const __m256i r1_0 = _mm512_castsi512_si256(r1); - const __m256i r1_1 = _mm512_extracti32x8_epi32(r1, 1); - const __m256i r2_0 = _mm512_castsi512_si256(r2); - const __m256i r2_1 = _mm512_extracti32x8_epi32(r2, 1); - const __m256i r3_0 = _mm512_castsi512_si256(r3); - const __m256i r3_1 = _mm512_extracti32x8_epi32(r3, 1); - _mm256_storeu_epi8(packed_ptr + 0 * 16 * 4, r0_0); - _mm256_storeu_epi8(packed_ptr + 2 * 16 * 4, r0_1); - _mm256_storeu_epi8(packed_ptr + 4 * 16 * 4, r1_0); - _mm256_storeu_epi8(packed_ptr + 6 * 16 * 4, r1_1); - _mm256_storeu_epi8(packed_ptr + 1 * 16 * 4, r2_0); - _mm256_storeu_epi8(packed_ptr + 3 * 16 * 4, r2_1); - _mm256_storeu_epi8(packed_ptr + 5 * 16 * 4, r3_0); - _mm256_storeu_epi8(packed_ptr + 7 * 16 * 4, r3_1); + for (int i = 0; i < 8; ++i) { + for (int s = 0; s < 4; ++s) { + in_data[0][i][s] = src_ptr0[i * 4 + s]; + in_data[1][i][s] = src_ptr1[i * 4 + s]; + in_data[2][i][s] = src_ptr2[i * 4 + s]; + in_data[3][i][s] = src_ptr3[i * 4 + s]; + in_data[4][i][s] = src_ptr4[i * 4 + s]; + in_data[5][i][s] = src_ptr5[i * 4 + s]; + in_data[6][i][s] = src_ptr6[i * 4 + s]; + in_data[7][i][s] = src_ptr7[i * 4 + s]; + } + } + // i: chunks, j: kHalfLayoutCols, s: Layout::Rows. + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + for (int s = 0; s < 4; ++s) { + // 16 * 4 * i is offset for each block, that is + // (Layout::kCols * Layout::kRows * i) + packed_ptr[(16 * i + j) * 4 + s] = in_data[j][i][s] ^ input_xor; + } + if (sums_ptr) { + for (int s = 0; s < 4; ++s) { + sums_ptr[j] += in_data[j][i][s] ^ input_xor; + } + } + } } } else if (available_src_rows > 0) { RUY_DCHECK_LT(available_src_rows >> 2, kNumChunkedSrcRows); - const __mmask32 row_mask = - (static_cast(1) << available_src_rows) - 1; - + int i = 0; + // Consume chunks of 4 rows that are complete. + for (; i < (available_src_rows >> 2); ++i) { + for (int s = 0; s < 4; ++s) { + in_data[0][i][s] = src_ptr0[i * 4 + s]; + in_data[1][i][s] = src_ptr1[i * 4 + s]; + in_data[2][i][s] = src_ptr2[i * 4 + s]; + in_data[3][i][s] = src_ptr3[i * 4 + s]; + in_data[4][i][s] = src_ptr4[i * 4 + s]; + in_data[5][i][s] = src_ptr5[i * 4 + s]; + in_data[6][i][s] = src_ptr6[i * 4 + s]; + in_data[7][i][s] = src_ptr7[i * 4 + s]; + } + } + // Consume any incomplete chunk. + if (i < ((available_src_rows + 3) >> 2)) { + int s = 0; + for (; s < (available_src_rows & 3); ++s) { + in_data[0][i][s] = src_ptr0[i * 4 + s]; + in_data[1][i][s] = src_ptr1[i * 4 + s]; + in_data[2][i][s] = src_ptr2[i * 4 + s]; + in_data[3][i][s] = src_ptr3[i * 4 + s]; + in_data[4][i][s] = src_ptr4[i * 4 + s]; + in_data[5][i][s] = src_ptr5[i * 4 + s]; + in_data[6][i][s] = src_ptr6[i * 4 + s]; + in_data[7][i][s] = src_ptr7[i * 4 + s]; + } + RUY_DCHECK_LE(s, 4); + for (; s < 4; ++s) { + // j: kHalfLayoutCols. + for (int j = 0; j < 8; ++j) { + in_data[j][i][s] = zero_point; + } + } + ++i; + } // We do not care what goes into the trailing buffer, but we want // in_data[...] ^ input_xor == 0 for irrelevant values in the summation. // - // We compensate for padding-with-zero_point by initializing the - // summations with the compensating offset, effectively - // ((input_xor ^ input_xor) - (zero_point ^ input_xor)) * + // It might prove better in optimized code to pad uniformly with + // zero_point, and compensate by initializing the summations with the + // compensating offset, effectively + // ((input_xor - zero_point) ^ input_xor) * // 4 * (8 - ((available_src_rows + 3) >> 2)). + for (; i < 8; ++i) { + for (int s = 0; s < 4; ++s) { + for (int j = 0; j < 8; ++j) { + in_data[j][i][s] = input_xor; + } + } + } + // We loop through [0, 8) rather than + // [0, (available_src_rows + 3) >> 2), since that emulates what we might + // do in fully-optimized code. // - // Note that (zero_point ^ input_xor) is performed in 8-bits and then - // cast. - sums_adjustment += -(zero_point ^ input_xor) * 4 * - (8 - ((available_src_rows + 3) >> 2)); - - __m512i t0, t1, t2, t3; - __m512i r0, r1, r2, r3; - const __m512i input_xor_v = _mm512_set1_epi8(input_xor); - const __m256i zero_point_v = _mm256_set1_epi8(zero_point); - - t0 = MaskLoaduTwo(row_mask, zero_point_v, src_ptr0, src_ptr4); - t1 = MaskLoaduTwo(row_mask, zero_point_v, src_ptr1, src_ptr5); - t2 = MaskLoaduTwo(row_mask, zero_point_v, src_ptr2, src_ptr6); - t3 = MaskLoaduTwo(row_mask, zero_point_v, src_ptr3, src_ptr7); - - r0 = _mm512_unpacklo_epi32(t0, t1); - r2 = _mm512_unpackhi_epi32(t0, t1); - r1 = _mm512_unpacklo_epi32(t2, t3); - r3 = _mm512_unpackhi_epi32(t2, t3); - - t0 = _mm512_unpacklo_epi64(r0, r1); - t2 = _mm512_unpackhi_epi64(r0, r1); - t1 = _mm512_unpacklo_epi64(r2, r3); - t3 = _mm512_unpackhi_epi64(r2, r3); - - r0 = _mm512_shuffle_i32x4(t0, t1, 0x88); - r1 = _mm512_shuffle_i32x4(t0, t1, 0xdd); - r2 = _mm512_shuffle_i32x4(t2, t3, 0x88); - r3 = _mm512_shuffle_i32x4(t2, t3, 0xdd); - - r0 = _mm512_xor_si512(r0, input_xor_v); - r1 = _mm512_xor_si512(r1, input_xor_v); - r2 = _mm512_xor_si512(r2, input_xor_v); - r3 = _mm512_xor_si512(r3, input_xor_v); - - const __m256i r0_0 = _mm512_castsi512_si256(r0); - const __m256i r0_1 = _mm512_extracti32x8_epi32(r0, 1); - const __m256i r1_0 = _mm512_castsi512_si256(r1); - const __m256i r1_1 = _mm512_extracti32x8_epi32(r1, 1); - const __m256i r2_0 = _mm512_castsi512_si256(r2); - const __m256i r2_1 = _mm512_extracti32x8_epi32(r2, 1); - const __m256i r3_0 = _mm512_castsi512_si256(r3); - const __m256i r3_1 = _mm512_extracti32x8_epi32(r3, 1); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r0_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r0_1)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r1_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r1_1)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r2_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r2_1)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r3_0)); - sums_8x4_16bit = - _mm512_add_epi16(sums_8x4_16bit, _mm512_cvtepi8_epi16(r3_1)); - _mm256_storeu_epi8(trailing_buf + 0 * 16 * 4, r0_0); - _mm256_storeu_epi8(trailing_buf + 2 * 16 * 4, r0_1); - _mm256_storeu_epi8(trailing_buf + 4 * 16 * 4, r1_0); - _mm256_storeu_epi8(trailing_buf + 6 * 16 * 4, r1_1); - _mm256_storeu_epi8(trailing_buf + 1 * 16 * 4, r2_0); - _mm256_storeu_epi8(trailing_buf + 3 * 16 * 4, r2_1); - _mm256_storeu_epi8(trailing_buf + 5 * 16 * 4, r3_0); - _mm256_storeu_epi8(trailing_buf + 7 * 16 * 4, r3_1); + // i: chunks, j: kHalfLayoutCols, s: Layout::Rows. + if (sums_ptr) { + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + for (int s = 0; s < 4; ++s) { + trailing_buf[(16 * i + j) * 4 + s] = + in_data[j][i][s] ^ input_xor; + sums_ptr[j] = sums_ptr[j] + (in_data[j][i][s] ^ input_xor); + } + } + } + } else { + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + for (int s = 0; s < 4; ++s) { + trailing_buf[(16 * i + j) * 4 + s] = + in_data[j][i][s] ^ input_xor; + } + } + } + } } packed_ptr += 16 * kNumChunkedSrcRows; @@ -386,39 +290,16 @@ inline void HalfPack8bitAvx512(const std::int8_t* src_ptr, src_ptr7 += src_inc7; } } - - if (sums_ptr) { - const __m256i sums_adjustment_v = _mm256_set1_epi32(sums_adjustment); - - __m256i sums = _mm256_loadu_epi32(sums_ptr); - const __m512i ones_16bit = _mm512_set1_epi16(1); - const __m512i idx = - _mm512_set_epi32(15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0); - - // The sums have been performed across columns, and now we have 4x16-bit - // sums packed together. We use madd for pairwise 32-bit sums, then we - // deinterlace the neighbours, finshing up by adding them to the stored - // accumulated sums. - const __m512i sums_8x2_32bit = - _mm512_madd_epi16(sums_8x4_16bit, ones_16bit); - const __m512i sums_2x8_32bit = - _mm512_permutexvar_epi32(idx, sums_8x2_32bit); - sums = _mm256_add_epi32(sums, sums_adjustment_v); - sums = _mm256_add_epi32(sums, _mm512_castsi512_si256(sums_2x8_32bit)); - sums = _mm256_add_epi32(sums, _mm512_extracti32x8_epi32(sums_2x8_32bit, 1)); - - _mm256_storeu_epi32(sums_ptr, sums); - } } inline __m512 LoaduTwo(const float* addr_lo, const float* addr_hi) { - const __m512 lower_filled = _mm512_castps256_ps512(_mm256_loadu_ps(addr_lo)); + __m512 lower_filled = _mm512_castps256_ps512(_mm256_loadu_ps(addr_lo)); return _mm512_insertf32x8(lower_filled, _mm256_loadu_ps(addr_hi), 1); } inline __m512 MaskLoaduTwo(__mmask8 row_mask, const float* addr_lo, const float* addr_hi) { - const __m512 lower_filled = + __m512 lower_filled = _mm512_castps256_ps512(_mm256_maskz_loadu_ps(row_mask, addr_lo)); return _mm512_insertf32x8(lower_filled, _mm256_maskz_loadu_ps(row_mask, addr_hi), 1); diff --git a/tensorflow/lite/g3doc/guide/android.md b/tensorflow/lite/g3doc/guide/android.md index 8ebc859f846f9a..8b8324aa8d6e8f 100644 --- a/tensorflow/lite/g3doc/guide/android.md +++ b/tensorflow/lite/g3doc/guide/android.md @@ -132,7 +132,7 @@ in the `.tf_configure.bazelrc` file in the root folder: ```shell build --action_env ANDROID_NDK_HOME="/usr/local/android/android-ndk-r17c" -build --action_env ANDROID_NDK_API_LEVEL="18" +build --action_env ANDROID_NDK_API_LEVEL="21" build --action_env ANDROID_BUILD_TOOLS_VERSION="28.0.3" build --action_env ANDROID_SDK_API_LEVEL="23" build --action_env ANDROID_SDK_HOME="/usr/local/android/android-sdk-linux" diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD index c137888214d835..f717c785dc4e59 100644 --- a/tensorflow/lite/kernels/BUILD +++ b/tensorflow/lite/kernels/BUILD @@ -542,6 +542,7 @@ cc_library( "@farmhash_archive//:farmhash", "@flatbuffers", ], + alwayslink = 1, ) cc_library( @@ -622,6 +623,7 @@ cc_library( "//tensorflow/lite:framework", "//tensorflow/lite/c:c_api_internal", ], + alwayslink = 1, ) # The builtin_ops target will resolve to optimized kernels when available. This diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD index 50b936668092ee..f2dc8c91ea3afa 100644 --- a/tensorflow/lite/kernels/internal/BUILD +++ b/tensorflow/lite/kernels/internal/BUILD @@ -565,7 +565,10 @@ cc_library( "optimized/neon_tensor_utils.h", "optimized/neon_tensor_utils_impl.h", ], - copts = NEON_FLAGS_IF_APPLICABLE + HARD_FP_FLAGS_IF_APPLICABLE, + copts = NEON_FLAGS_IF_APPLICABLE + HARD_FP_FLAGS_IF_APPLICABLE + [ + "-Wno-deprecated-declarations", + "-Wno-unused-function", + ], deps = [ ":common", ":compatibility", diff --git a/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_google.h b/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_google.h index de10f2c9259e99..1eb65c5bd5c9a4 100644 --- a/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_google.h +++ b/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_google.h @@ -91,7 +91,6 @@ typedef unsigned __int64 uint64_t; #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceDefault.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h" -#include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceSycl.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorDimensionList.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h" @@ -149,7 +148,6 @@ typedef unsigned __int64 uint64_t; #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorScan.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorTrace.h" -#include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorSycl.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorDevice.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h" diff --git a/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_oss.h b/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_oss.h index 5b54024ac5a0e4..027dd479af5cf9 100644 --- a/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_oss.h +++ b/tensorflow/lite/kernels/internal/optimized/eigen_tensor_reduced_instantiations_oss.h @@ -91,7 +91,6 @@ typedef unsigned __int64 uint64_t; #include "unsupported/Eigen/CXX11/src/Tensor/TensorDeviceDefault.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h" -#include "unsupported/Eigen/CXX11/src/Tensor/TensorDeviceSycl.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorDimensionList.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h" @@ -149,7 +148,6 @@ typedef unsigned __int64 uint64_t; #include "unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorScan.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorTrace.h" -#include "unsupported/Eigen/CXX11/src/Tensor/TensorSycl.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorDevice.h" #include "unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h" diff --git a/tensorflow/lite/toco/tflite/op_version.cc b/tensorflow/lite/toco/tflite/op_version.cc index 39258339e0e27c..a7a829e77e368c 100644 --- a/tensorflow/lite/toco/tflite/op_version.cc +++ b/tensorflow/lite/toco/tflite/op_version.cc @@ -74,7 +74,7 @@ string GetMinimumRuntimeVersionForModel(const Model& model) { {{OperatorType::kCast, 1}, "1.5.0"}, {{OperatorType::kConcatenation, 1}, "1.5.0"}, {{OperatorType::kConcatenation, 2}, "1.14.0"}, - {{OperatorType::kDepthToSpace, 1}, kPendingReleaseOpVersion}, + {{OperatorType::kDepthToSpace, 1}, "2.1.0"}, {{OperatorType::kFakeQuant, 1}, "1.5.0"}, {{OperatorType::kFakeQuant, 2}, "1.10.0"}, {{OperatorType::kFullyConnected, 1}, "1.5.0"}, @@ -82,7 +82,7 @@ string GetMinimumRuntimeVersionForModel(const Model& model) { {{OperatorType::kFullyConnected, 3}, "1.14.0"}, {{OperatorType::kFullyConnected, 4}, "1.14.0"}, {{OperatorType::kFullyConnected, 5}, "2.0.0"}, - {{OperatorType::kFullyConnected, 6}, kPendingReleaseOpVersion}, + {{OperatorType::kFullyConnected, 6}, "2.1.0"}, {{OperatorType::kGather, 1}, "1.6.0"}, {{OperatorType::kGather, 2}, "1.14.0"}, {{OperatorType::kGather, 3}, "1.15.0"}, @@ -145,7 +145,7 @@ string GetMinimumRuntimeVersionForModel(const Model& model) { {{OperatorType::kSplitV, 1}, "1.13.1"}, {{OperatorType::kStridedSlice, 1}, "1.6.0"}, {{OperatorType::kStridedSlice, 2}, "1.14.0"}, - {{OperatorType::kStridedSlice, 3}, kPendingReleaseOpVersion}, + {{OperatorType::kStridedSlice, 3}, "2.1.0"}, {{OperatorType::kTopK_V2, 1}, "1.7.0"}, {{OperatorType::kTopK_V2, 2}, "1.14.0"}, {{OperatorType::kArgMax, 1}, "1.9.0"}, @@ -205,7 +205,7 @@ string GetMinimumRuntimeVersionForModel(const Model& model) { {{OperatorType::kElu, 1}, "1.14.0"}, {{OperatorType::kRound, 1}, "1.14.0"}, {{OperatorType::kRelu, 1}, "1.5.0"}, - {{OperatorType::kRelu, 2}, kPendingReleaseOpVersion}, + {{OperatorType::kRelu, 2}, "2.1.0"}, {{OperatorType::kRelu1, 1}, "1.5.0"}, {{OperatorType::kPRelu, 1}, "1.8.0"}, {{OperatorType::kExp, 1}, "1.7.0"}, diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index f4f3fa7b1b163f..f2ca67521f257c 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1766,6 +1766,16 @@ py_library( ], ) +tf_py_test( + name = "framework_constant_op_test", + size = "small", + srcs = ["framework/constant_op_test.py"], + additional_deps = [ + ":constant_op", + ], + main = "framework/constant_op_test.py", +) + tf_py_test( name = "framework_registry_test", size = "small", @@ -3551,24 +3561,26 @@ py_library( ":loss_scale", ":unconnected_gradients", ":util", + "//tensorflow/python/distribute:distribute_lib", "//tensorflow/python/eager:backprop", ], ) -py_test( +cuda_py_test( name = "loss_scaling_gradient_tape_test", size = "medium", srcs = ["training/experimental/loss_scaling_gradient_tape_test.py"], - python_version = "PY3", - deps = [ + additional_deps = [ ":client_testlib", ":constant_op", + ":framework_test_combinations_lib", ":loss_scale", ":loss_scaling_gradient_tape", + "@absl_py//absl/testing:parameterized", + "//third_party/py/numpy", "//tensorflow/python/compat:v2_compat", + "//tensorflow/python/distribute:mirrored_strategy", "//tensorflow/python/eager:def_function", - "//third_party/py/numpy", - "@absl_py//absl/testing:parameterized", ], ) diff --git a/tensorflow/python/data/kernel_tests/shuffle_test.py b/tensorflow/python/data/kernel_tests/shuffle_test.py index b2d2d23a8fa871..7f801e1b5f4dfc 100644 --- a/tensorflow/python/data/kernel_tests/shuffle_test.py +++ b/tensorflow/python/data/kernel_tests/shuffle_test.py @@ -32,6 +32,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed from tensorflow.python.ops import array_ops +from tensorflow.python.ops import check_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -308,6 +309,28 @@ def consume(): consume() self.assertAllEqual(self.evaluate(counter_var), 10) + @combinations.generate(test_base.default_test_combinations()) + def testEmptyDataset(self): + dataset = dataset_ops.Dataset.from_tensors(1) + + def map_fn(x): + with ops.control_dependencies([check_ops.assert_equal(x, 0)]): + return x + + dataset = dataset.map(map_fn) + dataset = dataset.cache() + dataset = dataset.shuffle(buffer_size=10).repeat() + + get_next = self.getNext(dataset) + + # First time around, we get an error for the failed assertion. + with self.assertRaises(errors.InvalidArgumentError): + self.evaluate(get_next()) + + # Second time around, we get an EOF because the cached dataset is empty. + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index f49e0a65e3fac3..ea4c138ea90cee 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -1176,7 +1176,6 @@ def shuffle(self, buffer_size, seed=None, reshuffle_each_iteration=None): [1, 0, 2] >>> list(dataset.as_numpy_iterator()) # doctest: +SKIP [1, 0, 2] - ``` Args: buffer_size: A `tf.int64` scalar `tf.Tensor`, representing the number of @@ -1441,19 +1440,18 @@ def padded_batch(self, [[ 6 7 -1] [ 8 -1 -1]] >>> # Components of nested elements can be padded independently. - >>> elements = [{'v1': [1, 2, 3], 'v2': [10]}, - ... {'v1': [4, 5], 'v2': [11, 12]}] + >>> elements = [([1, 2, 3], [10]), + ... ([4, 5], [11, 12])] >>> dataset = tf.data.Dataset.from_generator( - ... lambda: iter(elements), {'v1': tf.int32, 'v2': tf.int32}) - >>> # Pad 'val1' to length 4, and 'val2' to the smallest size that fits. + ... lambda: iter(elements), (tf.int32, tf.int32)) + >>> # Pad the first component of the tuple to length 4, and the second + >>> # component to the smallest size that fits. >>> dataset = dataset.padded_batch(2, - ... padded_shapes={'v1': [4], 'v2': [None]}, - ... padding_values={'v1': -1, 'v2': 100}) + ... padded_shapes=([4], [None]), + ... padding_values=(-1, 100)) >>> list(dataset.as_numpy_iterator()) - [{'v1': array([[ 1, 2, 3, -1], - [ 4, 5, -1, -1]], dtype=int32), 'v2': array([[ 10, 100], - [ 11, 12]], dtype=int32)}] - + [(array([[ 1, 2, 3, -1], [ 4, 5, -1, -1]], dtype=int32), + array([[ 10, 100], [ 11, 12]], dtype=int32))] See also `tf.data.experimental.dense_to_sparse_batch`, which combines elements that may have different shapes into a `tf.SparseTensor`. diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py index f7afe995a07cf0..ff8d02f9d713b0 100644 --- a/tensorflow/python/data/ops/iterator_ops.py +++ b/tensorflow/python/data/ops/iterator_ops.py @@ -585,11 +585,6 @@ def __init__(self, dataset=None, components=None, element_spec=None): self._flat_output_shapes = structure.get_flat_tensor_shapes( self._element_spec) self._iterator_resource, self._deleter = components - # Delete the resource when this object is deleted - self._resource_deleter = IteratorResourceDeleter( - handle=self._iterator_resource, - device=self._device, - deleter=self._deleter) else: if (components is not None or element_spec is not None): raise ValueError(error_message) diff --git a/tensorflow/python/data/ops/multi_device_iterator_ops.py b/tensorflow/python/data/ops/multi_device_iterator_ops.py index 0eb3a95d5a1bbd..4ed39c2d2f64bf 100644 --- a/tensorflow/python/data/ops/multi_device_iterator_ops.py +++ b/tensorflow/python/data/ops/multi_device_iterator_ops.py @@ -425,7 +425,7 @@ def _serialize(self): def _component_specs(self): specs = [ tensor_spec.TensorSpec([], dtypes.resource), - tensor_spec.TensorSpec([], dtypes.scalar) + tensor_spec.TensorSpec([], dtypes.variant) ] for _ in range(len(self._devices)): specs.append(iterator_ops.IteratorSpec(self._element_spec)) @@ -565,11 +565,11 @@ def __init__(self, self._device_iterators.append(iterator) iterator_handles.append(iterator._iterator_resource) # pylint: disable=protected-access - self._resource_deleter = MultiDeviceIteratorResourceDeleter( - multi_device_iterator=self._multi_device_iterator_resource, - iterators=iterator_handles, - device=self._source_device, - deleter=self._deleter) + self._resource_deleter = MultiDeviceIteratorResourceDeleter( + multi_device_iterator=self._multi_device_iterator_resource, + iterators=iterator_handles, + device=self._source_device, + deleter=self._deleter) def get_next(self, device=None): """Returns the next element given a `device`, else returns all in a list.""" diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index 236f2fb63741e8..7eb9baac19b8e4 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -704,6 +704,35 @@ cuda_py_test( ], ) +cuda_py_test( + name = "distributed_callbacks_test", + size = "medium", + srcs = ["lib/distributed_callbacks_test.py"], + additional_deps = [ + ":check_numerics_callback", + ":debug_events_writer", + ":dumping_callback", + ":dumping_callback_test_lib", + "//third_party/py/numpy", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + "//tensorflow/python:variables", + "//tensorflow/python/distribute:combinations", + "//tensorflow/python/distribute:mirrored_strategy", + "//tensorflow/python/distribute:strategy_combinations", + "//tensorflow/python/keras", + ], + tags = [ + "guitar", + "multi_and_single_gpu", + "no_rocm", + "no_windows", # TODO(b/142475891): Enable this test on Windows. + "no_windows_gpu", # TODO(b/130551176) + ], + xla_enable_strict_auto_jit = False, # Node names are different with autojit +) + cuda_py_test( name = "dumping_callback_test", size = "medium", @@ -720,7 +749,7 @@ cuda_py_test( "//tensorflow/python:variables", "//tensorflow/python/keras", ], - shard_count = 6, + shard_count = 8, tags = [ "no_windows", # TODO(b/142475891): Enable this test on Windows. ], diff --git a/tensorflow/python/debug/lib/check_numerics_callback.py b/tensorflow/python/debug/lib/check_numerics_callback.py index ad28b2a43efa3d..c3de23b5607c03 100644 --- a/tensorflow/python/debug/lib/check_numerics_callback.py +++ b/tensorflow/python/debug/lib/check_numerics_callback.py @@ -87,6 +87,8 @@ b"Unpack", ) +_state = threading.local() + def limit_string_length(string, max_len=50): """Limit the length of input string. @@ -217,66 +219,69 @@ def _debug_summary(x): debug_event_pb2.TensorDebugMode.REDUCE_INF_NAN_THREE_SLOTS)) -def _check_numerics_callback(op_type, - inputs, - attrs, - outputs, - op_name=None, - graph=None): - """Eager-function unified callback for checking numerics.""" - del attrs, op_name # Unused - op_type_bytes = compat.as_bytes(op_type) - is_v1_graph_mode = not ops.executing_eagerly_outside_functions() - if (op_type_bytes in op_callbacks_common.OP_CALLBACK_SKIP_OPS or - op_type_bytes in SAFE_OPS): - return - if graph: - # Under graph mode. Insert check_numerics op. - instrumented_outputs = [] - for slot, output in enumerate(outputs): - if (output.dtype.is_floating and - (op_type_bytes, slot) not in IGNORE_OP_OUTPUTS): - checked_output = array_ops.check_numerics( - # TF v2 has automatic control dependencies added to stateful async - # ops, which allows us to run check_numerics asynchronously. - # In the above case we use debug_summary to reduce all output - # tensors asynchronously from the op being checked and then process - # the tensor summary with check_numerics. - output if is_v1_graph_mode else _debug_summary(output), - get_check_numerics_error_message( - slot, - len(outputs), - op_type, - output, - inputs, - graph=graph, - traceback=output.op.traceback)) - _CHECK_NUMERICS_INPUT_LOOKUP[graph][checked_output.name] = output - instrumented_outputs.append( - checked_output if is_v1_graph_mode else output) - else: - instrumented_outputs.append(output) - return instrumented_outputs - else: - if op_type_bytes == b"CheckNumerics": - # TODO(b/140334369): Remove this special casing logic once op_callback. - # automatically prevents infinite recursion in eager mode. - return - # Under eager mode. Eagerly execute check_numerics op. - for slot, output in enumerate(outputs): - if (output.dtype.is_floating and - (op_type_bytes, slot) not in IGNORE_OP_OUTPUTS): - array_ops.check_numerics( - output, - get_check_numerics_error_message( - slot, len(outputs), op_type, output, inputs, - stack_height_limit=_state.config.stack_height_limit, - path_length_limit=_state.config.path_length_limit)) - - -CheckNumericsConfig = collections.namedtuple( - "CheckNumericsConfig", "stack_height_limit path_length_limit") -_state = threading.local() +class CheckNumericsCallback(object): + """Wrapper for the numerics-checking callback for thread locality.""" + + def __init__(self, stack_height_limit, path_length_limit): + self._stack_height_limit = stack_height_limit + self._path_length_limit = path_length_limit + + def callback(self, + op_type, + inputs, + attrs, + outputs, + op_name=None, + graph=None): + """Eager-function unified callback for checking numerics.""" + del attrs, op_name # Unused + op_type_bytes = compat.as_bytes(op_type) + is_v1_graph_mode = not ops.executing_eagerly_outside_functions() + if (op_type_bytes in op_callbacks_common.OP_CALLBACK_SKIP_OPS or + op_type_bytes in SAFE_OPS): + return None + if graph: + # Under graph mode. Insert check_numerics op. + instrumented_outputs = [] + for slot, output in enumerate(outputs): + if (output.dtype.is_floating and + (op_type_bytes, slot) not in IGNORE_OP_OUTPUTS): + checked_output = array_ops.check_numerics( + # TF v2 has automatic control dependencies added to stateful async + # ops, which allows us to run check_numerics asynchronously. + # In the above case we use debug_summary to reduce all output + # tensors asynchronously from the op being checked and then + # process the tensor summary with check_numerics. + output if is_v1_graph_mode else _debug_summary(output), + get_check_numerics_error_message( + slot, + len(outputs), + op_type, + output, + inputs, + graph=graph, + traceback=output.op.traceback)) + _CHECK_NUMERICS_INPUT_LOOKUP[graph][checked_output.name] = output + instrumented_outputs.append( + checked_output if is_v1_graph_mode else output) + else: + instrumented_outputs.append(output) + return instrumented_outputs + else: + if op_type_bytes == b"CheckNumerics": + # TODO(b/140334369): Remove this special casing logic once op_callback. + # automatically prevents infinite recursion in eager mode. + return None + # Under eager mode. Eagerly execute check_numerics op. + for slot, output in enumerate(outputs): + if (output.dtype.is_floating and + (op_type_bytes, slot) not in IGNORE_OP_OUTPUTS): + array_ops.check_numerics( + output, + get_check_numerics_error_message( + slot, len(outputs), op_type, output, inputs, + stack_height_limit=self._stack_height_limit, + path_length_limit=self._path_length_limit)) @tf_export("debugging.enable_check_numerics") @@ -362,12 +367,10 @@ def square_log_x_plus_1(x): path_length_limit: Limit to the file path included in the printed stack trace. Applicable only to ops in `tf.function`s (graphs). """ - - if not hasattr(_state, "config"): - _state.config = CheckNumericsConfig( - stack_height_limit=stack_height_limit, - path_length_limit=path_length_limit) - op_callbacks.add_op_callback(_check_numerics_callback) + if not hasattr(_state, "check_numerics_callback"): + _state.check_numerics_callback = CheckNumericsCallback( + stack_height_limit, path_length_limit) + op_callbacks.add_op_callback(_state.check_numerics_callback.callback) logging.info( "Enabled check-numerics callback in thread %s", @@ -387,8 +390,11 @@ def disable_check_numerics(): This method takes effect only on the thread in which it is called. """ + if not hasattr(_state, "check_numerics_callback"): + return try: - op_callbacks.remove_op_callback(_check_numerics_callback) + op_callbacks.remove_op_callback(_state.check_numerics_callback.callback) + delattr(_state, "check_numerics_callback") logging.info( "Disabled check-numerics callback in thread %s", threading.current_thread().name) diff --git a/tensorflow/python/debug/lib/check_numerics_callback_test.py b/tensorflow/python/debug/lib/check_numerics_callback_test.py index 426ad946d743e9..50b97ef49a08f5 100644 --- a/tensorflow/python/debug/lib/check_numerics_callback_test.py +++ b/tensorflow/python/debug/lib/check_numerics_callback_test.py @@ -571,7 +571,6 @@ def testExpectedNaNOpOutputs(self): self.assertTrue(np.isnan(batch_mean.squeeze())) self.assertTrue(np.isnan(batch_variance.squeeze())) - # TODO(cais): Tests for Infs and NaNs during distributed execution. # TODO(cais): Benchmark the slowdown due to callbacks and inserted nodes. if __name__ == "__main__": diff --git a/tensorflow/python/debug/lib/distributed_callbacks_test.py b/tensorflow/python/debug/lib/distributed_callbacks_test.py new file mode 100644 index 00000000000000..89f54f4b5dd79c --- /dev/null +++ b/tensorflow/python/debug/lib/distributed_callbacks_test.py @@ -0,0 +1,344 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tfdbg op callbacks running with various `DistributionStrategy`s.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python import keras +from tensorflow.python.debug.lib import check_numerics_callback +from tensorflow.python.debug.lib import dumping_callback +from tensorflow.python.debug.lib import dumping_callback_test_lib +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from tensorflow.python.eager import backprop +from tensorflow.python.eager import def_function +from tensorflow.python.framework import errors +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import googletest +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.training import gradient_descent + + +def filter_by_device_name(items, device_names, target_device_name): + """Filter a list of items by device name. + + Args: + items: A list of items to be filtered according to their corresponding + device names. + device_names: A list of the device names. Must have the same legnth + as `items`. + target_device_name: A `str` representing the desired device name. + + Returns: + Filtered items from `items`. + """ + assert len(items) == len(device_names) + assert all(device_names), "device_names are not all non-empty strings" + # Note: we use `endswith` instead of `==` for device-name filtering because + # in some cases, the device names from kernel/op execution can have slightly + # different values than the device names from + # `distribution.extended.worker_devices`. + return [items[i] for i, device_name in enumerate(device_names) + if device_name.endswith(target_device_name)] + + +def filter_by_device_name_and_op_type( + items, device_names, op_types, target_device_name, target_op_type): + assert len(items) == len(device_names) + assert len(items) == len(op_types) + assert all(device_names), "device_names are not all non-empty strings" + assert all(op_types), "op_types are not all non-empty strings" + return [items[i] for i, device_name in enumerate(device_names) + if device_name.endswith(target_device_name) + and op_types[i] == target_op_type] + + +class MiniModel(keras.Model): + """Minimal subclassed Keras model.""" + + def __init__(self, generate_infinity=False): + super(MiniModel, self).__init__(name="") + self._generate_infinity = generate_infinity + self.fc = keras.layers.Dense( + 1, kernel_initializer="ones", bias_initializer="ones", + activation="linear") + + @def_function.function + def call(self, inputs, training=True): + y = self.fc(inputs) + if self._generate_infinity: + y = math_ops.divide(y, array_ops.zeros_like(y)) + return y + + +class DistributedDumpingCallbackTest( + dumping_callback_test_lib.DumpingCallbackTestBase, parameterized.TestCase): + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ], + inside_scope=[False, True], + # TODO(cais): Investigate that under V1 graph mode (mode="graph"), + # occasionally (~1-2% of time) the test runs into the following error: + # CancelledError: [_Derived_] Function was cancelled before it was + # started. + mode=["eager"], + )) + def testCheckingInfinityInMiniModelOnOneOrTwoDevices( + self, distribution, inside_scope): + if not inside_scope: + check_numerics_callback.enable_check_numerics() + with distribution.scope(): + if inside_scope: + check_numerics_callback.enable_check_numerics() + + mini_model = MiniModel(generate_infinity=True) + def train_step(): + with backprop.GradientTape() as tape: + loss = mini_model(array_ops.ones([1, 10])) + return tape.gradient(loss, mini_model.weights) + + caught_error = None + try: + distribution.experimental_run_v2(train_step) + except errors.InvalidArgumentError as error: + caught_error = error + self.assertTrue(caught_error) + self.assertTrue(re.search( + r"Detected Infinity or NaN.*\"RealDiv\"", caught_error.message)) + self.assertIn( + "-> | y = math_ops.divide(y, array_ops.zeros_like(y))", + caught_error.message) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ], + mode=["eager"], + tensor_debug_mode=["NO_TENSOR", "FULL_TENSOR"], + )) + def testDumpingMiniModel(self, distribution, tensor_debug_mode): + with distribution.scope(): + writer = dumping_callback.enable_dump_debug_info( + self.dump_root, tensor_debug_mode=tensor_debug_mode) + + mini_model = MiniModel() + optimizer = gradient_descent.GradientDescentOptimizer(0.25) + + def train_step(): + with backprop.GradientTape() as tape: + loss = mini_model(array_ops.ones([1, 10])) + grads = tape.gradient(loss, mini_model.weights) + grads_and_vars = zip(grads, mini_model.weights) + optimizer.apply_gradients(grads_and_vars) + + distribution.experimental_run_v2(train_step) + + updated_var_values = self.evaluate(mini_model.variables) + num_devices = len(distribution.extended.worker_devices) + assert num_devices in [1, 2] + # TODO(cais): We currently refrain from asserting the + # element-by-element values of the variable updates. The values seem to + # vary among builds. On some builds, it's 0.75; on others, it's 1.0. + # This variation is seen in the MirroredCPUAndGPU and OneDeviceGPU + # strategies. Needs investigation. + # if num_devices == 1: + # self.assertAllEqual(0.75 * np.ones([10, 1]), updated_var_values[0]) + # self.assertAllEqual([0.75], updated_var_values[1]). + # else: + # self.assertAllEqual(0.5 * np.ones([10, 1]), updated_var_values[0]) + # self.assertAllEqual([0.5], updated_var_values[1]) + self.assertEqual(updated_var_values[0].shape, (10, 1)) + self.assertEqual(updated_var_values[1].shape, (1,)) + + writer.FlushNonExecutionFiles() + writer.FlushExecutionFiles() + + stack_frame_by_id = self._readAndCheckSourceFilesAndStackFrames() + (context_ids, _, + op_name_to_op_type) = self._readAndCheckGraphsFile(stack_frame_by_id) + (op_names, device_names, _, + tensor_values) = self._readAndCheckGraphExecutionTracesFile(context_ids) + executed_op_types = [op_name_to_op_type[op_name] for op_name in op_names] + + device_name_0 = distribution.extended.worker_devices[0] + logging.info("device_name_0 = %s", device_name_0) + if num_devices > 1: + device_name_1 = distribution.extended.worker_devices[1] + logging.info("device_name_1 = %s", device_name_1) + + device_0_executed_op_types = filter_by_device_name( + executed_op_types, device_names, device_name_0) + if num_devices > 1: + device_1_executed_op_types = filter_by_device_name( + executed_op_types, device_names, device_name_1) + # Verify graph-execution traces are available for both devices. + # We don't assert MatMul occurs exactly once because the gradient of MatMul + # involves MatMul. + self.assertIn("MatMul", device_0_executed_op_types) + self.assertEqual(device_0_executed_op_types.count("BiasAdd"), 1) + if num_devices > 1: + self.assertIn("MatMul", device_1_executed_op_types) + self.assertEqual(device_1_executed_op_types.count("BiasAdd"), 1) + + if tensor_debug_mode == "NO_TENSOR": + for value_list in tensor_values: + for tensor_value in value_list: + self.assertEqual(tensor_value.dtype, np.float32) + self.assertEqual(tensor_value.shape, []) + elif tensor_debug_mode == "FULL_TENSOR": + device_0_matmul_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_0, + "MatMul") + device_0_bias_add_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_0, + "BiasAdd") + self.assertAllClose(device_0_matmul_values[0], [[10.0]]) + self.assertAllClose(device_0_bias_add_values[0], [[11.0]]) + if num_devices > 1: + device_1_matmul_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_1, + "MatMul") + device_1_bias_add_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_1, + "BiasAdd") + self.assertAllClose(device_1_matmul_values[0], [[10.0]]) + self.assertAllClose(device_1_bias_add_values[0], [[11.0]]) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ], + mode=["eager"], + tensor_debug_mode=["NO_TENSOR", "FULL_TENSOR"], + )) + def testKerasModelFitOnOneOrTwoDevices(self, distribution, tensor_debug_mode): + writer = dumping_callback.enable_dump_debug_info( + self.dump_root, tensor_debug_mode=tensor_debug_mode) + + with distribution.scope(): + model = keras.Sequential() + model.add(keras.layers.Dense( + units=10, input_shape=[5], activation="relu")) + model.add(keras.layers.Dense(units=1)) + model.compile(loss="mse", optimizer="sgd") + + batch_size = 20 + x = np.ones([batch_size, 5]) + y = np.ones([batch_size, 1]) + epochs = 1 + history = model.fit(x, y, epochs=epochs, verbose=0) + self.assertLen(history.history["loss"], epochs) + + writer.FlushNonExecutionFiles() + writer.FlushExecutionFiles() + + stack_frame_by_id = self._readAndCheckSourceFilesAndStackFrames() + (context_ids, _, + op_name_to_op_type) = self._readAndCheckGraphsFile(stack_frame_by_id) + (op_names, device_names, _, + tensor_values) = self._readAndCheckGraphExecutionTracesFile(context_ids) + + # Eager execution of tf.function should be recorded. + executed_op_types, _, _, _, _ = self._readAndCheckExecutionFile() + fit_functions = [op_type for op_type in executed_op_types + if "_distributed_function" in op_type] + self.assertLen(fit_functions, epochs) + + num_devices = len(distribution.extended.worker_devices) + + device_name_0 = distribution.extended.worker_devices[0] + logging.info("device_name_0 = %s", device_name_0) + if num_devices > 1: + device_name_1 = distribution.extended.worker_devices[1] + logging.info("device_name_1 = %s", device_name_1) + + executed_op_types = [op_name_to_op_type[op_name] for op_name in op_names] + device_0_executed_op_types = filter_by_device_name( + executed_op_types, device_names, device_name_0) + if num_devices > 1: + device_1_executed_op_types = filter_by_device_name( + executed_op_types, device_names, device_name_1) + + self.assertIn("MatMul", device_0_executed_op_types) + self.assertIn("BiasAdd", device_0_executed_op_types) + self.assertIn("Relu", device_0_executed_op_types) + self.assertIn("ReluGrad", device_0_executed_op_types) + if num_devices > 1: + # If there are two devices involved, assert the ops inside tf.functions + # are executed and recorded for the equal numbers of times by the + # dumping op-callback. + self.assertEqual(device_0_executed_op_types.count("MatMul"), + device_1_executed_op_types.count("MatMul")) + self.assertEqual(device_0_executed_op_types.count("BiasAdd"), + device_1_executed_op_types.count("BiasAdd")) + self.assertEqual(device_0_executed_op_types.count("Relu"), + device_1_executed_op_types.count("Relu")) + self.assertEqual(device_0_executed_op_types.count("ReluGrad"), + device_1_executed_op_types.count("ReluGrad")) + + if tensor_debug_mode == "NO_TENSOR": + for value_list in tensor_values: + for tensor_value in value_list: + self.assertEqual(tensor_value.dtype, np.float32) + self.assertEqual(tensor_value.shape, []) + elif tensor_debug_mode == "FULL_TENSOR": + gpu_0_relu_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_0, "Relu") + self.assertTrue(gpu_0_relu_values) + gpu_0_relu_grad_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_0, + "ReluGrad") + self.assertTrue(gpu_0_relu_grad_values) + if num_devices > 1: + gpu_1_relu_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_1, + "Relu") + self.assertTrue(gpu_1_relu_values) + for i in range(len(gpu_0_relu_values)): + self.assertEqual(gpu_0_relu_values[i].shape, + gpu_1_relu_values[i].shape) + gpu_1_relu_grad_values = filter_by_device_name_and_op_type( + tensor_values, device_names, executed_op_types, device_name_1, + "ReluGrad") + self.assertTrue(gpu_1_relu_grad_values) + for i in range(len(gpu_0_relu_grad_values)): + self.assertEqual( + gpu_0_relu_grad_values[i].shape, gpu_1_relu_grad_values[i].shape) + + +if __name__ == "__main__": + googletest.main() diff --git a/tensorflow/python/debug/lib/dumping_callback.py b/tensorflow/python/debug/lib/dumping_callback.py index 1d0dc7fe8635fb..2bd471a5493d66 100644 --- a/tensorflow/python/debug/lib/dumping_callback.py +++ b/tensorflow/python/debug/lib/dumping_callback.py @@ -19,7 +19,6 @@ from __future__ import print_function import atexit -import collections import re import socket import threading @@ -42,11 +41,8 @@ from tensorflow.python.util import tf_stack from tensorflow.python.util.tf_export import tf_export -DumpingConfig = collections.namedtuple( - "DumpingConfig", - "dump_root tensor_debug_mode circular_buffer_size " - "op_regex tensor_dtypes") _state = threading.local() +DEFAULT_TENSOR_DEBUG_MODE = "NO_TENSOR" @ops.RegisterGradient("DebugIdentityV2") @@ -56,272 +52,340 @@ def _debug_identity_v2_grad(op, dy): return dy -def _get_writer(): - """Get the debug events writer for the currently configured dump root.""" - # TODO(cais): Explore caching the object for possible performance gain. - # TODO(cais): Rename circular_buffer_size to circular_buffer_size in C++ and - # Python-bindng code. - return debug_events_writer.DebugEventsWriter( - _state.config.dump_root, - circular_buffer_size=_state.config.circular_buffer_size) - - def _get_id(): """Get a short unique ID.""" return str(uuid.uuid4()) -def _get_context_id(context): - """Get a unique ID for an op-construction context (e.g., a graph). - - If the graph has been encountered before, reuse the same unique ID. - - Args: - context: A context to get the unique ID for. Must be hashable. E.g., a Graph - object. - - Returns: - A unique ID for the context. - """ - if context not in _state.context_to_id: - _state.context_to_id[context] = _get_id() - return _state.context_to_id[context] +class _DumpingCallback(object): + """An object holding the states surrouding the dumping callback.""" + def __init__(self, + dump_root, + tensor_debug_mode, + circular_buffer_size, + op_regex, + tensor_dtypes): + self._dump_root = dump_root + self._tensor_debug_mode = tensor_debug_mode + self._circular_buffer_size = circular_buffer_size + self._op_regex = op_regex + self._tensor_dtypes = tensor_dtypes -def _write_source_file_content(file_path): - """Send the content of a source file via debug-events writer. - - Args: - file_path: Path to the source file. - - Returns: - An int index for the file. - """ - if file_path not in _state.source_file_paths: - lines = None - if source_utils.is_extension_uncompiled_python_source(file_path): - try: - lines, _ = source_utils.load_source(file_path) - except IOError: - # Accept the fact that some source files are not readable. Here we use - # best effort to send the source-file contents. - pass - writer = _get_writer() - writer.WriteSourceFile(debug_event_pb2.SourceFile( - file_path=file_path, host_name=_state.hostname, lines=lines)) - _state.source_file_paths.append(file_path) - return _state.source_file_paths.index(file_path) - - -def _process_stack_frames(): - """Process stack frames. - - Send the content of source-files, on a best-effort basis. - - Returns: - A list of stack frame IDs. - """ - stack_frames = tf_stack.extract_stack() - stack_frame_ids = [] - writer = None - for file_path, lineno, func, _ in stack_frames: - if (file_path, lineno, func) not in _state.stack_frame_to_id: - stack_frame_id = _get_id() - _state.stack_frame_to_id[(file_path, lineno, func)] = stack_frame_id - file_index = _write_source_file_content(file_path) - file_line_col = graph_debug_info_pb2.GraphDebugInfo.FileLineCol( - file_index=file_index, line=lineno, func=func) - stack_frame_with_id = debug_event_pb2.StackFrameWithId( - id=stack_frame_id, file_line_col=file_line_col) - writer = _get_writer() - writer.WriteStackFrameWithId(stack_frame_with_id) - stack_frame_ids.append(_state.stack_frame_to_id[(file_path, lineno, func)]) - - code_location = debug_event_pb2.CodeLocation( - host_name=_state.hostname, stack_frame_ids=stack_frame_ids) - return code_location - - -def _should_dump_tensor(op_type, dtype): - """Determine if the given tensor's value will be dumped. - - The determination is made given the configurations such as `op_regex`, - `tensor_dtypes`. - - Args: - op_type: Name of the op's type, as a string (e.g., "MatMul"). - dtype: The dtype of the tensor, as a `dtypes.DType` object. - - Returns: - A bool indicating whether the tensor's value will be dumped. - """ - should_dump = True - if _state.config.op_regex: - should_dump = (should_dump and - re.match(_state.config.op_regex, op_type)) - if _state.config.tensor_dtypes: - if isinstance(_state.config.tensor_dtypes, (list, tuple)): - should_dump = (should_dump and - any(dtype == dtype_item for dtype_item - in _state.config.tensor_dtypes)) - else: # A callable that takes a DType argument and return a boolean. - should_dump = should_dump and _state.config.tensor_dtypes(dtype) - return should_dump - - -def _instrument_symbolic_tensors(tensors, op_type, op_name, tfdbg_context_id): - """Add debugging instrumentation for symbolic (i.e., non-eager) tensors. - - The detailed fashion in which the tensors are instrumented is determined - by the tensor_debug_mode configured for the currently enabled dumping - callback. - - Args: - tensors: A tuple of Tensors to instrument. It is assumed that their ordering - corresponds to the ordering of output tensors of an original op. Output - slot indices (0-based) will be generated based on the ordering. - op_type: Name of the op type of the node that emits `tensors` (e.g., - "MatMul"), as a string. - op_name: Name of the node that emits `tensors` (e.g., "dense_1/MatMul"), as - a string. - tfdbg_context_id: A unique ID for the context that the op belongs to (e.g., - a graph). - - Returns: - Non-eager Tensors that override the `tensors` as the output of the op - that originally generated `tensors`. In some cases (e.g., non-V1 graph - mode), this may be `None`, as the instrumentation can simply rely on - automatic control dependencies (see `auto_control_deps.py`) instead of - tensor overriding. - """ - tensor_debug_mode = _state.config.tensor_debug_mode - debug_urls = ["file://%s" % _state.config.dump_root] - is_v1_graph_mode = not ops.executing_eagerly_outside_functions() - instrumented_tensors = [] if is_v1_graph_mode else None - for output_slot, tensor in enumerate(tensors): - if not _should_dump_tensor(op_type, tensor.dtype): - if is_v1_graph_mode: - instrumented_tensors.append(tensor) - continue - + self._hostname = socket.gethostname() + # A list of source-file paths. + self._source_file_paths = [] + # A map from stack frame (FileLineCol) to unique ID. + self._stack_frame_to_id = dict() + # Mapping op context to unique ID. + self._context_to_id = dict() + self._source_file_paths_lock = threading.Lock() + self._stack_frame_to_id_lock = threading.Lock() + self._context_to_id_lock = threading.Lock() + self._writer = None + + @property + def dump_root(self): + return self._dump_root + + @dump_root.setter + def dump_root(self, dump_root): + if self._dump_root != dump_root: + self._dump_root = dump_root + self._writer = None + + @property + def tensor_debug_mode(self): + return self._tensor_debug_mode + + @property + def circular_buffer_size(self): + return self._circular_buffer_size + + def get_writer(self): + """Get the debug events writer for the currently configured dump root.""" + if not self._writer: + self._writer = debug_events_writer.DebugEventsWriter( + self._dump_root, + circular_buffer_size=self._circular_buffer_size) + return self._writer + + def _get_context_id(self, context): + """Get a unique ID for an op-construction context (e.g., a graph). + + If the graph has been encountered before, reuse the same unique ID. + + Args: + context: A context to get the unique ID for. Must be hashable. E.g., a + Graph object. + + Returns: + A unique ID for the context. + """ + # Use the double-checked lock pattern to optimize the common case. + if context in self._context_to_id: # 1st check, without lock. + return self._context_to_id[context] + with self._context_to_id_lock: + if context not in self._context_to_id: # 2nd check, with lock. + self._context_to_id[context] = _get_id() + return self._context_to_id[context] + + def _write_source_file_content(self, file_path): + """Send the content of a source file via debug-events writer. + + Args: + file_path: Path to the source file. + + Returns: + An int index for the file. + """ + if file_path in self._source_file_paths: + return self._source_file_paths.index(file_path) + with self._source_file_paths_lock: + if file_path not in self._source_file_paths: + lines = None + if source_utils.is_extension_uncompiled_python_source(file_path): + try: + lines, _ = source_utils.load_source(file_path) + except IOError: + # Accept the fact that some source files are not readable. Here we + # use best effort to send the source-file contents. + pass + writer = self.get_writer() + writer.WriteSourceFile(debug_event_pb2.SourceFile( + file_path=file_path, host_name=self._hostname, lines=lines)) + self._source_file_paths.append(file_path) + return self._source_file_paths.index(file_path) + + def _process_stack_frames(self): + """Process stack frames. + + Send the content of source-files, on a best-effort basis. + + Returns: + A list of stack frame IDs. + """ + stack_frames = tf_stack.extract_stack() + stack_frame_ids = [] + writer = None + for file_path, lineno, func, _ in stack_frames: + if (file_path, lineno, func) in self._stack_frame_to_id: + stack_frame_ids.append( + self._stack_frame_to_id[(file_path, lineno, func)]) + continue + with self._stack_frame_to_id_lock: + if (file_path, lineno, func) not in self._stack_frame_to_id: + stack_frame_id = _get_id() + self._stack_frame_to_id[(file_path, lineno, func)] = stack_frame_id + file_index = self._write_source_file_content(file_path) + file_line_col = graph_debug_info_pb2.GraphDebugInfo.FileLineCol( + file_index=file_index, line=lineno, func=func) + stack_frame_with_id = debug_event_pb2.StackFrameWithId( + id=stack_frame_id, file_line_col=file_line_col) + writer = self.get_writer() + writer.WriteStackFrameWithId(stack_frame_with_id) + stack_frame_ids.append( + self._stack_frame_to_id[(file_path, lineno, func)]) + + code_location = debug_event_pb2.CodeLocation( + host_name=self._hostname, stack_frame_ids=stack_frame_ids) + return code_location + + def _instrument_symbolic_tensors(self, + tensors, + op_type, + op_name, + tfdbg_context_id): + """Add debugging instrumentation for symbolic (i.e., non-eager) tensors. + + The detailed fashion in which the tensors are instrumented is determined + by the tensor_debug_mode configured for the currently enabled dumping + callback. + + Args: + tensors: A tuple of Tensors to instrument. It is assumed that their + ordering corresponds to the ordering of output tensors of an original + op. Output slot indices (0-based) will be generated based on the + ordering. + op_type: Type name of the op that emits the Tensors (e.g., "MatMul"). + op_name: Name of the op that emits the Tensors (e.g., "dense_1/MatMul"). + tfdbg_context_id: A unique ID for the context that the op belongs to + (e.g., a graph). + + Returns: + Non-eager Tensors that override the `tensors` as the output of the op + that originally generated `tensors`. In some cases (e.g., non-V1 graph + mode), this may be `None`, as the instrumentation can simply rely on + automatic control dependencies (see `auto_control_deps.py`) instead of + tensor overriding. + """ + tensor_debug_mode = self._tensor_debug_mode + debug_urls = ["file://%s" % self._dump_root] + is_v1_graph_mode = not ops.executing_eagerly_outside_functions() + instrumented_tensors = [] if is_v1_graph_mode else None if tensor_debug_mode == debug_event_pb2.TensorDebugMode.NO_TENSOR: - # Except in V1 graph mode + control flow, debug_identity_v2 trigger auto - # control dependency because it's a stateful op. - debug_tensor = gen_debug_ops.debug_identity_v2( - # Use an empty (shape=[0]) float32 tensor for the NO_TENSOR mode - # as a low-overhead placeholder, since no actual tensor value is - # traced. - constant_op.constant([], dtype=dtypes.float32), - tfdbg_context_id=tfdbg_context_id, - op_name=op_name, - output_slot=output_slot, - tensor_debug_mode=_state.config.tensor_debug_mode, - debug_urls=debug_urls) - if is_v1_graph_mode: - # TODO(cais): Evaluate performance optimization options. For the - # `NO_TENSOR` debug mode, an alternative is to add `debug_tensor` as a - # control dependency of `tensor.op` without an additional identity op. - identity = array_ops.identity(tensor) - identity.op._add_control_input( # pylint: disable=protected-access - debug_tensor.op) - instrumented_tensors.append(identity) + for output_slot, tensor in enumerate(tensors): + if (not self._should_dump_tensor(op_type, tensor.dtype) or + not tensor.dtype.is_numpy_compatible): + # Instrumenting DT_VARIANT and DT_RESOURCE type tensors under + # V1 graph mode is known to have issues. TODO(cais): Investigate. + if is_v1_graph_mode: + instrumented_tensors.append(tensor) + continue + if is_v1_graph_mode and not tensor.dtype.is_numpy_compatible: + + instrumented_tensors.append(tensor) + continue + # Except in V1 graph mode + control flow, debug_identity_v2 trigger auto + # control dependency because it's a stateful op. + debug_tensor = gen_debug_ops.debug_identity_v2( + # Use an empty (shape=[0]) float32 tensor for the NO_TENSOR mode + # as a low-overhead placeholder, since no actual tensor value is + # traced. + constant_op.constant([], dtype=dtypes.float32), + tfdbg_context_id=tfdbg_context_id, + op_name=op_name, + output_slot=output_slot, + tensor_debug_mode=self._tensor_debug_mode, + debug_urls=debug_urls) + if is_v1_graph_mode: + # TODO(cais): Evaluate performance optimization options. For the + # `NO_TENSOR` debug mode, an alternative is to add `debug_tensor` as a + # control dependency of `tensor.op` without an additional identity op. + identity = array_ops.identity(tensor) + identity.op._add_control_input( # pylint: disable=protected-access + debug_tensor.op) + instrumented_tensors.append(identity) + return instrumented_tensors elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.FULL_TENSOR: - debug_tensor = gen_debug_ops.debug_identity_v2( - tensor, - tfdbg_context_id=tfdbg_context_id, - op_name=op_name, - output_slot=output_slot, - tensor_debug_mode=_state.config.tensor_debug_mode, - debug_urls=debug_urls) - if is_v1_graph_mode: - instrumented_tensors.append(debug_tensor) + for output_slot, tensor in enumerate(tensors): + if (not self._should_dump_tensor(op_type, tensor.dtype) or + not tensor.dtype.is_numpy_compatible): + # Instrumenting DT_VARIANT and DT_RESOURCE type tensors under + # V1 graph mode is known to have issues. TODO(cais): Investigate. + if is_v1_graph_mode: + instrumented_tensors.append(tensor) + continue + debug_tensor = gen_debug_ops.debug_identity_v2( + tensor, + tfdbg_context_id=tfdbg_context_id, + op_name=op_name, + output_slot=output_slot, + tensor_debug_mode=self._tensor_debug_mode, + debug_urls=debug_urls) + if is_v1_graph_mode: + instrumented_tensors.append(debug_tensor) + return instrumented_tensors else: raise NotImplementedError( - "Symbolic tensor instrumentation is not implemented for debug " - "mode %s" % _state.config.tensor_debug_mode) - return instrumented_tensors - + "Symbolic tensor instrumentation is not implemented for debug mode " + "%s" % self._tensor_debug_mode) + + def _dump_eager_tensors(self, tensors, op_type, input_tensor_ids): + """Dump the value of eager tensors. + + The destination of the dumping is determined by the dump_root of the + currently enabled dumping callback. The tensors may be transformed prior to + dumping (e.g., reduced as summary statistics such as minimum, maximum and + arithmetic mean). The details of this transformation (if any) depends on + the tensor_debug_mode of the currently enabled dumping callback. + + Args: + tensors: The EagerTensors whose values are to be dumped, with or without + value transform. + op_type: Type of the op that generates the tensors, as a string. + input_tensor_ids: IDs of the input EagerTensors to the op. + + Returns: + A tfdbg Execution protocol buffer. + """ + tensor_debug_mode = self._tensor_debug_mode + output_tensor_ids = [ + t._id for t in tensors] # pylint:disable=protected-access + if tensor_debug_mode == debug_event_pb2.TensorDebugMode.NO_TENSOR: + return debug_event_pb2.Execution( + op_type=op_type, + num_outputs=len(tensors), + input_tensor_ids=input_tensor_ids, + output_tensor_ids=output_tensor_ids, + tensor_debug_mode=tensor_debug_mode, + code_location=self._process_stack_frames()) + elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.FULL_TENSOR: + execution_proto = debug_event_pb2.Execution( + op_type=op_type, + num_outputs=len(tensors), + input_tensor_ids=input_tensor_ids, + output_tensor_ids=output_tensor_ids, + tensor_debug_mode=tensor_debug_mode, + code_location=self._process_stack_frames()) + for tensor in tensors: + if (self._should_dump_tensor(op_type, tensor.dtype) and + tensor.dtype.is_numpy_compatible): + execution_proto.tensor_protos.append( + tensor_util.make_tensor_proto(tensor.numpy())) + return execution_proto + else: + raise NotImplementedError( + "Tensor instrumentation is not implemented for debug mode %s yet " % + self._tensor_debug_mode) + + def callback(self, + op_type, + inputs, + attrs, + outputs, + op_name=None, + graph=None): + """Op callback for tracing (dumping) a TF program's execution.""" + del attrs # Unused + + writer = self.get_writer() + if graph: + context_id = self._get_context_id(graph) + assert op_name is not None + graph_op_creation = debug_event_pb2.GraphOpCreation( + op_type=op_type, + op_name=op_name, + graph_name=graph.name if hasattr(graph, "name") else None, + graph_id=context_id, + input_names=[input_tensor.name for input_tensor in inputs], + num_outputs=len(outputs), + code_location=self._process_stack_frames()) + writer.WriteGraphOpCreation(graph_op_creation) + if outputs and compat.as_bytes( + op_type) not in op_callbacks_common.OP_CALLBACK_SKIP_OPS: + return self._instrument_symbolic_tensors( + outputs, op_type, op_name, context_id) + else: + input_ids = [t._id for t in inputs] # pylint:disable=protected-access + writer.WriteExecution( + self._dump_eager_tensors(outputs, op_type, input_ids)) -def _dump_eager_tensors(tensors, op_type, input_tensor_ids): - """Dump the value of eager tensors. + def _should_dump_tensor(self, op_type, dtype): + """Determine if the given tensor's value will be dumped. - The destination of the dumping is determined by the dump_root of the currently - enabled dumping callback. The tensors may be transformed prior to dumping - (e.g., reduced as summary statistics such as minimum, maximum and arithmetic - mean). The details of this transformation (if any) depends on the - tensor_debug_mode of the currently enabled dumping callback. + The determination is made given the configurations such as `op_regex`, + `tensor_dtypes`. - Args: - tensors: The EagerTensors whose values are to be dumped, with or without - value transform. - op_type: Type of the op that generates the tensors, as a string. - input_tensor_ids: IDs of the input EagerTensors to the op. + Args: + op_type: Name of the op's type, as a string (e.g., "MatMul"). + dtype: The dtype of the tensor, as a `dtypes.DType` object. - Returns: - A tfdbg Execution protocol buffer. - """ - tensor_debug_mode = _state.config.tensor_debug_mode - output_tensor_ids = [ - t._id for t in tensors] # pylint:disable=protected-access - if tensor_debug_mode == debug_event_pb2.TensorDebugMode.NO_TENSOR: - return debug_event_pb2.Execution( - op_type=op_type, - num_outputs=len(tensors), - input_tensor_ids=input_tensor_ids, - output_tensor_ids=output_tensor_ids, - tensor_debug_mode=tensor_debug_mode, - code_location=_process_stack_frames()) - elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.FULL_TENSOR: - execution_proto = debug_event_pb2.Execution( - op_type=op_type, - num_outputs=len(tensors), - input_tensor_ids=input_tensor_ids, - output_tensor_ids=output_tensor_ids, - tensor_debug_mode=tensor_debug_mode, - code_location=_process_stack_frames()) - for tensor in tensors: - if (_should_dump_tensor(op_type, tensor.dtype) and - tensor.dtype.is_numpy_compatible): - execution_proto.tensor_protos.append( - tensor_util.make_tensor_proto(tensor.numpy())) - return execution_proto - else: - raise NotImplementedError( - "Tensor instrumentation is not implemented for debug mode %s yet " % - _state.config.tensor_debug_mode) - - -def _dumping_callback(op_type, - inputs, - attrs, - outputs, - op_name=None, - graph=None): - """Op callback for tracing a TF program's execution.""" - del attrs # Unused - - writer = _get_writer() - if graph: - context_id = _get_context_id(graph) - assert op_name is not None - graph_op_creation = debug_event_pb2.GraphOpCreation( - op_type=op_type, - op_name=op_name, - graph_name=graph.name if hasattr(graph, "name") else None, - graph_id=context_id, - input_names=[input_tensor.name for input_tensor in inputs], - num_outputs=len(outputs), - code_location=_process_stack_frames()) - writer.WriteGraphOpCreation(graph_op_creation) - if outputs and compat.as_bytes( - op_type) not in op_callbacks_common.OP_CALLBACK_SKIP_OPS: - return _instrument_symbolic_tensors(outputs, op_type, op_name, context_id) - else: - input_ids = [t._id for t in inputs] # pylint:disable=protected-access - writer.WriteExecution(_dump_eager_tensors(outputs, op_type, input_ids)) - - -DEFAULT_TENSOR_DEBUG_MODE = "NO_TENSOR" + Returns: + A bool indicating whether the tensor's value will be dumped. + """ + should_dump = True + if self._op_regex: + should_dump = (should_dump and + re.match(self._op_regex, op_type)) + if self._tensor_dtypes: + if isinstance(self._tensor_dtypes, (list, tuple)): + should_dump = (should_dump and + any(dtype == dtype_item for dtype_item + in self._tensor_dtypes)) + else: # A callable that takes a DType argument and return a boolean. + should_dump = should_dump and self._tensor_dtypes(dtype) + return should_dump @tf_export("debugging.experimental.enable_dump_debug_info") @@ -416,6 +480,8 @@ def enable_dump_debug_info(dump_root, # TODO(cais): Revise the "UIs (currently under construction)" part of the doc # string above. # TODO(cais): Add Python code example to the doc string above. + global _state + tensor_debug_mode_keys = debug_event_pb2.TensorDebugMode.keys() if tensor_debug_mode not in tensor_debug_mode_keys: raise ValueError( @@ -429,25 +495,6 @@ def enable_dump_debug_info(dump_root, "tfdbg dumping: support for tensor debug mode %s is not " "implemented yet" % tensor_debug_mode) - if (hasattr(_state, "config") and - _state.config.circular_buffer_size != circular_buffer_size): - raise ValueError( - "There is already a dumping callback configured with a different " - "circular-buffer size (%d). Therefore the newly request " - "circular-buffer size (%d) will not be honored." % - (_state.config.circular_buffer_size, circular_buffer_size)) - - if (hasattr(_state, "config") and - _state.config.tensor_debug_mode != tensor_debug_mode): - raise ValueError( - "There is already a dumping callback configured for dump root " - "%s with a different " - "tensor-debug mode (%s). Therefore the newly request " - "tensor-debug mode (%s) size will not be honored." % - (_state.config.dump_root, - tensor_debug_mode_keys[_state.config.tensor_debug_mode], - tensor_debug_mode_keys[tensor_debug_mode])) - # Validate the types of tensor_dtypes. if tensor_dtypes is not None: if (not isinstance(tensor_dtypes, (list, tuple)) and @@ -460,30 +507,41 @@ def enable_dump_debug_info(dump_root, tensor_dtypes = [ dtypes.as_dtype(dtype_item) for dtype_item in tensor_dtypes] - if not hasattr(_state, "config") or _state.config.dump_root != dump_root: - _state.config = DumpingConfig( - dump_root=dump_root, - tensor_debug_mode=tensor_debug_mode, - circular_buffer_size=int(circular_buffer_size), - op_regex=re.compile(op_regex) if op_regex else None, - tensor_dtypes=tensor_dtypes) - _state.hostname = socket.gethostname() - # A list of source-file paths. - _state.source_file_paths = [] - # A map from stack frame (FileLineCol) to unique ID. - _state.stack_frame_to_id = dict() - # Mapping op context to unique ID. - _state.context_to_id = dict() + if hasattr(_state, "dumping_callback"): + if _state.dumping_callback.circular_buffer_size != circular_buffer_size: + raise ValueError( + "There is already a dumping callback configured with a different " + "circular-buffer size (%d). Therefore the newly request " + "circular-buffer size (%d) will not be honored." % + (_state.dumping_callback.circular_buffer_size, circular_buffer_size)) + if _state.dumping_callback.tensor_debug_mode != tensor_debug_mode: + raise ValueError( + "There is already a dumping callback configured for dump root " + "%s with a different " + "tensor-debug mode (%s). Therefore the newly request " + "tensor-debug mode (%s) size will not be honored." % + (_state.dumping_callback.dump_root, + tensor_debug_mode_keys[_state.dumping_callback.tensor_debug_mode], + tensor_debug_mode_keys[tensor_debug_mode])) + else: + _state.dumping_callback = _DumpingCallback(dump_root, + tensor_debug_mode, + circular_buffer_size, + op_regex, + tensor_dtypes) + op_callbacks.add_op_callback(_state.dumping_callback.callback) + + if _state.dumping_callback.dump_root != dump_root: + _state.dumping_callback.dump_root = dump_root - op_callbacks.add_op_callback(_dumping_callback) logging.info( "Enabled dumping callback in thread %s " "(dump root: %s, tensor debug mode: %s)", - threading.current_thread().name, _state.config.dump_root, - tensor_debug_mode) + threading.current_thread().name, + _state.dumping_callback.dump_root, tensor_debug_mode) atexit.register(disable_dump_debug_info) - return _get_writer() + return _state.dumping_callback.get_writer() @tf_export("debugging.experimental.disable_dump_debug_info") @@ -495,10 +553,10 @@ def disable_dump_debug_info(): `enable_dump_debug_info()` has been made, calling this method is a no-op. Calling this method more than once is idempotent. """ - if hasattr(_state, "config"): - dump_root = _state.config.dump_root - delattr(_state, "config") + if hasattr(_state, "dumping_callback"): + dump_root = _state.dumping_callback.dump_root debug_events_writer.DebugEventsWriter(dump_root).Close() - op_callbacks.remove_op_callback(_dumping_callback) + op_callbacks.remove_op_callback(_state.dumping_callback.callback) + delattr(_state, "dumping_callback") logging.info("Disabled dumping callback in thread %s (dump root: %s)", threading.current_thread().name, dump_root) diff --git a/tensorflow/python/debug/lib/dumping_callback_test.py b/tensorflow/python/debug/lib/dumping_callback_test.py index 25115589a0e2f7..8cc0242c062ccb 100644 --- a/tensorflow/python/debug/lib/dumping_callback_test.py +++ b/tensorflow/python/debug/lib/dumping_callback_test.py @@ -219,8 +219,9 @@ def sin1p_log_sum(x, y): # Session.run() in v1 graph mode, so doesn't get logged to the # .execution file. executed_op_types, _, _, _, _ = self._readAndCheckExecutionFile() + executed_op_types = [op_type for op_type in executed_op_types + if "sin1p_log_sum" in op_type] self.assertLen(executed_op_types, 1) - self.assertIn("sin1p_log_sum", executed_op_types[0]) stack_frame_by_id = self._readAndCheckSourceFilesAndStackFrames() (context_ids, op_types, @@ -601,7 +602,8 @@ def testDisableTracingWorks(self, tensor_debug_mode): ("NoTensor", "NO_TENSOR"), ("FullTensor", "FULL_TENSOR"), ) - def testMultiThreadedExecution(self, tensor_debug_mode): + def testMultiThreadedExecutionWithSameSetting(self, tensor_debug_mode): + """Dumping from multiple threads using the same setting.""" writer = dumping_callback.enable_dump_debug_info( self.dump_root, tensor_debug_mode=tensor_debug_mode) x = variables.Variable(10.0, dtype=dtypes.float32) @@ -658,6 +660,64 @@ def increase_x(): ] self.assertAllClose(mul_values, [6.0, 6.0, 6.0, 6.0]) + def testMultiThreadedDumpingWithDifferentSettings(self): + dump_root_1 = os.path.join(self.dump_root, "dump_root_1") + dump_root_2 = os.path.join(self.dump_root, "dump_root_2") + v1 = variables.Variable(10.0, dtype=dtypes.float32) + v2 = variables.Variable(3.0, dtype=dtypes.float32) + + def add_negative_v1_squared_to_itself(): + writer = dumping_callback.enable_dump_debug_info( + dump_root_1, tensor_debug_mode="FULL_TENSOR") + # Run in a loop to facilitate interleaving between threads. + for _ in range(3): + v1.assign_add(-(v1 ** 2.0)) + writer.FlushNonExecutionFiles() + writer.FlushExecutionFiles() + + def add_negative_v2_squared_to_itself(): + writer = dumping_callback.enable_dump_debug_info( + dump_root_2, tensor_debug_mode="FULL_TENSOR") + v2_squared = v2 ** 2.0 + # Since dumping is disabled before the Neg op is called, no tensor data + # should be dumped from the op, but this shouldn't affect the dumping of + # the tensor data from the Neg op in `add_negative_v1_squared_to_itself`. + # Both behavior is checked below. + dumping_callback.disable_dump_debug_info() + negative_v2_squared = -v2_squared + v2.assign_add(negative_v2_squared) + writer.FlushNonExecutionFiles() + writer.FlushExecutionFiles() + + # v2 is mutated on a sub-thread. + sub_thread = threading.Thread(target=add_negative_v2_squared_to_itself) + sub_thread.start() + add_negative_v1_squared_to_itself() # v1 is mutated on the main thread. + sub_thread.join() + # 10 - 10 * 10 = -90. + # -90 - (-90 * -90) = -8190. + # -8190 - (-8190 * -8190) = -67084290. + self.assertAllClose(v1.read_value(), -67084290.0) + self.assertAllClose(v2.read_value(), -6.0) + + (executed_op_types, _, _, _, + tensor_values) = self._readAndCheckExecutionFile(dump_root=dump_root_1) + v1_squared_values = [ + tensor_values[i] for i, op_type in enumerate(executed_op_types) + if op_type == "Pow"] + negative_v1_squared_values = [ + tensor_values[i] for i, op_type in enumerate(executed_op_types) + if op_type == "Neg"] + self.assertAllClose(v1_squared_values, [[100.0], [8100.0], [67076100.0]]) + self.assertAllClose( + negative_v1_squared_values, [[-100.0], [-8100.0], [-67076100.0]]) + + (executed_op_types, _, _, _, + tensor_values) = self._readAndCheckExecutionFile(dump_root=dump_root_2) + self.assertNotIn("Neg", executed_op_types) + v2_squared_values = tensor_values[executed_op_types.index("Pow")] + self.assertAllClose(v2_squared_values, [9.0]) + @parameterized.named_parameters( ("NoTensor", "NO_TENSOR"), ("FullTensor", "FULL_TENSOR"), diff --git a/tensorflow/python/debug/lib/dumping_callback_test_lib.py b/tensorflow/python/debug/lib/dumping_callback_test_lib.py index 5010c069a21099..b4c891feb7b5bf 100644 --- a/tensorflow/python/debug/lib/dumping_callback_test_lib.py +++ b/tensorflow/python/debug/lib/dumping_callback_test_lib.py @@ -23,6 +23,7 @@ import socket import tempfile +from tensorflow.core.framework import types_pb2 from tensorflow.python.debug.lib import check_numerics_callback from tensorflow.python.debug.lib import debug_events_reader from tensorflow.python.debug.lib import dumping_callback @@ -137,9 +138,13 @@ def _readAndCheckGraphsFile(self, stack_frame_by_id): self.assertIn(stack_frame_id, stack_frame_by_id) return context_ids, op_types, op_name_to_op_type - def _readAndCheckExecutionFile(self): + def _readAndCheckExecutionFile(self, dump_root=None): """Read and verify the content of the .execution debug-event file. + Args: + dump_root: Optional argument that can be used to override the default + dump root to read the data from. + Returns: executed_op_types: Types of ops that are created, as a `list` of `str`. input_tensor_ids: Input tensor IDs for each of the ops executed, as a @@ -153,7 +158,8 @@ def _readAndCheckExecutionFile(self): execution event. Each item of the inner `list` corresponds to one output tensor slot of the executed op or Function. """ - reader = debug_events_reader.DebugEventsReader(self.dump_root) + dump_root = self.dump_root if dump_root is None else dump_root + reader = debug_events_reader.DebugEventsReader(dump_root) execution_iter = reader.execution_iterator() prev_wall_time = 1 executed_op_types = [] @@ -213,7 +219,10 @@ def _readAndCheckGraphExecutionTracesFile(self, context_ids): self.assertIn(graph_execution_trace.tfdbg_context_id, context_ids) output_slots.append(graph_execution_trace.output_slot) dtype = dtypes.DType(graph_execution_trace.tensor_proto.dtype) - if dtype.is_numpy_compatible: # pylint:disable=protected-access + if (dtype.is_numpy_compatible and + dtype._type_enum != types_pb2.DT_STRING): # pylint:disable=protected-access + # TODO(cais): Figure out how to properly convert string tensor proto to + # numpy representation. tensor_values.append( tensor_util.MakeNdarray(graph_execution_trace.tensor_proto)) else: diff --git a/tensorflow/python/distribute/device_util.py b/tensorflow/python/distribute/device_util.py index d1295f27019d33..db6009d1a45e7f 100644 --- a/tensorflow/python/distribute/device_util.py +++ b/tensorflow/python/distribute/device_util.py @@ -38,13 +38,17 @@ def canonicalize(d, default=None): Note: This uses "job:localhost" as the default if executing eagerly. Args: - d: a device string. + d: a device string or tf.config.LogicalDevice default: a string for default device if d doesn't have all components. Returns: a canonicalized device string. """ - d = tf_device.DeviceSpec.from_string(d) + if isinstance(d, context.LogicalDevice): + d = tf_device.DeviceSpec.from_string(d.name) + else: + d = tf_device.DeviceSpec.from_string(d) + assert d.device_type is None or d.device_type == d.device_type.upper(), ( "Device type '%s' must be all-caps." % (d.device_type,)) # Fill in missing device fields using defaults. diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py index 8c90af0c300dd4..58c155a543424c 100644 --- a/tensorflow/python/distribute/mirrored_strategy.py +++ b/tensorflow/python/distribute/mirrored_strategy.py @@ -38,6 +38,7 @@ from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.eager import tape +from tensorflow.python.framework import config from tensorflow.python.framework import constant_op from tensorflow.python.framework import device as tf_device from tensorflow.python.framework import dtypes @@ -206,7 +207,8 @@ def _is_device_list_single_worker(devices): """Checks whether the devices list is for single or multi-worker. Args: - devices: a list of device strings, either local or for remote devices. + devices: a list of device strings or tf.config.LogicalDevice objects, for + either local or for remote devices. Returns: a boolean indicating whether these device strings are for local or for @@ -215,7 +217,10 @@ def _is_device_list_single_worker(devices): Raises: ValueError: if device strings are not consistent. """ - specs = (tf_device.DeviceSpec.from_string(d) for d in devices) + specs = [] + for d in devices: + name = d.name if isinstance(d, context.LogicalDevice) else d + specs.append(tf_device.DeviceSpec.from_string(name)) num_workers = len({(d.job, d.task, d.replica) for d in specs}) all_local = all(d.job in (None, "localhost") for d in specs) any_local = any(d.job in (None, "localhost") for d in specs) @@ -321,9 +326,10 @@ def _infer_num_gpus_per_worker(devices): def all_local_devices(num_gpus=None): - if num_gpus is None: - num_gpus = context.num_gpus() - return device_util.local_devices_from_num_gpus(num_gpus) + devices = config.list_logical_devices("GPU") + if num_gpus is not None: + devices = devices[:num_gpus] + return devices or config.list_logical_devices("CPU") def all_devices(): @@ -867,6 +873,7 @@ def __init__(self, dist, coord, replica_id, device_map, variable_creator_fn, ctx = context.context() self.in_eager = ctx.executing_eagerly() self.record_thread_local_summary_state() + self.record_thread_local_eager_context_state() self.context_device_policy = ( pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy( ctx._context_handle)) # pylint: disable=protected-access @@ -892,6 +899,7 @@ def run(self): if self.coord.should_stop(): return self.restore_thread_local_summary_state() + self.restore_thread_local_eager_context_state() # TODO(josh11b): Use current logical device instead of 0 here. with self.coord.stop_on_exception(), \ _enter_graph(self._init_graph, self._init_in_eager), \ @@ -920,7 +928,6 @@ def record_thread_local_summary_state(self): self._summary_recording = summary_state.is_recording self._summary_recording_distribution_strategy = ( summary_state.is_recording_distribution_strategy) - # TODO(b/125892694): record other fields in EagerContext. def restore_thread_local_summary_state(self): """Restore thread local summary state from self.""" @@ -931,7 +938,18 @@ def restore_thread_local_summary_state(self): summary_state.is_recording = self._summary_recording summary_state.is_recording_distribution_strategy = ( self._summary_recording_distribution_strategy) - # TODO(b/125892694): restore other fields in EagerContext. + + def record_thread_local_eager_context_state(self): + ctx = context.context() + eager_context_state = ctx._thread_local_data # pylint: disable=protected-access + self._eager_context_op_callbacks = eager_context_state.op_callbacks + # TODO(b/125892694): record other fields in EagerContext. + + def restore_thread_local_eager_context_state(self): + ctx = context.context() + eager_context_state = ctx._thread_local_data # pylint: disable=protected-access + eager_context_state.op_callbacks = self._eager_context_op_callbacks + # TODO(b/125892694): record other fields in EagerContext. class MirroredReplicaContext(distribute_lib.ReplicaContext): diff --git a/tensorflow/python/distribute/multi_worker_continuous_run_test.py b/tensorflow/python/distribute/multi_worker_continuous_run_test.py index 8785b56d1b9432..19790a0d69fb2e 100644 --- a/tensorflow/python/distribute/multi_worker_continuous_run_test.py +++ b/tensorflow/python/distribute/multi_worker_continuous_run_test.py @@ -26,6 +26,7 @@ from tensorflow.python.distribute import collective_all_reduce_strategy from tensorflow.python.distribute import combinations from tensorflow.python.distribute import multi_process_runner +from tensorflow.python.distribute import multi_process_runner_util from tensorflow.python.distribute import multi_worker_test_base as test_base from tensorflow.python.distribute import reduce_util from tensorflow.python.eager import context @@ -75,9 +76,11 @@ def worker_fn(): for _ in range(100): worker_step_fn() - multi_process_runner.MultiProcessRunner().run( - worker_fn, - cluster_spec=test_base.create_cluster_spec(num_workers=num_workers)) + # TODO(b/141948186): Remove this `with` block once b/141948186 is resolved. + with multi_process_runner_util.try_run_and_except_connection_error(self): + multi_process_runner.MultiProcessRunner().run( + worker_fn, + cluster_spec=test_base.create_cluster_spec(num_workers=num_workers)) if __name__ == '__main__': diff --git a/tensorflow/python/distribute/multi_worker_util.py b/tensorflow/python/distribute/multi_worker_util.py index c804ed9b8bcada..4d89b2fab08520 100644 --- a/tensorflow/python/distribute/multi_worker_util.py +++ b/tensorflow/python/distribute/multi_worker_util.py @@ -53,7 +53,10 @@ def _validate_cluster_spec(cluster_spec, task_type, task_id): It checks: 0) None of `cluster_spec`, `task_type`, and `task_id` is `None`. 1) task type is one of "chief", "worker" or "evaluator". - 2) whether there is such a task type as `task_type` in the `cluster_spec`. + 2) whether there is such a task type as `task_type` in the `cluster_spec`. The + only exception is `evaluator`. In other words, it is still a valid + configuration when `task_type` is `evaluator` but it doesn't appear in + `cluster_spec`. This is to be compatible with `TF_CONFIG` in Estimator. 3) whether there is at most one "chief" job. 4) whether there is at most one "evaluator" job. 5) whether the `task_id` is smaller than the number of tasks for that @@ -76,7 +79,7 @@ def _validate_cluster_spec(cluster_spec, task_type, task_id): "Unrecognized task_type: %r, valid task types are: \"chief\", " "\"worker\", \"evaluator\" and \"ps\"." % task_type) - if task_type and task_type not in cluster_spec: + if task_type and task_type not in cluster_spec and task_type != "evaluator": raise ValueError("`task_type` %r not found in cluster_spec." % task_type) if len(cluster_spec.get("chief", [])) > 1: @@ -85,7 +88,8 @@ def _validate_cluster_spec(cluster_spec, task_type, task_id): if len(cluster_spec.get("evaluator", [])) > 1: raise ValueError("There must be at most one 'evaluator' job.") - if task_id >= len(cluster_spec[task_type]): + # The `evaluator` job is allowed to be missing in `cluster_spec`. + if task_type in cluster_spec and task_id >= len(cluster_spec[task_type]): raise ValueError( "The `task_id` %d exceeds the maximum id of %s." % (task_id, task_type)) diff --git a/tensorflow/python/distribute/multi_worker_util_test.py b/tensorflow/python/distribute/multi_worker_util_test.py index dbe57b24e08f34..6a51e71ded77af 100644 --- a/tensorflow/python/distribute/multi_worker_util_test.py +++ b/tensorflow/python/distribute/multi_worker_util_test.py @@ -237,5 +237,30 @@ def testLocalLeader(self): multi_worker_util.collective_leader(cluster_spec, None, 0), "") +# Most of the validation logic is tested by above tests except for some. +class ClusterSpecValidationTest(test.TestCase): + + def testEvaluatorNotInCluster(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "worker": ["127.0.0.1:8964", "127.0.0.1:2333"], + "ps": ["127.0.0.1:1926", "127.0.0.1:3141"] + } + multi_worker_util._validate_cluster_spec(cluster_spec, "chief", 0) + multi_worker_util._validate_cluster_spec(cluster_spec, "worker", 0) + multi_worker_util._validate_cluster_spec(cluster_spec, "ps", 0) + multi_worker_util._validate_cluster_spec(cluster_spec, "evaluator", 0) + + def testWorkerNotInCluster(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "ps": ["127.0.0.1:1926", "127.0.0.1:3141"] + } + multi_worker_util._validate_cluster_spec(cluster_spec, "evaluator", 0) + with self.assertRaisesRegexp( + ValueError, "`task_type` 'worker' not found in cluster_spec."): + multi_worker_util._validate_cluster_spec(cluster_spec, "worker", 0) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/distribute/remote_mirrored_strategy_eager_test.py b/tensorflow/python/distribute/remote_mirrored_strategy_eager_test.py index 1389ec393e026b..36ec919a57566b 100644 --- a/tensorflow/python/distribute/remote_mirrored_strategy_eager_test.py +++ b/tensorflow/python/distribute/remote_mirrored_strategy_eager_test.py @@ -30,7 +30,7 @@ def get_gpus(): gpus = context.context().list_logical_devices("GPU") actual_gpus = [] for gpu in gpus: - if "localhost" not in gpu.name: + if "job" in gpu.name: actual_gpus.append(gpu.name) return actual_gpus diff --git a/tensorflow/python/distribute/tpu_strategy.py b/tensorflow/python/distribute/tpu_strategy.py index 34c9578c65be73..bf5d4baba2c27f 100644 --- a/tensorflow/python/distribute/tpu_strategy.py +++ b/tensorflow/python/distribute/tpu_strategy.py @@ -37,6 +37,7 @@ from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver from tensorflow.python.eager import context from tensorflow.python.eager import def_function +from tensorflow.python.eager import function from tensorflow.python.framework import constant_op from tensorflow.python.framework import device_spec from tensorflow.python.framework import dtypes @@ -82,6 +83,29 @@ def maybe_init_scope(): yield +def validate_experimental_run_function(fn): + """Validate the function passed into strategy.experimental_run_v2.""" + + # We allow three types of functions/objects passed into TPUStrategy + # experimental_run_v2 in eager mode: + # 1. a user annotated tf.function + # 2. a ConcreteFunction, this is mostly what you get from loading a saved + # model. + # 3. a callable object and the `__call__` method itself is a tf.function. + # + # Otherwise we return an error, because we don't support eagerly running + # experimental_run_v2 in TPUStrategy. + + if context.executing_eagerly() and not isinstance( + fn, def_function.Function) and not isinstance( + fn, function.ConcreteFunction) and not (callable(fn) and isinstance( + fn.__call__, def_function.Function)): + raise NotImplementedError( + "TPUStrategy.experimental_run_v2(fn, ...) does not support eager " + "execution. Either convert `fn` into a tf.function or consider " + "calling strategy.experimental_run_v2 inside a tf.function.") + + @tf_export("distribute.experimental.TPUStrategy", v1=[]) class TPUStrategy(distribute_lib.Strategy): """TPU distribution strategy implementation.""" @@ -89,14 +113,36 @@ class TPUStrategy(distribute_lib.Strategy): def __init__(self, tpu_cluster_resolver=None, device_assignment=None): - """Initializes the TPUStrategy object. + """Synchronous training in TPU donuts or Pods. + + To construct a TPUStrategy object, you need to run the + initialization code as below: + + ```python + resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=FLAGS.tpu) + tf.config.experimental_connect_to_cluster(resolver) + tf.tpu.experimental.initialize_tpu_system(resolver) + strategy = tf.distribute.experimental.TPUStrategy(resolver) + ``` + + While using distribution strategies, the variables created within strategy's + scope will be replicated across all the replicas and can be kept in sync + using all-reduce algorithms. + + To run TF2 programs on TPUs, you can either use `.compile` and + `.fit` APIs in `tf.keras` with TPUStrategy, or write your own customized + training loop by calling `strategy.experimental_run_v2` directly. Note that + TPUStrategy doesn't support pure eager execution, so please make sure the + function passed into `strategy.experimental_run_v2` is a `tf.function` or + `strategy.experimental_run_v2` us called inside a `tf.function` if running + in eager mode. Args: tpu_cluster_resolver: A tf.distribute.cluster_resolver.TPUClusterResolver, - which provides information about the TPU cluster. + which provides information about the TPU cluster. device_assignment: Optional `tf.tpu.experimental.DeviceAssignment` to - specify the placement of replicas on the TPU cluster. Currently only - supports the usecase of using a single core within a TPU cluster. + specify the placement of replicas on the TPU cluster. Currently only + supports the usecase of using a single core within a TPU cluster. """ super(TPUStrategy, self).__init__(TPUExtended( self, tpu_cluster_resolver, device_assignment=device_assignment)) @@ -111,6 +157,8 @@ def __init__(self, # This implementation runs a single step. It does not use infeed or outfeed. def experimental_run_v2(self, fn, args=(), kwargs=None): """See base class.""" + validate_experimental_run_function(fn) + # Note: the target function is converted to graph even when in Eager mode, # so autograph is on by default here. fn = autograph.tf_convert(fn, ag_ctx.control_status_ctx()) @@ -156,6 +204,8 @@ def steps_per_run(self): # can use the default implementation. # This implementation runs a single step. It does not use infeed or outfeed. def experimental_run_v2(self, fn, args=(), kwargs=None): + validate_experimental_run_function(fn) + """See base class.""" fn = autograph.tf_convert(fn, ag_ctx.control_status_ctx()) return self.extended.tpu_run(fn, args, kwargs) @@ -699,7 +749,7 @@ def replicated_fn(replica_id, replica_args, replica_kwargs): ] # Workaround for `tpu.replicate` behaviour when single `Tensor` returned. - if result[0] is None: + if result[0] is None or isinstance(result[0], ops.Operation): replicate_outputs = [None] * len(replicate_outputs) else: replicate_outputs = [ diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 4a76bd79513dcc..5bc654c21849b4 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -775,9 +775,11 @@ cuda_py_test( ":def_function", ":test", ":remote", + "@absl_py//absl/testing:parameterized", "//tensorflow/python:resource_variable_ops", ], grpc_enabled = True, + shard_count = 2, tags = [ "no_oss", # This test launches local server. "optonly", # times out diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 8e3efbf1afc02a..23cfbd44972db8 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -910,7 +910,6 @@ def testUnconnectedGradientsVariablesZeros(self): dz_dx = g.gradient(z, x, unconnected_gradients='zero') self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], self.evaluate(dz_dx)) - @test_util.assert_no_new_tensors @test_util.run_in_graph_and_eager_modes def testUnknownUnconnectedGradientsValueGiven(self): x = constant_op.constant(1.0) diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 1f13f163e0e081..f2ab167e24c0fa 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -65,11 +65,7 @@ MIRRORING_NONE = pywrap_tensorflow.TFE_MIRRORING_NONE MIRRORING_ALL = pywrap_tensorflow.TFE_MIRRORING_ALL -# TODO(b/143164764): Currently _KEEP_ALIVE_SECS is set to a very long time -# (i.e. 30 days) because the server may deadlock when destroying the eager -# context. This may cause memory leak in the headless TPU case, we should change -# it back to 600 once the deadlock is fixed. -_KEEP_ALIVE_SECS = 2592000 +_KEEP_ALIVE_SECS = 600 _python_eager_context_create_counter = monitoring.Counter( "/tensorflow/api/python/eager_context_create_counter", @@ -408,6 +404,7 @@ def __init__(self, if execution_mode is None: execution_mode = SYNC self._default_is_async = execution_mode == ASYNC + self._lazy_remote_inputs_copy = False self._server_def = server_def self._collective_ops_server_def = None self._collective_leader = None @@ -473,8 +470,12 @@ def _initialize_logical_devices(self): dev_name = pywrap_tensorflow.TF_DeviceListName(device_list, i) self._context_devices.append(pydev.canonical_name(dev_name)) spec = pydev.DeviceSpec.from_string(dev_name) + # If the job is localhost, we assume that the cluster has not yet been + # configured and thus clear the job, replica & task. + if spec.job == "localhost": + spec = spec.replace(job=None, replica=None, task=None) self._logical_devices.append( - LogicalDevice(name=dev_name, device_type=spec.device_type)) + LogicalDevice(name=spec.to_string(), device_type=spec.device_type)) dev_type = pywrap_tensorflow.TF_DeviceListType(device_list, i) if dev_type == "GPU": self._num_gpus += 1 @@ -502,6 +503,9 @@ def ensure_initialized(self): opts, self._mirroring_policy) if self._default_is_async == ASYNC: pywrap_tensorflow.TFE_ContextOptionsSetAsync(opts, True) + if self._lazy_remote_inputs_copy: + pywrap_tensorflow.TFE_ContextOptionsSetLazyRemoteInputsCopy( + opts, True) context_handle = pywrap_tensorflow.TFE_NewContext(opts) finally: pywrap_tensorflow.TFE_DeleteContextOptions(opts) @@ -1445,6 +1449,22 @@ def mirroring_policy(self, policy): pywrap_tensorflow.TFE_ContextSetThreadLocalMirroringPolicy( self._handle, self._mirroring_policy) + @property + def lazy_remote_inputs_copy(self): + return self._lazy_remote_inputs_copy + + @lazy_remote_inputs_copy.setter + def lazy_remote_inputs_copy(self, lazy_copy): + """Sets whether to copy remote inputs lazily for functions.""" + if not isinstance(lazy_copy, bool): + raise ValueError("Expecting a boolean but got %s" % type(lazy_copy)) + + if self._lazy_remote_inputs_copy != lazy_copy: + if self._initialized: + raise ValueError( + "lazy_remote_inputs_copy should be set before being initialized.") + self._lazy_remote_inputs_copy = lazy_copy + def enable_run_metadata(self): """Enables tracing of op execution via RunMetadata. diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py index e7b4a6f84b2a5a..68d8a3ead7e662 100644 --- a/tensorflow/python/eager/def_function.py +++ b/tensorflow/python/eager/def_function.py @@ -405,7 +405,7 @@ def embedding_matmul(a, b): self._implements = experimental_implements self._autograph = autograph self._experimental_autograph_options = experimental_autograph_options - self.experimental_relax_shapes = experimental_relax_shapes + self._experimental_relax_shapes = experimental_relax_shapes self._experimental_compile = experimental_compile self._created_variables = None # GUARDED_BY(self._lock) self._stateful_fn = None # GUARDED_BY(self._lock) @@ -458,7 +458,7 @@ def _defun(self, fn): attributes=attributes, autograph=self._autograph, experimental_autograph_options=self._experimental_autograph_options, - experimental_relax_shapes=self.experimental_relax_shapes) + experimental_relax_shapes=self._experimental_relax_shapes) def _initialize(self, args, kwds, add_initializers_to=None): """Initializes, on the first call. @@ -514,7 +514,7 @@ def _clone(self, python_function): autograph=self._autograph, experimental_implements=self._implements, experimental_autograph_options=self._experimental_autograph_options, - experimental_relax_shapes=self.experimental_relax_shapes, + experimental_relax_shapes=self._experimental_relax_shapes, experimental_compile=self._experimental_compile) def _decorate(self, decorator): @@ -728,13 +728,20 @@ def initialize_variables(): resource_variable_ops.var_is_initialized_op(v.handle)) var_is_initialized = array_ops.stack(var_is_initialized).numpy() + inits = [] for (v, init), is_initialized in zip(initializers, var_is_initialized): with ops.init_scope(): if is_initialized: continue + inits.append(init) + if inits: op_map = lift_to_graph.lift_to_graph( - [init], ops.get_default_graph(), op_map=op_map) + inits, ops.get_default_graph(), op_map=op_map) + for (v, init), is_initialized in zip(initializers, var_is_initialized): + with ops.init_scope(): + if is_initialized: + continue v.assign(op_map[init], read_value=False) with ops.init_scope(): diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py index a38ba73cae45fb..c7f8a25ae45aac 100644 --- a/tensorflow/python/eager/def_function_test.py +++ b/tensorflow/python/eager/def_function_test.py @@ -137,6 +137,19 @@ def fn(x): self.assertAllEqual(fn(constant_op.constant(1.0)), 2.0) + def testFunctionMultipleVariableInitializer(self): + + state = [] + + @def_function.function + def fn(x): + if not state: + state.append(variables.Variable(lambda: 2.0)) + state.append(variables.Variable(lambda: 5.0)) + return state[0] * x, state[1] * x + + self.assertAllEqual(fn(constant_op.constant(1.0)), [2.0, 5.0]) + def testFunctionInitializationFunction(self): state = [] @@ -668,7 +681,7 @@ def testClone(self, input_signature, autograph, autograph_options, implements, self.assertEqual(autograph, cloned._autograph) self.assertEqual(implements, cloned._implements) self.assertEqual(autograph_options, cloned._experimental_autograph_options) - self.assertEqual(relax_shapes, cloned.experimental_relax_shapes) + self.assertEqual(relax_shapes, cloned._experimental_relax_shapes) self.assertEqual(compile_, cloned._experimental_compile) # This test does not run with XLA JIT support linked in so we can only check diff --git a/tensorflow/python/eager/forwardprop_test.py b/tensorflow/python/eager/forwardprop_test.py index b214c908c37100..10cd14b6a61dc9 100644 --- a/tensorflow/python/eager/forwardprop_test.py +++ b/tensorflow/python/eager/forwardprop_test.py @@ -469,6 +469,13 @@ def _bn_fused(x_arg, scale_arg, offset_arg): atol=1e-3) def testFusedBatchNormGradsInference(self): + + if test.is_built_with_rocm(): + # This test was addeded recently and has been failing on the ROCm + # platform, since it was added. + # TODO(rocm): do root cause analysis of test failure and fix it. + self.skipTest("Test fails on ROCm platform, needs further analysis") + x_shape = [4, 10, 10, 2] increment = 3. / math_ops.reduce_prod( constant_op.constant(x_shape, dtype=dtypes.float32)) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 6c807e61746821..810fc86b30da1e 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -54,6 +54,7 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_spec from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import custom_gradient from tensorflow.python.ops import default_gradient from tensorflow.python.ops import functional_ops @@ -125,8 +126,12 @@ def _make_input_signature_hashable(elem, variable_map=None): CacheKey = collections.namedtuple("CacheKey", [ - "input_signature", "parent_graph", "device_functions", "colocation_stack", - "in_cross_replica_context" + "input_signature", + "parent_graph", + "device_functions", + "colocation_stack", + "in_cross_replica_context", + "xla_context_id", ]) @@ -356,6 +361,23 @@ def _inference_name(n): return "__inference_%s_%s" % (n, ops.uid()) +def _enclosing_xla_context(): + """Returns the XLAControlFlowContext, which exists inside a tpu.rewrite().""" + graph = ops.get_default_graph() + while graph is not None: + # pylint: disable=protected-access + context_ = graph._get_control_flow_context() + # pylint: enable=protected-access + while context_ is not None: + if isinstance(context_, control_flow_ops.XLAControlFlowContext): + return context_ + context_ = context_.outer_context + # This may be a FuncGraph due to defuns or v2 control flow. We need to + # find the original graph with the XLAControlFlowContext. + graph = getattr(graph, "outer_graph", None) + return None + + class _EagerDefinedFunctionDeleter(object): """Unregister function from eager context.""" @@ -2488,7 +2510,14 @@ def _cache_key(self, args, kwargs, include_tensor_ranks_only=False): # already. executing_eagerly = ctx.executing_eagerly() parent_graph = None + xla_context_id = 0 if not executing_eagerly: + # We want to force function retracing for each different + # XLAControlFlowContext, so add `xla_context_id` to the cache key. + tpu_context = _enclosing_xla_context() + if tpu_context is not None: + xla_context_id = id(tpu_context) + with ops.init_scope(): # The graph, or whether we're executing eagerly, should be a part of the # cache key so we don't improperly capture tensors such as variables. @@ -2529,11 +2558,9 @@ def _cache_key(self, args, kwargs, include_tensor_ranks_only=False): pass return CacheKey( - _make_input_signature_hashable(input_signature), - parent_graph, - device_functions, - colocation_stack, - in_cross_replica_context) + _make_input_signature_hashable(input_signature), parent_graph, + device_functions, colocation_stack, in_cross_replica_context, + xla_context_id) def _create_graph_function(self, args, kwargs, override_flat_arg_shapes=None): """Create a `ConcreteFunction` from `args` and `kwargs`.""" @@ -3192,7 +3219,8 @@ def bound_method_wrapper(*args, **kwargs): tf_decorator.make_decorator(bound_method, bound_method_wrapper), name=original_function._name, autograph=original_function._autograph, - input_signature=original_function.input_signature) + input_signature=original_function.input_signature, + experimental_relax_shapes=original_function._experimental_relax_shapes) # pylint: enable=protected-access # And we wrap the function with tf_decorator so inspection works correctly diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 20b21a478e491b..2653b4d31dde31 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -323,6 +323,29 @@ def func(a): self.assertTrue(unknown_dim[0]) self.assertLen(total_function_cache(func), 2) + def testInputShapeRelaxationOnInstanceMethod(self): + # Test that experimental_relax_shapes is passed during + # instance method bounding. + unknown_dim = [False] + + class Foo(object): + + @def_function.function(experimental_relax_shapes=True) + def func(self, a): + if a._shape_tuple()[0] is None: + unknown_dim[0] = True + return a + 1 + + foo = Foo() + foo.func(constant_op.constant([])) + self.assertFalse(unknown_dim[0]) + + foo.func(constant_op.constant([1.0])) + self.assertFalse(unknown_dim[0]) + + foo.func(constant_op.constant([1.0, 2.0])) + self.assertTrue(unknown_dim[0]) + def testCapturesVariables(self): a = variables.Variable(1.0, trainable=False) b = variables.Variable(1.0) diff --git a/tensorflow/python/eager/remote_test.py b/tensorflow/python/eager/remote_test.py index 7008b0a124f0e1..d0030774fde4c7 100644 --- a/tensorflow/python/eager/remote_test.py +++ b/tensorflow/python/eager/remote_test.py @@ -20,6 +20,7 @@ import random +from absl.testing import parameterized import numpy as np from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver @@ -39,7 +40,7 @@ from tensorflow.python.training.server_lib import ClusterSpec -class SingleWorkerTest(test.TestCase): +class SingleWorkerTest(test.TestCase, parameterized.TestCase): def setUp(self): super(SingleWorkerTest, self).setUp() @@ -55,6 +56,7 @@ def tearDown(self): # Reset the context to avoid polluting other test cases. context._reset_context() + @test_util.eager_lazy_remote_copy_on_and_off def testMultiDeviceFunctionBasic(self): @def_function.function @@ -69,6 +71,7 @@ def basic(i): self.assertAllEqual(basic(constant_op.constant([2])).numpy(), [5]) self.assertAllEqual(basic(constant_op.constant([1])).numpy(), [4]) + @test_util.eager_lazy_remote_copy_on_and_off def testMultiDeviceFunctionVariable(self): with ops.device('/job:worker/replica:0/task:0/cpu:0'): variable_b = variables.Variable(1) @@ -79,6 +82,7 @@ def with_variable(i): self.assertAllEqual(with_variable(constant_op.constant([2])).numpy(), [3]) + @test_util.eager_lazy_remote_copy_on_and_off def testMultiDeviceFunctionRemoteOutput(self): with ops.device('/job:worker/replica:0/task:0/cpu:0'): variable_b = variables.Variable(1) @@ -134,6 +138,7 @@ def testShapeError_OpByOp(self): self.assertIn('Dimensions must be equal', cm.exception.message) + @test_util.eager_lazy_remote_copy_on_and_off def testShapeError_Function(self): @def_function.function @@ -150,7 +155,7 @@ def matmul_func(x, y): self.assertIn('Dimensions must be equal', cm.exception.message) -class MultiWorkersTest(test.TestCase): +class MultiWorkersTest(test.TestCase, parameterized.TestCase): def setUp(self): super(MultiWorkersTest, self).setUp() @@ -167,6 +172,7 @@ def tearDown(self): # Reset the context to avoid polluting other test cases. context._reset_context() + @test_util.eager_lazy_remote_copy_on_and_off def testMultiDeviceFunctionOnLocalDevice(self): with ops.device('/job:worker/replica:0/task:1'): variable_b = variables.Variable(1.0) @@ -180,6 +186,7 @@ def remote_function(i): self.assertAllEqual(remote_function(constant_op.constant([1.0])), [3.0]) + @test_util.eager_lazy_remote_copy_on_and_off def testMultiDeviceFunctionOnRemoteDevice(self): with ops.device('/job:worker/replica:0/task:1'): variable_b = variables.Variable(1.0) @@ -209,6 +216,7 @@ def remote_function(i): with ops.device('/job:worker/replica:0/task:0/device:GPU:0'): self.assertAllEqual(remote_function(constant_op.constant([1.0])), [3.0]) + @test_util.eager_lazy_remote_copy_on_and_off def testMultiDeviceWhileLoopOnRemoteDevice(self): with ops.device('/job:worker/replica:0/task:1'): variable_b = variables.Variable(1.0) @@ -241,6 +249,7 @@ def body(i, _): with ops.device('/job:worker/replica:0/task:0/device:GPU:0'): self.assertAllEqual(remote_function(constant_op.constant([1.0])), [3.0]) + @test_util.eager_lazy_remote_copy_on_and_off def testSimpleParameterServer(self): with ops.device('/job:worker/task:2/device:CPU:0'): @@ -263,7 +272,7 @@ def worker_fn(): _GRPC_PREFIX = 'grpc://' -class MultiJobsTest(test.TestCase): +class MultiJobsTest(test.TestCase, parameterized.TestCase): def setUp(self): super(MultiJobsTest, self).setUp() @@ -288,6 +297,7 @@ def tearDown(self): # Reset the context to avoid polluting other test cases. context._reset_context() + @test_util.eager_lazy_remote_copy_on_and_off def testSimpleParameterServer(self): remote.connect_to_cluster(self._cluster) @@ -307,6 +317,7 @@ def worker_fn(): with ops.device('/job:my_worker/task:1/device:CPU:0'): self.assertAllEqual(worker_fn(), 8) + @test_util.eager_lazy_remote_copy_on_and_off def testConnectWithClusterResolver(self): remote.connect_to_cluster(self._cluster_resolver) @@ -325,10 +336,12 @@ def worker_fn(): with ops.device('/job:my_worker/task:1/device:CPU:0'): self.assertAllEqual(worker_fn(), 8) + @test_util.eager_lazy_remote_copy_on_and_off def testConnectToClusterTwiceOk(self): remote.connect_to_cluster(self._cluster_resolver) remote.connect_to_cluster(self._cluster_resolver) + @test_util.eager_lazy_remote_copy_on_and_off def testConnectToClusterOnMismatchedDevice(self): remote.connect_to_cluster(self._cluster_resolver) @@ -338,6 +351,7 @@ def testConnectToClusterOnMismatchedDevice(self): with self.assertRaises(ValueError): remote.connect_to_cluster(self._cluster_resolver) + @test_util.eager_lazy_remote_copy_on_and_off def testConnectToClusterWithLocalMaster(self): local_resolver = SimpleClusterResolver(ClusterSpec({}), master='local') remote.connect_to_cluster(local_resolver) diff --git a/tensorflow/python/framework/constant_op_test.py b/tensorflow/python/framework/constant_op_test.py new file mode 100644 index 00000000000000..da0fb64fde6f2b --- /dev/null +++ b/tensorflow/python/framework/constant_op_test.py @@ -0,0 +1,61 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tensorflow.python.framework.constant_op.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.platform import test + + +class ConstantOpTest(test.TestCase, parameterized.TestCase): + + @parameterized.parameters( + dtypes.bfloat16, + dtypes.complex128, + dtypes.complex64, + dtypes.double, + dtypes.float16, + dtypes.float32, + dtypes.float64, + dtypes.half, + dtypes.int16, + dtypes.int32, + dtypes.int64, + dtypes.int8, + dtypes.qint16, + dtypes.qint32, + dtypes.qint8, + dtypes.quint16, + dtypes.quint8, + dtypes.uint16, + dtypes.uint32, + dtypes.uint64, + dtypes.uint8, + ) + def test_convert_string_to_number(self, dtype): + with self.assertRaises(TypeError): + constant_op.constant("hello", dtype) + + +if __name__ == "__main__": + ops.enable_eager_execution() + test.main() diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 8a273e834be1b7..5a6991f931ea62 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -3092,8 +3092,16 @@ def _as_graph_def(self, from_version=None, add_shapes=False): op = func_graph.get_operation_by_name(node.name) except KeyError: continue + outputs = op.outputs + + if op.type == "StatefulPartitionedCall": + # Filter out any extra outputs (possibly added by function + # backpropagation rewriting). + num_outputs = len(node.attr["Tout"].list.type) + outputs = outputs[:num_outputs] + node.attr["_output_shapes"].list.shape.extend( - [output.get_shape().as_proto() for output in op.outputs]) + [output.get_shape().as_proto() for output in outputs]) return graph, self._version diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index f96a4a58822f3c..cda81a57d29c71 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -1019,6 +1019,21 @@ def function_in_eager(): return decorator +def eager_lazy_remote_copy_on_and_off(f): + """Execute the test method w/o lazy tensor copy for function remote inputs.""" + + @parameterized.named_parameters([("WithLazyRemoteCopy", True), ("", False)]) + @functools.wraps(f) + def decorator(self, lazily_remote_copy, *args, **kwargs): + if lazily_remote_copy: + context.context().lazy_remote_inputs_copy = True + else: + context.context().lazy_remote_inputs_copy = False + f(self, *args, **kwargs) + + return decorator + + def run_in_graph_and_eager_modes(func=None, config=None, use_gpu=True, diff --git a/tensorflow/python/keras/activations.py b/tensorflow/python/keras/activations.py index 6606f76f40d6d8..f56f1c7e2e7b0a 100644 --- a/tensorflow/python/keras/activations.py +++ b/tensorflow/python/keras/activations.py @@ -260,9 +260,8 @@ def sigmoid(x): >>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32) >>> b = tf.keras.activations.sigmoid(a) - >>> b.numpy() - array([0. , 0.26894143, 0.5 , 0.7310586 , 1. ], - dtype=float32) + >>> b.numpy() >= 0.0 + array([ True, True, True, True, True]) Arguments: x: Input tensor. diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py index d7a005bad5b9d1..913464ae47948b 100644 --- a/tensorflow/python/keras/engine/base_layer_test.py +++ b/tensorflow/python/keras/engine/base_layer_test.py @@ -1009,12 +1009,14 @@ def call(self, inputs, training=None): 'mse', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils.should_run_tf_function()) - _, train_metric = model.train_on_batch(np.ones((2, 3)), + for _ in range(3): + _, train_metric = model.train_on_batch(np.ones((2, 3)), + np.ones((2, 3))) + + self.assertEqual(train_metric, 2 * 3) + _, test_metric = model.test_on_batch(np.ones((2, 3)), np.ones((2, 3))) - self.assertEqual(train_metric, 2 * 3) - _, test_metric = model.test_on_batch(np.ones((2, 3)), - np.ones((2, 3))) - self.assertEqual(test_metric, 0) + self.assertEqual(test_metric, 0) def test_if_training_pattern_update(self): diff --git a/tensorflow/python/keras/engine/base_preprocessing_layer.py b/tensorflow/python/keras/engine/base_preprocessing_layer.py index 2f4bc8060aca59..29df47446733d2 100644 --- a/tensorflow/python/keras/engine/base_preprocessing_layer.py +++ b/tensorflow/python/keras/engine/base_preprocessing_layer.py @@ -28,8 +28,10 @@ from tensorflow.python.keras.engine import training_generator from tensorflow.python.keras.engine.base_layer import Layer from tensorflow.python.ops import math_ops +from tensorflow.python.util.tf_export import keras_export +@keras_export('keras.layers.experimental.preprocessing.PreprocessingLayer') class PreprocessingLayer(Layer): """Base class for PreprocessingLayers.""" __metaclass__ = abc.ABCMeta diff --git a/tensorflow/python/keras/engine/data_adapter.py b/tensorflow/python/keras/engine/data_adapter.py index 3f2cc2eaa6509d..50db978e77ad2b 100644 --- a/tensorflow/python/keras/engine/data_adapter.py +++ b/tensorflow/python/keras/engine/data_adapter.py @@ -19,6 +19,7 @@ from __future__ import print_function import abc +import collections import itertools import math import random @@ -45,6 +46,10 @@ except ImportError: scipy_sparse = None +try: + import pandas as pd # pylint: disable=g-import-not-at-top +except ImportError: + pd = None try: # In Python2 unicode is a scalar type @@ -214,8 +219,12 @@ def can_handle(x, y=None): if y is not None: flat_inputs += nest.flatten(y) + tensor_types = (ops.Tensor, np.ndarray) + if pd: + tensor_types = (ops.Tensor, np.ndarray, pd.Series, pd.DataFrame) + def _is_tensor(v): - if isinstance(v, (ops.Tensor, np.ndarray)): + if isinstance(v, tensor_types): return True return False @@ -736,6 +745,7 @@ def __init__(self, x, y=None, sample_weights=None, standardize_function=None, # Since we have to know the dtype of the python generator when we build the # dataset, we have to look at a batch to infer the structure. peek, x = self._peek_and_restore(x) + assert_not_namedtuple(peek) (peek, wrap_in_tuple, elements_to_keep, partial_sample_weight, sample_weight_modes, nested_shape, nested_dtypes @@ -1085,3 +1095,18 @@ def broadcast_sample_weight_modes(target_structure, sample_weight_modes): "structure:\n {}\n to \n {}".format(target_str, mode_str)) return sample_weight_modes + + +def assert_not_namedtuple(x): + if (isinstance(x, tuple) and + # TODO(b/144192902): Use a namedtuple checking utility. + hasattr(x, "_fields") and + isinstance(x._fields, collections.Sequence) and + all(isinstance(f, six.string_types) for f in x._fields)): + raise ValueError( + "Received namedtuple ({}) with fields `{}` as input. namedtuples " + "cannot, in general, be unambiguously resolved into `x`, `y`, " + "and `sample_weight`. For this reason Keras has elected not to " + "support them. If you would like the value to be unpacked, " + "please explicitly convert it to a tuple before passing it to " + "Keras.".format(x.__class__, x._fields)) diff --git a/tensorflow/python/keras/engine/data_adapter_test.py b/tensorflow/python/keras/engine/data_adapter_test.py index bce5b923b92052..63d04d97ad6813 100644 --- a/tensorflow/python/keras/engine/data_adapter_test.py +++ b/tensorflow/python/keras/engine/data_adapter_test.py @@ -160,6 +160,88 @@ def test_training_numpy(self): run_eagerly=testing_utils.should_run_eagerly()) self.model.fit(self.numpy_input, self.numpy_target, batch_size=5) + def test_can_handle_pandas(self): + try: + import pandas as pd # pylint: disable=g-import-not-at-top + except ImportError: + self.skipTest('Skipping test because pandas is not installed.') + self.assertTrue(self.adapter_cls.can_handle(pd.DataFrame(self.numpy_input))) + self.assertTrue( + self.adapter_cls.can_handle(pd.DataFrame(self.numpy_input)[0])) + self.assertTrue( + self.adapter_cls.can_handle( + pd.DataFrame(self.numpy_input), + pd.DataFrame(self.numpy_input)[0])) + + @keras_parameterized.run_all_keras_modes(always_skip_v1=True) + def test_training_pandas(self): + try: + import pandas as pd # pylint: disable=g-import-not-at-top + except ImportError: + self.skipTest('Skipping test because pandas is not installed.') + input_a = keras.Input(shape=(3,), name='input_a') + input_b = keras.Input(shape=(3,), name='input_b') + input_c = keras.Input(shape=(1,), name='input_b') + + x = keras.layers.Dense(4, name='dense_1')(input_a) + y = keras.layers.Dense(3, name='dense_2')(input_b) + z = keras.layers.Dense(1, name='dense_3')(input_c) + + model_1 = keras.Model(inputs=input_a, outputs=x) + model_2 = keras.Model(inputs=[input_a, input_b], outputs=[x, y]) + model_3 = keras.Model(inputs=input_c, outputs=z) + + model_1.compile(optimizer='rmsprop', loss='mse') + model_2.compile(optimizer='rmsprop', loss='mse') + + input_a_np = np.random.random((10, 3)) + input_b_np = np.random.random((10, 3)) + input_a_df = pd.DataFrame(input_a_np) + input_b_df = pd.DataFrame(input_b_np) + + output_a_df = pd.DataFrame(np.random.random((10, 4))) + output_b_df = pd.DataFrame(np.random.random((10, 3))) + + model_1.fit(input_a_df, + output_a_df) + model_2.fit([input_a_df, input_b_df], + [output_a_df, output_b_df]) + model_1.fit([input_a_df], + [output_a_df]) + model_1.fit({'input_a': input_a_df}, + output_a_df) + model_2.fit({'input_a': input_a_df, 'input_b': input_b_df}, + [output_a_df, output_b_df]) + + model_1.evaluate(input_a_df, + output_a_df) + model_2.evaluate([input_a_df, input_b_df], + [output_a_df, output_b_df]) + model_1.evaluate([input_a_df], + [output_a_df]) + model_1.evaluate({'input_a': input_a_df}, + output_a_df) + model_2.evaluate({'input_a': input_a_df, 'input_b': input_b_df}, + [output_a_df, output_b_df]) + + # Verify predicting on pandas vs numpy returns the same result + predict_1_pandas = model_1.predict(input_a_df) + predict_2_pandas = model_2.predict([input_a_df, input_b_df]) + predict_3_pandas = model_3.predict(input_a_df[0]) + + predict_1_numpy = model_1.predict(input_a_np) + predict_2_numpy = model_2.predict([input_a_np, input_b_np]) + predict_3_numpy = model_3.predict(np.asarray(input_a_df[0])) + + self.assertAllClose(predict_1_numpy, predict_1_pandas) + self.assertAllClose(predict_2_numpy, predict_2_pandas) + self.assertAllClose(predict_3_numpy, predict_3_pandas) + + # Extra ways to pass in dataframes + model_1.predict([input_a_df]) + model_1.predict({'input_a': input_a_df}) + model_2.predict({'input_a': input_a_df, 'input_b': input_b_df}) + def test_can_handle(self): self.assertTrue(self.adapter_cls.can_handle(self.tensor_input)) self.assertTrue( diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index b66c39d664a8fd..8e36b8a9e358dc 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -628,6 +628,8 @@ def fit(self, `(inputs, targets, sample_weights)`. - A generator or `keras.utils.Sequence` returning `(inputs, targets)` or `(inputs, targets, sample weights)`. + A more detailed description of unpacking behavior for iterator types + (Dataset, generator, Sequence) is given below. y: Target data. Like the input data `x`, it could be either Numpy array(s) or TensorFlow tensor(s). It should be consistent with `x` (you cannot have Numpy inputs and @@ -748,6 +750,30 @@ def fit(self, the generator as they can't be passed easily to children processes. **kwargs: Used for backwards compatibility. + Unpacking behavior for iterator-like inputs: + A common pattern is to pass a tf.data.Dataset, generator, or + tf.keras.utils.Sequence to the `x` argument of fit, which will in fact + yield not only features (x) but optionally targets (y) and sample weights. + Keras requires that the output of such iterator-likes be unambiguous. The + iterator should return a tuple of length 1, 2, or 3, where the optional + second and third elements will be used for y and sample_weight + respectively. Any other type provided will be wrapped in a length one + tuple, effectively treating everything as 'x'. When yielding dicts, they + should still adhere to the top-level tuple structure. + e.g. `({"x0": x0, "x1": x1}, y)`. Keras will not attempt to separate + features, targets, and weights from the keys of a single dict. + A notable unsupported data type is the namedtuple. The reason is that + it behaves like both an ordered datatype (tuple) and a mapping + datatype (dict). So given a namedtuple of the form: + `namedtuple("example_tuple", ["y", "x"])` + it is ambiguous whether to reverse the order of the elements when + interpreting the value. Even worse is a tuple of the form: + `namedtuple("other_tuple", ["x", "y", "z"])` + where it is unclear if the tuple was intended to be unpacked into x, y, + and sample_weight or passed through as a single element to `x`. As a + result the data processing code will simply raise a ValueError if it + encounters a namedtuple. (Along with instructions to remedy the issue.) + Returns: A `History` object. Its `History.history` attribute is a record of training loss values and metrics values @@ -817,6 +843,9 @@ def evaluate(self, if the model has named inputs. - A `tf.data` dataset. - A generator or `keras.utils.Sequence` instance. + A more detailed description of unpacking behavior for iterator types + (Dataset, generator, Sequence) is given in the `Unpacking behavior + for iterator-like inputs` section of `Model.fit`. y: Target data. Like the input data `x`, it could be either Numpy array(s) or TensorFlow tensor(s). It should be consistent with `x` (you cannot have Numpy inputs and @@ -870,6 +899,9 @@ def evaluate(self, multiprocessing, you should not pass non-picklable arguments to the generator as they can't be passed easily to children processes. + See the discussion of `Unpacking behavior for iterator-like inputs` for + `Model.fit`. + Returns: Scalar test loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs @@ -918,6 +950,9 @@ def predict(self, (in case the model has multiple inputs). - A `tf.data` dataset. - A generator or `keras.utils.Sequence` instance. + A more detailed description of unpacking behavior for iterator types + (Dataset, generator, Sequence) is given in the `Unpacking behavior + for iterator-like inputs` section of `Model.fit`. batch_size: Integer or `None`. Number of samples per gradient update. If unspecified, `batch_size` will default to 32. @@ -948,6 +983,10 @@ def predict(self, multiprocessing, you should not pass non-picklable arguments to the generator as they can't be passed easily to children processes. + See the discussion of `Unpacking behavior for iterator-like inputs` for + `Model.fit`. Note that Model.predict uses the same interpretation rules as + `Model.fit` and `Model.evaluate`, so inputs must be unambiguous for all + three methods. Returns: Numpy array(s) of predictions. @@ -1035,7 +1074,8 @@ class during training. This can be useful to tell the model to "pay if self._experimental_run_tf_function: outputs = training_v2_utils.train_on_batch( self, x, y=y, sample_weight=sample_weight, - class_weight=class_weight, reset_metrics=reset_metrics) + class_weight=class_weight, reset_metrics=reset_metrics, + standalone=True) outputs = (outputs['total_loss'] + outputs['output_losses'] + outputs['metrics']) outputs = [ @@ -1132,7 +1172,7 @@ def test_on_batch(self, x, y=None, sample_weight=None, reset_metrics=True): if self._experimental_run_tf_function: outputs = training_v2_utils.test_on_batch( self, x, y=y, sample_weight=sample_weight, - reset_metrics=reset_metrics) + reset_metrics=reset_metrics, standalone=True) outputs = (outputs['total_loss'] + outputs['output_losses'] + outputs['metrics']) outputs = [ @@ -1197,7 +1237,7 @@ def predict_on_batch(self, x): """ self._check_call_args('predict_on_batch') if self._experimental_run_tf_function: - return training_v2_utils.predict_on_batch(self, x) + return training_v2_utils.predict_on_batch(self, x, standalone=True) if (self._distribution_strategy and distribution_strategy_context.in_cross_replica_context()): diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py index e67bd7b5084d90..10e1190ed41542 100644 --- a/tensorflow/python/keras/engine/training_test.py +++ b/tensorflow/python/keras/engine/training_test.py @@ -18,8 +18,10 @@ from __future__ import division from __future__ import print_function +import collections import io import logging +import re import sys from absl.testing import parameterized @@ -753,6 +755,125 @@ def test_evaluate_predict_on_arrays(self): }) self.assertEqual(len(out), 2) + def _make_sequence_input_functions(self, input_type): + # train and test + xy_namedtuple = collections.namedtuple('xy_namedtuple', ['x', 'y']) + + # predict + x_namedtuple = collections.namedtuple('x_namedtuple', ['x']) + + if input_type == 'dataset': + dataset = dataset_ops.Dataset.range(16).map( + lambda _: array_ops.ones(shape=(1,))) + + xy_dataset = dataset_ops.Dataset.zip((dataset, dataset)).batch(4) + x_dataset = dataset.batch(4) + def xy_function(use_namedtuple): + return xy_dataset.map(xy_namedtuple) if use_namedtuple else xy_dataset + + def x_function(use_namedtuple): + return x_dataset.map(x_namedtuple) if use_namedtuple else x_dataset + + return xy_function, x_function + + elif input_type == 'generator': + def xy_generator(use_namedtuple): + x, y = np.ones((4, 1)), np.ones((4, 1)) + for _ in range(4): + if use_namedtuple: + yield xy_namedtuple(x, y) + else: + yield x, y + + def x_generator(use_namedtuple): + x = np.ones((4, 1)) + for _ in range(4): + if use_namedtuple: + yield x_namedtuple(x) + else: + yield x + + return xy_generator, x_generator + + elif input_type == 'sequence': + class XYSequence(data_utils.Sequence): + + def __init__(self, use_namedtuple): + self._use_namedtuple = use_namedtuple + super(XYSequence, self).__init__() + + def __getitem__(self, idx): + x, y = np.ones((4, 1)), np.ones((4, 1)) + if self._use_namedtuple: + return xy_namedtuple(x, y) + return x, y + + def __len__(self): + return 4 + + class XSequence(data_utils.Sequence): + + def __init__(self, use_namedtuple): + self._use_namedtuple = use_namedtuple + super(XSequence, self).__init__() + + def __getitem__(self, idx): + x = np.ones((4, 1)) + if self._use_namedtuple: + return x_namedtuple(x) + return x + + def __len__(self): + return 4 + + return XYSequence, XSequence + + @keras_parameterized.run_all_keras_modes(always_skip_v1=True) + @keras_parameterized.run_with_all_model_types + @parameterized.named_parameters( + ('dataset', 'dataset'), + ('generator', 'generator'), + ('sequence', 'sequence'), + ) + def test_sequence_input_types(self, input_type): + """Ensure that namedtuples and tuples are plumbed identically.""" + if not testing_utils.should_run_tf_function(): + self.skipTest('Improved checking is only present in data_adapter.') + + xy_function, x_function = self._make_sequence_input_functions(input_type) + fit_kwargs, evaluate_kwargs, predict_kwargs = {}, {}, {} + if input_type == 'generator': + fit_kwargs['steps_per_epoch'] = 4 + evaluate_kwargs['steps'] = 4 + predict_kwargs['steps'] = 4 + + model = testing_utils.get_small_mlp(1, 1, 1) + model.compile( + loss='mse', + optimizer='sgd', + run_eagerly=testing_utils.should_run_eagerly(), + experimental_run_tf_function=testing_utils.should_run_tf_function()) + + model.fit(xy_function(use_namedtuple=False), **fit_kwargs) + model.evaluate(xy_function(use_namedtuple=False), **evaluate_kwargs) + model.predict(x_function(use_namedtuple=False), **predict_kwargs) + + xy_pattern = re.escape( + "Received namedtuple () with fields " + "`('x', 'y')` as input.") + x_pattern = re.escape( + "Received namedtuple () with fields " + "`('x',)` as input.") + + with self.assertRaisesRegex(ValueError, xy_pattern): + model.fit(xy_function(use_namedtuple=True), **fit_kwargs) + + with self.assertRaisesRegex(ValueError, xy_pattern): + model.evaluate(xy_function(use_namedtuple=True), **evaluate_kwargs) + + with self.assertRaisesRegex(ValueError, x_pattern): + model.predict(x_function(use_namedtuple=True), **predict_kwargs) + @keras_parameterized.run_all_keras_modes @keras_parameterized.run_with_all_model_types def test_activity_regularizer_fit(self): diff --git a/tensorflow/python/keras/engine/training_v2.py b/tensorflow/python/keras/engine/training_v2.py index 3025d186668ce3..3aeccb2171e9a7 100644 --- a/tensorflow/python/keras/engine/training_v2.py +++ b/tensorflow/python/keras/engine/training_v2.py @@ -436,7 +436,7 @@ def _model_iteration( # tf.print('{} on {} steps.'.format(ModeKeys.TRAIN, steps_per_epoch)) training_context = TrainingContext() - if mode == ModeKeys.PREDICT: + if training_v2_utils._should_add_batch_index_to_element(strategy, mode): dataset = training_v2_utils._add_batch_index_to_element(dataset) dataset = strategy.experimental_distribute_dataset(dataset) @@ -662,6 +662,12 @@ def standardize_function(dataset): # Then we map using only the tensor standardization portion. def map_fn(x, y=None, sample_weights=None): """Tensor manipulation portion of standardization for Dataset.map.""" + if (y is None and sample_weights is None): + # namedtuples are forbidden because it is ambiguous if they should be + # unpacked. If y or sample_weights is present then `x` was not the + # top level structure, and the correct behavior is unambiguous. + data_adapter.assert_not_namedtuple(x) + standardized = model._standardize_tensors( x, y, sample_weights, run_eagerly=False, diff --git a/tensorflow/python/keras/engine/training_v2_utils.py b/tensorflow/python/keras/engine/training_v2_utils.py index 665a4a2639189a..04b058949fa620 100644 --- a/tensorflow/python/keras/engine/training_v2_utils.py +++ b/tensorflow/python/keras/engine/training_v2_utils.py @@ -46,19 +46,27 @@ from tensorflow.python.util import nest -def _get_or_make_execution_function(model, mode): - """Makes or reuses function to run one step of distributed model execution.""" +def _get_or_make_function(model, mode, key_fn, make_fn): + """Helper function for managing cached execution functions.""" model._init_distributed_function_cache_if_not_compiled() + key = key_fn(mode) + + function = dist_utils.get_distributed_function(model, key) + if function: + return function - # Use a key with 'v2' to distinguish from fall-back execution functions. - key = (mode, 'v2') - distributed_function = dist_utils.get_distributed_function(model, key) - if distributed_function: - return distributed_function + function = make_fn(model, mode) + dist_utils.set_distributed_function(model, key, function) + return function - distribution_function = _make_execution_function(model, mode) - dist_utils.set_distributed_function(model, key, distribution_function) - return distribution_function + +def _get_or_make_execution_function(model, mode): + """Makes or reuses function to run one step of distributed model execution.""" + return _get_or_make_function( + model, mode, + # Use a key with 'v2' to distinguish from fall-back execution functions. + key_fn=lambda m: (m, 'v2'), + make_fn=_make_execution_function) def _make_execution_function(model, mode): @@ -67,12 +75,12 @@ def _make_execution_function(model, mode): def distributed_function(input_iterator): """A single step of the distributed execution across replicas.""" - args = _prepare_feed_values(model, input_iterator, mode) # Call `Model.{train,test,predict}_on_batch` on every replica passing # PerReplicas as arguments. On every replica inside this call, each # PerReplica object will return the value for that replica. The outputs # are PerReplicas too. strategy = distribution_strategy_context.get_strategy() + args = _prepare_feed_values(model, input_iterator, mode, strategy) outputs = strategy.experimental_run_v2( per_replica_function, args=args) # Out of PerReplica outputs reduce or pick values to return. @@ -92,12 +100,38 @@ def execution_function(input_fn): return execution_function +def _get_or_make_on_batch_function(model, mode): + """Makes or reuses function to run one step of distributed model execution.""" + return _get_or_make_function( + model, mode, + # Use a key with 'v2' to distinguish from fall-back execution functions. + key_fn=lambda m: (m, 'v2_on_batch'), + make_fn=_make_on_batch_function) + + +def _make_on_batch_function(model, mode): + """Creates a function of Model.*_on_batch methods.""" + if mode == ModeKeys.TRAIN: + func = training_eager.train_on_batch + elif mode == ModeKeys.TEST: + func = training_eager.test_on_batch + else: + func = model + + if not model.run_eagerly: + # Pass `experimental_relax_shapes` to avoid retracing for dynamic batch size, + # variable length sequences, etc. + func = def_function.function(func, experimental_relax_shapes=True) + + return func + + def _non_none_constant_value(v): constant_value = tensor_util.constant_value(v) return constant_value if constant_value is not None else v -def _prepare_feed_values(model, inputs, mode): +def _prepare_feed_values(model, inputs, mode, strategy): """Prepare feed values to the model execution function. Arguments: @@ -106,6 +140,7 @@ def _prepare_feed_values(model, inputs, mode): model inputs may be lists, single values, or dicts mapping input feed names to values. mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT. + strategy: The current distribution strategy for the model. Returns: Feed values for the model in the given mode. This is a tuple of @@ -114,7 +149,7 @@ def _prepare_feed_values(model, inputs, mode): for inputs will always be wrapped in lists. """ # For predict, we need to extract the manually added batch_index first. - with_batch_index = mode == ModeKeys.PREDICT + with_batch_index = _should_add_batch_index_to_element(strategy, mode) inputs, targets, sample_weights, batch_index = _get_input_from_iterator( inputs, with_batch_index) @@ -175,7 +210,10 @@ def _predict_on_batch(x, y=None, sample_weights=None, batch_index=None): del y, sample_weights # Note that the x and batch_index is already per-replica value. result = predict_on_batch(model, x) - return (batch_index, result) + if batch_index is None: + return result + else: + return batch_index, result func = _predict_on_batch @@ -195,6 +233,9 @@ def _aggregate_predict_results(strategy, batch_outs, model): if not isinstance(batch_outs, list): batch_outs = [batch_outs] + with_batch_index = _should_add_batch_index_to_element( + strategy, ModeKeys.PREDICT) + # batch_outs is in following structure: # [ # replica_1_batch_index, replica_2_batch_index, ...., replica_x_batch_index, @@ -202,14 +243,19 @@ def _aggregate_predict_results(strategy, batch_outs, model): # ...... # replica_1_output_y, replica_2_output_y, ...., replica_x_output_y, # ] - batch_index, batch_outs = batch_outs[:num_replicas], batch_outs[num_replicas:] - batch_index = dist_utils.concat_along_batch_dimension(batch_index) - # Reorder the batch_index for it to do proper gather. Eg, if the original - # index is [0, 2, 4, 6, 1, 3, 5, 7], then the index for gather should be - # [0, 4, 1, 5, 2, 6, 3, 7]. - batch_index = np.argsort(batch_index) - # Only need to gather if the batch index is not sorted. - need_batch_index_gather = np.any(np.diff(batch_index) < 0) + # The replica_x_batch_index is optional and depended on teh strategy type. + if with_batch_index: + batch_index, batch_outs = (batch_outs[:num_replicas], + batch_outs[num_replicas:]) + batch_index = dist_utils.concat_along_batch_dimension(batch_index) + # Reorder the batch_index for it to do proper gather. Eg, if the original + # index is [0, 2, 4, 6, 1, 3, 5, 7], then the index for gather should be + # [0, 4, 1, 5, 2, 6, 3, 7]. + batch_index = np.argsort(batch_index) + # Only need to gather if the batch index is not sorted. + need_batch_index_gather = np.any(np.diff(batch_index) < 0) + else: + need_batch_index_gather = False total_batch_outs = [] for i in range(num_outputs): @@ -286,13 +332,36 @@ def _add_batch_index_to_element(dataset): return dataset.map(lambda *inp: (math_ops.range(_get_batch_size(inp)), inp)) +def _should_add_batch_index_to_element(strategy, mode): + """Whether or not the batch index should be added to the input dataset. + + See docstring of _add_batch_index_to_element() for more details. So far the + batch index is only need when using TPUStrategy with a multi-worker setting. + We will try to avoid adding batch index for other cases since it has the + performance implication. + + Args: + strategy: the current distribution strategy for the model. + mode: the current mode (Training/Eval/Predict) for the model. + Returns: + Boolean, whether the batch index should be added for the input data to + preserve the ordering. + """ + # TODO(priyag, rxsang): Come up a better way to determine when the batch index + # should be added. + return (mode == ModeKeys.PREDICT + and dist_utils.is_tpu_strategy(strategy) + and strategy.num_replicas_in_sync > 1) + + def train_on_batch( model, x, y=None, sample_weight=None, class_weight=None, - reset_metrics=True): + reset_metrics=True, + standalone=False): """Runs a single gradient update on a single batch of data. Arguments: @@ -324,6 +393,8 @@ class during training. This can be useful to tell the model to "pay reset_metrics: If `True`, the metrics returned will be only for this batch. If `False`, the metrics will be statefully accumulated across batches. + standalone: If True, this method is not called as part of + Model.fit/evaluate/predict and can therefore be tf.function'd. Returns: Scalar training loss @@ -348,7 +419,13 @@ class during training. This can be useful to tell the model to "pay # at this point because of the check above. `train_on_batch` is being run # for each replica by `model._distribution_strategy` and the same code path # as Eager is expected to be taken. - outputs = training_eager.train_on_batch( + + if standalone: + train_on_batch_fn = _get_or_make_on_batch_function(model, ModeKeys.TRAIN) + else: + train_on_batch_fn = training_eager.train_on_batch + + outputs = train_on_batch_fn( model, x, y, @@ -362,7 +439,8 @@ class during training. This can be useful to tell the model to "pay return outputs -def test_on_batch(model, x, y=None, sample_weight=None, reset_metrics=True): +def test_on_batch(model, x, y=None, sample_weight=None, reset_metrics=True, + standalone=False): """Test the model on a single batch of samples. Arguments: @@ -392,6 +470,8 @@ def test_on_batch(model, x, y=None, sample_weight=None, reset_metrics=True): reset_metrics: If `True`, the metrics returned will be only for this batch. If `False`, the metrics will be statefully accumulated across batches. + standalone: If True, this method is not called as part of + Model.fit/evaluate/predict and can therefore be tf.function'd. Returns: Scalar test loss (if the model has a single output and no metrics) @@ -411,7 +491,13 @@ def test_on_batch(model, x, y=None, sample_weight=None, reset_metrics=True): x, y, sample_weight=sample_weight, extract_tensors_from_dataset=True) batch_size = array_ops.shape(nest.flatten(x, expand_composites=True)[0])[0] - outputs = training_eager.test_on_batch( + + if standalone: + test_on_batch_fn = _get_or_make_on_batch_function(model, ModeKeys.TEST) + else: + test_on_batch_fn = training_eager.test_on_batch + + outputs = test_on_batch_fn( model, x, y, @@ -425,7 +511,7 @@ def test_on_batch(model, x, y=None, sample_weight=None, reset_metrics=True): return outputs -def predict_on_batch(model, x): +def predict_on_batch(model, x, standalone=False): """Returns predictions for a single batch of samples. Arguments: @@ -436,6 +522,8 @@ def predict_on_batch(model, x): - A TensorFlow tensor, or a list of tensors (in case the model has multiple inputs). - A `tf.data` dataset. + standalone: If True, this method is not called as part of + Model.fit/evaluate/predict and can therefore be tf.function'd. Returns: Numpy array(s) of predictions. @@ -458,5 +546,11 @@ def predict_on_batch(model, x): if len(inputs) == 1: inputs = inputs[0] + if standalone: + predict_on_batch_fn = _get_or_make_on_batch_function( + model, ModeKeys.PREDICT) + else: + predict_on_batch_fn = model + with backend.eager_learning_phase_scope(0): - return model(inputs) # pylint: disable=not-callable + return predict_on_batch_fn(inputs) # pylint: disable=not-callable diff --git a/tensorflow/python/keras/engine/training_v2_utils_test.py b/tensorflow/python/keras/engine/training_v2_utils_test.py index 84f90fe9a820b5..4499ad3c8c65f7 100644 --- a/tensorflow/python/keras/engine/training_v2_utils_test.py +++ b/tensorflow/python/keras/engine/training_v2_utils_test.py @@ -21,6 +21,7 @@ import collections from absl.testing import parameterized +import mock import numpy as np @@ -81,17 +82,20 @@ def dense_map_fn(i): start = 0 for batch in distributed_data: - batch_result = self.predict_loop(batch) - final_result = training_v2_utils._aggregate_predict_results( - self.strategy, batch_result, self.mock_model) - - # Make sure the dense result is in a sorted order. - expected_result = np.arange( - start=start, stop=start+self.batch_size).reshape((-1, 1)) - expected_result = np.tile(expected_result, 6).reshape( - (-1,) + self.dense_shape) - self.assertAllClose(final_result[0], expected_result) - start += self.batch_size + with mock.patch.object(training_v2_utils, + '_should_add_batch_index_to_element', + fake_should_add_batch_index_to_element): + batch_result = self.predict_loop(batch) + final_result = training_v2_utils._aggregate_predict_results( + self.strategy, batch_result, self.mock_model) + + # Make sure the dense result is in a sorted order. + expected_result = np.arange( + start=start, stop=start+self.batch_size).reshape((-1, 1)) + expected_result = np.tile(expected_result, 6).reshape( + (-1,) + self.dense_shape) + self.assertAllClose(final_result[0], expected_result) + start += self.batch_size @combinations.generate(combinations.combine(tf_api_version=[1, 2], mode='eager')) @@ -108,14 +112,17 @@ def sparse_map_fn(i): start = 0 for batch in distributed_data: - batch_result = self.predict_loop(batch) - final_result = training_v2_utils._aggregate_predict_results( - self.strategy, batch_result, self.mock_model) - - # Make sure the dense result is in a sorted order. - expected_values = np.arange(start=start, stop=start+self.batch_size) - self.assertAllClose(final_result[0].values, expected_values) - start += self.batch_size + with mock.patch.object(training_v2_utils, + '_should_add_batch_index_to_element', + fake_should_add_batch_index_to_element): + batch_result = self.predict_loop(batch) + final_result = training_v2_utils._aggregate_predict_results( + self.strategy, batch_result, self.mock_model) + + # Make sure the dense result is in a sorted order. + expected_values = np.arange(start=start, stop=start+self.batch_size) + self.assertAllClose(final_result[0].values, expected_values) + start += self.batch_size @combinations.generate(combinations.combine(tf_api_version=[1, 2], mode='eager')) @@ -129,14 +136,24 @@ def ragged_map_fn(i): start = 0 for batch in distributed_data: - batch_result = self.predict_loop(batch) - final_result = training_v2_utils._aggregate_predict_results( - self.strategy, batch_result, self.mock_model) - - # Make sure the dense result is in a sorted order. - expected_values = np.arange(start=start, stop=start+self.batch_size) - self.assertAllClose(final_result[0].flat_values, expected_values) - start += self.batch_size + with mock.patch.object(training_v2_utils, + '_should_add_batch_index_to_element', + fake_should_add_batch_index_to_element): + batch_result = self.predict_loop(batch) + final_result = training_v2_utils._aggregate_predict_results( + self.strategy, batch_result, self.mock_model) + + # Make sure the dense result is in a sorted order. + expected_values = np.arange(start=start, stop=start+self.batch_size) + self.assertAllClose(final_result[0].flat_values, expected_values) + start += self.batch_size + + +def fake_should_add_batch_index_to_element(strategy, mode): + # Ignore the strategy instance check since we were using the MirroredStrategy + # for testing. + del strategy + return mode == ModeKeys.PREDICT if __name__ == '__main__': diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py index 7655e5f6e0ea0a..07cb1bdf1b3164 100644 --- a/tensorflow/python/keras/layers/__init__.py +++ b/tensorflow/python/keras/layers/__init__.py @@ -18,12 +18,32 @@ from __future__ import division from __future__ import print_function +from tensorflow.python import tf2 + # Generic layers. # pylint: disable=g-bad-import-order +# pylint: disable=g-import-not-at-top from tensorflow.python.keras.engine.input_layer import Input from tensorflow.python.keras.engine.input_layer import InputLayer from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.base_preprocessing_layer import PreprocessingLayer + +# Preprocessing layers. +if tf2.enabled(): + from tensorflow.python.keras.layers.preprocessing.normalization import Normalization + from tensorflow.python.keras.layers.preprocessing.normalization_v1 import Normalization as NormalizationV1 + NormalizationV2 = Normalization + from tensorflow.python.keras.layers.preprocessing.text_vectorization import TextVectorization + from tensorflow.python.keras.layers.preprocessing.text_vectorization_v1 import TextVectorization as TextVectorizationV1 + TextVectorizationV2 = TextVectorization +else: + from tensorflow.python.keras.layers.preprocessing.normalization_v1 import Normalization + from tensorflow.python.keras.layers.preprocessing.normalization import Normalization as NormalizationV2 + NormalizationV1 = Normalization + from tensorflow.python.keras.layers.preprocessing.text_vectorization_v1 import TextVectorization + from tensorflow.python.keras.layers.preprocessing.text_vectorization import TextVectorization as TextVectorizationV2 + TextVectorizationV1 = TextVectorization # Advanced activations. from tensorflow.python.keras.layers.advanced_activations import LeakyReLU @@ -114,8 +134,14 @@ # Normalization layers. from tensorflow.python.keras.layers.normalization import LayerNormalization -from tensorflow.python.keras.layers.normalization import BatchNormalization -from tensorflow.python.keras.layers.normalization_v2 import BatchNormalization as BatchNormalizationV2 +if tf2.enabled(): + from tensorflow.python.keras.layers.normalization_v2 import BatchNormalization + from tensorflow.python.keras.layers.normalization import BatchNormalization as BatchNormalizationV1 + BatchNormalizationV2 = BatchNormalization +else: + from tensorflow.python.keras.layers.normalization import BatchNormalization + from tensorflow.python.keras.layers.normalization_v2 import BatchNormalization as BatchNormalizationV2 + BatchNormalizationV1 = BatchNormalization # Kernelized layers. from tensorflow.python.keras.layers.kernelized import RandomFourierFeatures @@ -156,14 +182,32 @@ from tensorflow.python.keras.layers.recurrent import PeepholeLSTMCell from tensorflow.python.keras.layers.recurrent import SimpleRNN -from tensorflow.python.keras.layers.recurrent import GRU -from tensorflow.python.keras.layers.recurrent import GRUCell -from tensorflow.python.keras.layers.recurrent import LSTM -from tensorflow.python.keras.layers.recurrent import LSTMCell -from tensorflow.python.keras.layers.recurrent_v2 import GRU as GRU_v2 -from tensorflow.python.keras.layers.recurrent_v2 import GRUCell as GRUCell_v2 -from tensorflow.python.keras.layers.recurrent_v2 import LSTM as LSTM_v2 -from tensorflow.python.keras.layers.recurrent_v2 import LSTMCell as LSTMCell_v2 +if tf2.enabled(): + from tensorflow.python.keras.layers.recurrent_v2 import GRU + from tensorflow.python.keras.layers.recurrent_v2 import GRUCell + from tensorflow.python.keras.layers.recurrent_v2 import LSTM + from tensorflow.python.keras.layers.recurrent_v2 import LSTMCell + from tensorflow.python.keras.layers.recurrent import GRU as GRUV1 + from tensorflow.python.keras.layers.recurrent import GRUCell as GRUCellV1 + from tensorflow.python.keras.layers.recurrent import LSTM as LSTMV1 + from tensorflow.python.keras.layers.recurrent import LSTMCell as LSTMCellV1 + GRUV2 = GRU + GRUCellV2 = GRUCell + LSTMV2 = LSTM + LSTMCellV2 = LSTMCell +else: + from tensorflow.python.keras.layers.recurrent import GRU + from tensorflow.python.keras.layers.recurrent import GRUCell + from tensorflow.python.keras.layers.recurrent import LSTM + from tensorflow.python.keras.layers.recurrent import LSTMCell + from tensorflow.python.keras.layers.recurrent_v2 import GRU as GRUV2 + from tensorflow.python.keras.layers.recurrent_v2 import GRUCell as GRUCellV2 + from tensorflow.python.keras.layers.recurrent_v2 import LSTM as LSTMV2 + from tensorflow.python.keras.layers.recurrent_v2 import LSTMCell as LSTMCellV2 + GRUV1 = GRU + GRUCellV1 = GRUCell + LSTMV1 = LSTM + LSTMCellV1 = LSTMCell # Convolutional-recurrent layers. from tensorflow.python.keras.layers.convolutional_recurrent import ConvLSTM2D diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py index aad66429b75000..3ebf94490d8e07 100644 --- a/tensorflow/python/keras/layers/core.py +++ b/tensorflow/python/keras/layers/core.py @@ -20,11 +20,13 @@ import copy import sys +import textwrap import types as python_types import warnings import numpy as np +from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -47,6 +49,8 @@ from tensorflow.python.ops import sparse_ops from tensorflow.python.ops import standard_ops from tensorflow.python.ops import variable_scope +from tensorflow.python.platform import tf_logging +from tensorflow.python.training.tracking import base as trackable from tensorflow.python.util import nest from tensorflow.python.util import tf_inspect from tensorflow.python.util.tf_export import keras_export @@ -690,7 +694,7 @@ class Lambda(Layer): can be used when constructing `Sequential` and Functional API models. `Lambda` layers are best suited for simple operations or quick experimentation. For more advanced usecases, follow - [this guide](https://www.tensorflow.org/alpha/guide/keras/custom_layers_and_models) + [this guide](https://www.tensorflow.org/guide/keras/custom_layers_and_models) for subclassing `tf.keras.layers.Layer`. The main reason to subclass `tf.keras.layers.Layer` instead of using a @@ -721,30 +725,34 @@ def antirectifier(x): model.add(Lambda(antirectifier)) ``` - Variables can be created within a `Lambda` layer. Like with - other layers, these variables will be created only once and reused - if the `Lambda` layer is called on new inputs. If creating more - than one variable in a given `Lambda` instance, be sure to use - a different name for each variable. Note that calling sublayers - from within a `Lambda` is not supported. + Variables: + While it is possible to use Variables with Lambda layers, this practice is + discouraged as it can easily lead to bugs. For instance, consider the + following layer: - Example of variable creation: + ```python + scale = tf.Variable(1.) + scale_layer = tf.keras.layers.Lambda(lambda x: x * scale) + ``` - ```python - def linear_transform(x): - v1 = tf.Variable(1., name='multiplier') - v2 = tf.Variable(0., name='bias') - return x*v1 + v2 - - linear_layer = Lambda(linear_transform) - model.add(linear_layer) - model.add(keras.layers.Dense(10, activation='relu')) - model.add(linear_layer) # Reuses existing Variables - ``` + Because scale_layer does not directly track the `scale` variable, it will + not appear in `scale_layer.trainable_weights` and will therefore not be + trained if `scale_layer` is used in a Model. + + A better pattern is to write a subclassed Layer: + + ```python + class ScaleLayer(tf.keras.layers.Layer): + def __init__(self): + super(ScaleLayer, self).__init__() + self.scale = tf.Variable(1.) + + def call(self, inputs): + return inputs * self.scale + ``` - Note that creating two instances of `Lambda` using the same function - will *not* share Variables between the two instances. Each instance of - `Lambda` will create and manage its own weights. + In general, Lambda layers can be convenient for simple stateless + computation, but anything more complex should use a subclass Layer instead. Arguments: function: The function to be evaluated. Takes input tensor as first @@ -769,22 +777,24 @@ def linear_transform(x): Output shape: Specified by `output_shape` argument """ + @trackable.no_automatic_dependency_tracking def __init__(self, function, output_shape=None, mask=None, arguments=None, **kwargs): super(Lambda, self).__init__(**kwargs) + + self.arguments = arguments or {} self.function = function - self.arguments = arguments if arguments else {} + if mask is not None: self.supports_masking = True self.mask = mask self._supports_ragged_inputs = True self._output_shape = output_shape - self._variable_dict = {} - # These attributes are inherited from `Layer`. - self._trainable_weights = [] - self._non_trainable_weights = [] - function_args = tf_inspect.getfullargspec(self.function).args + # Warning on every invocation will be quite irksome in Eager mode. + self._already_warned = False + + function_args = tf_inspect.getfullargspec(function).args self._fn_expects_training_arg = 'training' in function_args self._fn_expects_mask_arg = 'mask' in function_args @@ -818,26 +828,69 @@ def _add_batch(shape): return nest.map_structure(_add_batch, output_shapes) def call(self, inputs, mask=None, training=None): - arguments = self.arguments + # We must copy for thread safety, but it only needs to be a shallow copy. + kwargs = {k: v for k, v in self.arguments.items()} if self._fn_expects_mask_arg: - arguments['mask'] = mask + kwargs['mask'] = mask if self._fn_expects_training_arg: - arguments['training'] = training - with variable_scope.variable_creator_scope(self._variable_creator): - return self.function(inputs, **arguments) - - def _variable_creator(self, next_creator, **kwargs): - name = kwargs['name'] - if name in self._variable_dict: - return self._variable_dict[name] - var = next_creator(**kwargs) - self._variable_dict[name] = var - if var.trainable: - self._trainable_weights.append(var) - else: - self._non_trainable_weights.append(var) - K.track_variable(var) - return var + kwargs['training'] = training + + created_variables = [] + def _variable_creator(next_creator, **kwargs): + var = next_creator(**kwargs) + created_variables.append(var) + return var + + with backprop.GradientTape(watch_accessed_variables=True) as tape,\ + variable_scope.variable_creator_scope(_variable_creator): + result = self.function(inputs, **kwargs) + self._check_variables(created_variables, tape.watched_variables()) + return result + + def _check_variables(self, created_variables, accessed_variables): + if not created_variables and not accessed_variables: + # In the common case that a Lambda layer does not touch a Variable, we + # don't want to incur the runtime cost of assembling any state used for + # checking only to immediately discard it. + return + + tracked_weights = set(v.experimental_ref() for v in self.weights) + untracked_new_vars = [v for v in created_variables + if v.experimental_ref() not in tracked_weights] + if untracked_new_vars: + variable_str = '\n'.join([' {}'.format(i) for i in untracked_new_vars]) + error_str = textwrap.dedent( + ''' + The following Variables were created within a Lambda layer ({name}) + but are not tracked by said layer: + {variable_str} + The layer cannot safely ensure proper Variable reuse across multiple + calls, and consquently this behavior is disallowed for safety. Lambda + layers are not well suited to stateful computation; instead, writing a + subclassed Layer is the recommend way to define layers with + Variables.''' + ).format(name=self.name, variable_str=variable_str) + raise ValueError(error_str) + + untracked_used_vars = [v for v in accessed_variables + if v.experimental_ref() not in tracked_weights] + if untracked_used_vars and not self._already_warned: + variable_str = '\n'.join([' {}'.format(i) for i in untracked_used_vars]) + self._warn(textwrap.dedent( + ''' + The following Variables were used a Lambda layer's call ({name}), but + are not present in its tracked objects: + {variable_str} + It is possible that this is intended behavior, but it is more likely + an omission. This is a strong indication that this layer should be + formulated as a subclassed Layer rather than a Lambda layer.''' + ).format(name=self.name, variable_str=variable_str)) + self._already_warned = True + + def _warn(self, msg): + # This method will be overridden in a unit test to raise an error, because + # self.assertWarns is not universally implemented. + return tf_logging.warn(msg) def compute_mask(self, inputs, mask=None): if callable(self.mask): diff --git a/tensorflow/python/keras/layers/core_test.py b/tensorflow/python/keras/layers/core_test.py index aa7b42d0e950aa..8e6ad99873cff2 100644 --- a/tensorflow/python/keras/layers/core_test.py +++ b/tensorflow/python/keras/layers/core_test.py @@ -18,6 +18,8 @@ from __future__ import division from __future__ import print_function +import textwrap + import numpy as np from tensorflow.python import keras @@ -225,17 +227,6 @@ def test_lambda_config_serialization(self): self.assertAllEqual(layer._output_shape, (1, 1)) self.assertAllEqual(layer.mask(1, True), True) - def test_lambda_with_variable(self): - - def fn(x): - return x * variables.Variable(2., name='multiplier') - - layer = keras.layers.Lambda(fn) - for _ in range(10): - layer(np.ones((10, 10), 'float32')) - self.assertLen(layer.trainable_weights, 1) - self.assertEqual(layer.trainable_weights[0].name, 'lambda/multiplier:0') - def test_lambda_with_training_arg(self): def fn(x, training=True): @@ -283,19 +274,25 @@ def add_one(inputs): expected_out = ragged_factory_ops.constant([[2.0], [3.0, 4.0]]) self.assertAllClose(out, expected_out) + class TestStatefulLambda(keras_parameterized.TestCase): @keras_parameterized.run_all_keras_modes @keras_parameterized.run_with_all_model_types def test_lambda_with_variable_in_model(self): - - def lambda_fn(x): - # Variable will only get created once. - v = variables.Variable(1., trainable=True) + v = variables.Variable(1., trainable=True) + def lambda_fn(x, v): return x * v - model = testing_utils.get_model_from_layers( - [keras.layers.Lambda(lambda_fn)], input_shape=(10,)) + # While it is generally not advised to mix Variables with Lambda layers, if + # the variables are explicitly set as attributes then they are still + # tracked. This is consistent with the base Layer behavior. + layer = keras.layers.Lambda(lambda_fn, arguments={'v': v}) + self.assertLen(layer.trainable_weights, 0) + layer.v = v + self.assertLen(layer.trainable_weights, 1) + + model = testing_utils.get_model_from_layers([layer], input_shape=(10,)) model.compile( keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae', @@ -306,6 +303,66 @@ def lambda_fn(x): self.assertLen(model.trainable_weights, 1) self.assertAllClose(keras.backend.get_value(model.trainable_weights[0]), 2.) + @keras_parameterized.run_all_keras_modes + @keras_parameterized.run_with_all_model_types + def test_creation_inside_lambda(self): + def lambda_fn(x): + scale = variables.Variable(1., trainable=True, name='scale') + shift = variables.Variable(1., trainable=True, name='shift') + return x * scale + shift + + expected_error = textwrap.dedent(r''' + ( )?The following Variables were created within a Lambda layer \(shift_and_scale\) + ( )?but are not tracked by said layer: + ( )? Int32Converter; // Floating-point support +// Returns `true` if `out` overflows when converted from `as_double`. +template +static inline bool CheckForOverflow(double as_double, T* out) { + return (sizeof(T) < sizeof(double) && std::isinf(*out) && + std::isfinite(as_double)); +} + +// There is no `std::isinf` that takes `Eigen::half` as argument but Eigen +// provides `Eigen::half_impl::isinf` instead. +template <> +inline bool CheckForOverflow(double as_double, Eigen::half* out) { + return (sizeof(Eigen::half) < sizeof(double) && + Eigen::half_impl::isinf(*out) && std::isfinite(as_double)); +} + template static const char* ConvertOneFloat(PyObject* v, T* out) { if (PyErr_Occurred()) { @@ -382,20 +398,19 @@ static const char* ConvertOneFloat(PyObject* v, T* out) { const double as_double = PyFloat_AS_DOUBLE(v); *out = static_cast(as_double); // Check for overflow - if (TF_PREDICT_FALSE(sizeof(T) < sizeof(double) && std::isinf(*out) && - std::isfinite(as_double))) { + if (TF_PREDICT_FALSE(CheckForOverflow(as_double, out))) { return ErrorOutOfRangeDouble; } return nullptr; } #if PY_MAJOR_VERSION < 3 if (PyInt_Check(v)) { - *out = PyInt_AS_LONG(v); + *out = static_cast(PyInt_AS_LONG(v)); return nullptr; } #endif if (PyLong_Check(v)) { - *out = PyLong_AsDouble(v); + *out = static_cast(PyLong_AsDouble(v)); if (PyErr_Occurred()) return ErrorOutOfRangeDouble; return nullptr; } @@ -444,13 +459,7 @@ struct ConverterTraits { static const tensorflow::DataType kTypeEnum = DT_HALF; static const char* ConvertScalar(PyObject* v, Eigen::half* out) { - // NOTE(nareshmodi): Is there a way to convert to C double without the - // intermediate Python double? This will help with ConvertOneFloat as well. - Safe_PyObjectPtr as_float = make_safe(PyNumber_Float(v)); - double v_double = PyFloat_AS_DOUBLE(as_float.get()); - *out = Eigen::half(v_double); - - return nullptr; + return ConvertOneFloat(v, out); } }; @@ -591,7 +600,9 @@ Status PySeqToTensor(PyObject* obj, DataType dtype, Tensor* ret) { break; case DT_HALF: - RETURN_STRING_AS_STATUS(NumpyHalfConverter::Convert(obj, shape, ret)); + if (NumpyHalfConverter::Convert(obj, shape, ret) == nullptr) + return Status::OK(); + break; case DT_INT64: if (Int64Converter::Convert(obj, shape, ret) == nullptr) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 0836e9f30c79d8..6a941015e621c9 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -109,6 +109,7 @@ tf_export(v1=["arg_max"])(arg_max) tf_export(v1=["arg_min"])(arg_min) + # This is set by resource_variable_ops.py. It is included in this way since # there is a circular dependency between math_ops and resource_variable_ops _resource_variable_type = None @@ -4187,3 +4188,36 @@ def reciprocal_no_nan(x, name=None): x = ops.convert_to_tensor(x, name="x") one = constant_op.constant(1, dtype=x.dtype.base_dtype, name="one") return gen_math_ops.div_no_nan(one, x, name=scope) + + +@tf_export("math.erfinv") +@dispatch.add_dispatch_support +def erfinv(x, name=None): + """Compute inverse error function. + + Given `x`, compute the inverse error function of `x`. This function + is the inverse of `tf.math.erf`. + + Args: + x: `Tensor` with type `float` or `double`. + name: A name for the operation (optional). + Returns: + Inverse error function of `x`. + """ + with ops.name_scope(name, "erfinv", [x]): + return gen_math_ops.erfinv(x) + + +@tf_export("math.ndtri") +@dispatch.add_dispatch_support +def ndtri(x, name=None): + """Compute quantile of Standard Normal. + + Args: + x: `Tensor` with type `float` or `double`. + name: A name for the operation (optional). + Returns: + Inverse error function of `x`. + """ + with ops.name_scope(name, "ndtri", [x]): + return gen_math_ops.ndtri(x) diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py index 6582f37d65be87..7ae9e22858bb12 100644 --- a/tensorflow/python/ops/special_math_ops_test.py +++ b/tensorflow/python/ops/special_math_ops_test.py @@ -436,6 +436,16 @@ def test_opt_einsum_cached(self): with test.mock.patch.object( opt_einsum, 'contract_path', wraps=opt_einsum.contract_path) as mock_contract_path: + + # explicitly clear the lru_cache contents for the method + # special_math_ops.get_opt_einsum_contract_path + # We need to do this because other tests in this file invoke that method + # with the same input args (as input_1 and input_2 above), and if + # those tests run before this test, then the call_count for the method + # mock_contract_path will not increment. + if six.PY3: + special_math_ops._get_opt_einsum_contract_path.cache_clear() + self.assertEqual(mock_contract_path.call_count, 0) self._check(*input_1) self.assertEqual(mock_contract_path.call_count, 1) diff --git a/tensorflow/python/platform/self_check.py b/tensorflow/python/platform/self_check.py index 33aed306467dc8..f6cf7705e1390d 100644 --- a/tensorflow/python/platform/self_check.py +++ b/tensorflow/python/platform/self_check.py @@ -42,17 +42,22 @@ def preload_check(): # we load the Python extension, so that we can raise an actionable error # message if they are not found. import ctypes # pylint: disable=g-import-not-at-top - if hasattr(build_info, "msvcp_dll_name"): - try: - ctypes.WinDLL(build_info.msvcp_dll_name) - except OSError: + if hasattr(build_info, "msvcp_dll_names"): + missing = [] + for dll_name in build_info.msvcp_dll_names.split(","): + try: + ctypes.WinDLL(dll_name) + except OSError: + missing.append(dll_name) + if missing: raise ImportError( - "Could not find %r. TensorFlow requires that this DLL be " - "installed in a directory that is named in your %%PATH%% " - "environment variable. You may install this DLL by downloading " - "Visual C++ 2015 Redistributable Update 3 from this URL: " - "https://www.microsoft.com/en-us/download/details.aspx?id=53587" - % build_info.msvcp_dll_name) + "Could not find the DLL(s) %r. TensorFlow requires that these DLLs " + "be installed in a directory that is named in your %%PATH%% " + "environment variable. You may install these DLLs by downloading " + '"Microsoft C++ Redistributable for Visual Studio 2015, 2017 and ' + '2019" for your platform from this URL: ' + "https://support.microsoft.com/help/2977003/the-latest-supported-visual-c-downloads" + % " or ".join(missing)) else: # TODO(mrry): Consider adding checks for the Linux and Mac OS X builds. pass diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 25106769c1563b..e3984c37657b34 100755 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -110,6 +110,7 @@ limitations under the License. %rename("%s") TFE_ContextOptionsSetDevicePlacementPolicy; %rename("%s") TFE_ContextOptionsSetMirroringPolicy; %rename("%s") TFE_ContextOptionsSetAsync; +%rename("%s") TFE_ContextOptionsSetLazyRemoteInputsCopy; %rename("%s") TFE_DeleteContextOptions; %rename("%s") TFE_Py_TensorShapeSlice; %rename("%s") TFE_Py_TensorShapeOnDevice; diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py index e178c362d04637..8662cbaea518c7 100644 --- a/tensorflow/python/saved_model/save_test.py +++ b/tensorflow/python/saved_model/save_test.py @@ -442,6 +442,53 @@ def f(unused_v): save.save(root, os.path.join(self.get_temp_dir(), "saved_model"), signatures=root.f) + def test_export_correct_output_shapes(self): + """Asserts that nodes are exported with the correct number of output shapes. + + After backpropagation rewrite, functions are rewritten with additional + outputs. When exporting to SavedModel, the shapes of the additional outputs + were incorrectly added to the FunctionDef proto (b/133666530). + """ + obj = tracking.AutoTrackable() + obj.v = variables.Variable(2.) + + @def_function.function(input_signature=[ + tensor_spec.TensorSpec(None, dtypes.float32)]) + def f(x): + return (math_ops.multiply(obj.v, x), + math_ops.multiply(obj.v, (x+1)), + None) + obj.f = f + + @def_function.function(input_signature=[ + tensor_spec.TensorSpec(None, dtypes.float32)]) + def g(x): + return obj.f(x)[1] + obj.g = g + + # After the following lines, the concrete functions of obj.g and obj.f are + # rewritten with many extra outputs. + with backprop.GradientTape(): + obj.g(constant_op.constant(3.0)) + + save_dir = os.path.join(self.get_temp_dir(), "saved_model") + save.save(obj, save_dir, signatures={"g": obj.g}) + graph_def = loader_impl.parse_saved_model(save_dir).meta_graphs[0].graph_def + + def assert_correct_number_of_output_shapes(node): + if node.op == "StatefulPartitionedCall": + fn_name = node.attr["f"].func.name + if fn_name.startswith("__inference_f"): + self.assertLen(node.attr["_output_shapes"].list.shape, 2) + if fn_name.startswith("__inference_g"): + self.assertLen(node.attr["_output_shapes"].list.shape, 1) + + for f in graph_def.library.function: + if(f.signature.name.startswith("__inference_f") or + f.signature.name.startswith("__inference_g")): + for node in f.node_def: + assert_correct_number_of_output_shapes(node) + class SavingOptionsTest(test.TestCase): diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl index 1ddf0bcb1fbc13..b2981b14209138 100644 --- a/tensorflow/python/tools/api/generator/api_init_files.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files.bzl @@ -104,6 +104,7 @@ KERAS_API_INIT_FILES = [ "keras/initializers/__init__.py", "keras/layers/__init__.py", "keras/layers/experimental/__init__.py", + "keras/layers/experimental/preprocessing/__init__.py", "keras/losses/__init__.py", "keras/metrics/__init__.py", "keras/mixed_precision/__init__.py", diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl index 62ecd8a284aa49..31e0c6ca457795 100644 --- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl @@ -129,6 +129,7 @@ KERAS_API_INIT_FILES_V1 = [ "keras/initializers/__init__.py", "keras/layers/__init__.py", "keras/layers/experimental/__init__.py", + "keras/layers/experimental/preprocessing/__init__.py", "keras/losses/__init__.py", "keras/metrics/__init__.py", "keras/mixed_precision/__init__.py", diff --git a/tensorflow/python/tools/api/generator/create_python_api.py b/tensorflow/python/tools/api/generator/create_python_api.py index 3af677322d67ee..80f663683c3ee0 100644 --- a/tensorflow/python/tools/api/generator/create_python_api.py +++ b/tensorflow/python/tools/api/generator/create_python_api.py @@ -243,11 +243,12 @@ def build(self): # from it using * import. Don't need this for lazy_loading because the # underscore symbols are already included in __all__ when passed in and # handled by TFModuleWrapper. + root_module_footer = '' if not self._lazy_loading: underscore_names_str = ', '.join( '\'%s\'' % name for name in self._underscore_names_in_root) - module_text_map[''] = module_text_map.get('', '') + ''' + root_module_footer = ''' _names_with_underscore = [%s] __all__ = [_s for _s in dir() if not _s.startswith('_')] __all__.extend([_s for _s in _names_with_underscore]) @@ -273,7 +274,7 @@ def build(self): footer_text_map[dest_module] = _DEPRECATION_FOOTER % ( dest_module, public_apis_name, deprecation, has_lite) - return module_text_map, footer_text_map + return module_text_map, footer_text_map, root_module_footer def format_import(self, source_module_name, source_name, dest_name): """Formats import statement. @@ -620,7 +621,11 @@ def create_api_files(output_files, packages, root_init_template, output_dir, os.makedirs(os.path.dirname(file_path)) open(file_path, 'a').close() - module_text_map, deprecation_footer_map = get_api_init_text( + ( + module_text_map, + deprecation_footer_map, + root_module_footer, + ) = get_api_init_text( packages, output_package, api_name, api_version, compat_api_versions, lazy_loading, use_relative_imports) @@ -652,6 +657,7 @@ def create_api_files(output_files, packages, root_init_template, output_dir, with open(root_init_template, 'r') as root_init_template_file: contents = root_init_template_file.read() contents = contents.replace('# API IMPORTS PLACEHOLDER', text) + contents = contents.replace('# __all__ PLACEHOLDER', root_module_footer) elif module in compat_module_to_template: # Read base init file for compat module with open(compat_module_to_template[module], 'r') as init_template_file: diff --git a/tensorflow/python/tools/api/generator/create_python_api_test.py b/tensorflow/python/tools/api/generator/create_python_api_test.py index 010f189dcb27f0..76404d6c82b33b 100644 --- a/tensorflow/python/tools/api/generator/create_python_api_test.py +++ b/tensorflow/python/tools/api/generator/create_python_api_test.py @@ -62,7 +62,7 @@ def tearDown(self): del sys.modules[_MODULE_NAME] def testFunctionImportIsAdded(self): - imports, _ = create_python_api.get_api_init_text( + imports, _, _ = create_python_api.get_api_init_text( packages=[create_python_api._DEFAULT_PACKAGE], output_package='tensorflow', api_name='tensorflow', @@ -97,7 +97,7 @@ def testFunctionImportIsAdded(self): msg='compat.v1 in %s' % str(imports.keys())) def testClassImportIsAdded(self): - imports, _ = create_python_api.get_api_init_text( + imports, _, _ = create_python_api.get_api_init_text( packages=[create_python_api._DEFAULT_PACKAGE], output_package='tensorflow', api_name='tensorflow', @@ -116,7 +116,7 @@ def testClassImportIsAdded(self): msg='%s not in %s' % (expected_import, str(imports))) def testConstantIsAdded(self): - imports, _ = create_python_api.get_api_init_text( + imports, _, _ = create_python_api.get_api_init_text( packages=[create_python_api._DEFAULT_PACKAGE], output_package='tensorflow', api_name='tensorflow', @@ -132,7 +132,7 @@ def testConstantIsAdded(self): msg='%s not in %s' % (expected, str(imports))) def testCompatModuleIsAdded(self): - imports, _ = create_python_api.get_api_init_text( + imports, _, _ = create_python_api.get_api_init_text( packages=[create_python_api._DEFAULT_PACKAGE], output_package='tensorflow', api_name='tensorflow', @@ -144,7 +144,7 @@ def testCompatModuleIsAdded(self): msg='compat.v1.test not in %s' % str(imports.keys())) def testNestedCompatModulesAreAdded(self): - imports, _ = create_python_api.get_api_init_text( + imports, _, _ = create_python_api.get_api_init_text( packages=[create_python_api._DEFAULT_PACKAGE], output_package='tensorflow', api_name='tensorflow', diff --git a/tensorflow/python/tpu/tpu_system_metadata.py b/tensorflow/python/tpu/tpu_system_metadata.py index 8628feee418479..1998e0e0aeb859 100644 --- a/tensorflow/python/tpu/tpu_system_metadata.py +++ b/tensorflow/python/tpu/tpu_system_metadata.py @@ -19,10 +19,10 @@ from __future__ import print_function import collections -import re from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as session_lib +from tensorflow.python.distribute import device_util from tensorflow.python.eager import context from tensorflow.python.framework import config from tensorflow.python.framework import device as tf_device @@ -35,8 +35,6 @@ _RETRY_TIMES = 12 * 24 # 1 day _INITIAL_TPU_SYSTEM_TIMEOUT_IN_MS = 300 * 1000 # 5 mins -_TPU_DEVICE_REG = re.compile(r'.*task:(\d+)/.*device:TPU:(\d+)$') - _DEFAULT_JOB_NAME = 'tpu_worker' _DEFAULT_COORDINATOR_JOB_NAME = 'coordinator' _LOCAL_MASTERS = ('', 'local') @@ -61,11 +59,11 @@ def _query_tpu_system_metadata(master_address, cluster_def=None, if context.executing_eagerly(): logical_devices = config.list_logical_devices() - devices = [] # We want the output type to match in both eager and session mode - for d in logical_devices: - devices.append(session_lib._DeviceAttributes(d.name, d.device_type, 0, 0)) # pylint: disable=protected-access + devices = [session_lib._DeviceAttributes(device_util.canonicalize(d.name), # pylint: disable=protected-access + d.device_type, 0, 0) + for d in logical_devices] else: # TODO(b/120564445): Replace with standard library for retries. retry_count = 1 @@ -97,11 +95,9 @@ def _query_tpu_system_metadata(master_address, cluster_def=None, raise ValueError(msg) for device in devices: - match = _TPU_DEVICE_REG.match(device.name) - if match: - host_id = match.group(1) - core_id = match.group(2) - device_dict[host_id].append(core_id) + spec = tf_device.DeviceSpec.from_string(device.name) + if spec.device_type == 'TPU': + device_dict[spec.task].append(spec.device_index) tpu_core_count += 1 num_of_cores_per_host = 0 diff --git a/tensorflow/python/training/experimental/loss_scaling_gradient_tape.py b/tensorflow/python/training/experimental/loss_scaling_gradient_tape.py index 4b75a74bc3b2a6..0afe4c78cafc3f 100644 --- a/tensorflow/python/training/experimental/loss_scaling_gradient_tape.py +++ b/tensorflow/python/training/experimental/loss_scaling_gradient_tape.py @@ -18,15 +18,16 @@ from __future__ import division from __future__ import print_function +from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.eager import backprop -from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops.unconnected_gradients import UnconnectedGradients from tensorflow.python.training.experimental import loss_scale as loss_scale_module from tensorflow.python.util import nest -from tensorflow.python.util.tf_export import tf_export -@tf_export("mixed_precision.experimental.LossScaleGradientTape", v1=[]) +# TODO(reedwm): Expose this. Currently it doesn't work with DistributionStrategy class LossScaleGradientTape(backprop.GradientTape): """A gradient tape that scales losses and unscales resulting gradients. @@ -60,6 +61,13 @@ class LossScaleGradientTape(backprop.GradientTape): grads = tape.gradient(loss, vars) opt.apply_gradients(zip(grads, vars)) ``` + + WARNING: Computing second-order (or higher) gradients with a + `LossScaleGradientTape` does not yet work properly when a + `tf.distribute.Strategy` is used. Computing second-order gradients will return + None instead of the gradient tensors. This only occurs when you nest multiple + gradient tapes under each other; if you do not nest them, this issue will not + occur. """ def __init__(self, @@ -133,22 +141,90 @@ def gradient(self, if self._tape is None: # pylint: disable=access-member-before-definition raise RuntimeError("GradientTape.gradient can only be called once on " "non-persistent tapes.") + if distribution_strategy_context.in_cross_replica_context(): + raise ValueError("LossScaleGradientTape.gradient() must be called in a " + "replica context.") + + # Note: DistributionStrategy does not support running a while loop in a + # replica context. So, we call `_compute_gradients_until_finite` in a cross- + # replica context. + replica_context = distribution_strategy_context.get_replica_context() + grads = replica_context.merge_call( + _compute_gradients_until_finite, + args=(self, self._loss_scale, target, sources, output_gradients, + unconnected_gradients)) - ready_to_update = False - grads = nest.map_structure(array_ops.zeros_like, sources) + if not self._outer_persistent: + self._tape = None # free up resources if a persistent tape was not needed + return grads - while not ready_to_update and self._loss_scale() > 1: - with self: # re-enter the gradient tape so it sees the loss scaling - loss_scale = self._loss_scale() - scaled_target = nest.map_structure(lambda t: t * loss_scale, target) - old_grads = super(LossScaleGradientTape, self).gradient( +def _compute_gradients_until_finite( + distribution, loss_scale_gradient_tapes, loss_scale, target, sources, + output_gradients, unconnected_gradients): + """Compute gradients and update the loss scale until the gradients are finite. + + This must be called in a cross-replica context. + + This is a function instead of a method of LossScaleGradientTape, as the `self` + parameter would be meaningless. There is one LossScaleGradientTape per + replica, but this function is called once total (not per replica), so there + cannot be a singular `self` parameter. + + Args: + distribution: The distribution strategy in effect. + loss_scale_gradient_tapes: A PerReplica value of LossScaleGradientTapes. + Contains the LossScaleGradientTape of each replica. + loss_scale: The loss scale to use to scale the loss and unscale the + gradient. + target: a list or nested structure of Tensors or Variables to be + differentiated. + sources: a list or nested structure of Tensors or Variables. `target` will + be differentiated against elements in `sources`. + output_gradients: Passed to GradientTape.gradient + unconnected_gradients: Pass to GradientTape.gradient. + + Returns: + The gradients of `target` with respect to `sources`. + """ + # Autograph cannot convert this function, so we must use an explicit + # tf.while_loop. + # TODO(b/143572314): Fix Autograph so that it can convert this function, then + # replace the tf.while_loop with a Python while loop. + + def cond(grads, ready_to_update): + """The condition of the while loop.""" + del grads + # Equivalent to: `not ready_to_update and loss_scale() > 1` + return math_ops.logical_and(math_ops.logical_not(ready_to_update), + math_ops.greater(loss_scale(), 1)) + + def body(grads, ready_to_update): + """The body of the while loop.""" + del grads, ready_to_update + def replica_fn(gradient_tape, target, sources, output_gradients): + """Scales the loss, computes the gradients, and unscales the gradients.""" + loss_scale_val = loss_scale() + with gradient_tape: # re-enter gradient tape so it sees the loss scaling + scaled_target = nest.map_structure(lambda t: t * loss_scale_val, target) + old_grads = super(LossScaleGradientTape, gradient_tape).gradient( scaled_target, sources, output_gradients, unconnected_gradients) - inv_loss_scale = 1.0 / self._loss_scale() + inv_loss_scale = 1.0 / loss_scale_val grads = nest.map_structure(lambda g: inv_loss_scale * g, old_grads) - # Check for non-finite gradients possibly resulting from scaling - _, ready_to_update = self._loss_scale.update(grads) - - if not self._outer_persistent: - self._tape = None # free up resources if a persistent tape was not needed - return grads + return grads + + # Switch to a replica-context to compute gradients once per replica. + grads = distribution.experimental_run_v2( + replica_fn, args=(loss_scale_gradient_tapes, target, sources, + output_gradients)) + # Check for non-finite gradients possibly resulting from scaling + _, ready_to_update = loss_scale.update(grads) + return grads, ready_to_update + + # Dummy value for initial_grads. The first iteration of the loop will + # overwrite `grads` to the actual gradients. + initial_grads = sources + initial_ready_to_update = False + grads, _ = control_flow_ops.while_loop( + cond, body, [initial_grads, initial_ready_to_update]) + return grads diff --git a/tensorflow/python/training/experimental/loss_scaling_gradient_tape_test.py b/tensorflow/python/training/experimental/loss_scaling_gradient_tape_test.py index b8c85a929da820..36d7d18a93b8d3 100644 --- a/tensorflow/python/training/experimental/loss_scaling_gradient_tape_test.py +++ b/tensorflow/python/training/experimental/loss_scaling_gradient_tape_test.py @@ -20,58 +20,137 @@ from absl.testing import parameterized import numpy as np from tensorflow.python.compat import v2_compat +from tensorflow.python.distribute import distribution_strategy_context +from tensorflow.python.distribute import mirrored_strategy +from tensorflow.python.distribute import values +from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_combinations from tensorflow.python.platform import test from tensorflow.python.training.experimental import loss_scale as loss_scale_module from tensorflow.python.training.experimental import loss_scaling_gradient_tape as lsgt +from tensorflow.python.util import nest + + +# If called outside any strategy.scope() calls, this will return the default +# strategy. +default_strategy_fn = distribution_strategy_context.get_strategy + + +def create_mirrored_strategy(): + if context.num_gpus() >= 1: + return mirrored_strategy.MirroredStrategy(['cpu:0', 'gpu:0']) + else: + return mirrored_strategy.MirroredStrategy(['cpu:0']) class LossScaleGradientTapeTest(test.TestCase, parameterized.TestCase): - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) - def test_basic_tapes_eager_mode(self, loss_scale): - x = constant_op.constant(3.0) - with lsgt.LossScaleGradientTape(loss_scale(32)) as g: - g.watch(x) - y = x * x - dy_dx = g.gradient(y, x) - self.assertEqual(self.evaluate(dy_dx), 6.0) + def _run_with_strategy(self, run_fn, strategy, use_tf_function=False): + """Runs `run_fn` under the DistributionStrategy `strategy`. - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) - def test_basic_tapes_graph_mode(self, loss_scale): - loss_scale = loss_scale(32) + Runs `run_fn` with `strategy.experimental_run_v2`. Returns a list of the + return values of `run_fn`, one per replica. - @def_function.function - def _inner_test(): + Args: + run_fn: The function to run. + strategy: The DistributionStrategy to run `run_fn` with. + use_tf_function: If True, call `run_fn` under a tf.function. + + Returns: + A list of tensors, each being the return value of `run_fn` from one + replica. If a nested structure is returned from `run_fn`, returns a + nested structure, where each element is a list of tensors. + """ + strategy_fn = lambda: strategy.experimental_run_v2(run_fn) + if use_tf_function: + strategy_fn = def_function.function(strategy_fn) + + results = strategy_fn() + + def convert_tensor_to_list(tensor): + if isinstance(tensor, values.DistributedValues): + return tensor.values + else: + return [tensor] + return nest.map_structure(convert_tensor_to_list, results) + + @test_combinations.generate(test_combinations.combine( + loss_scale=[loss_scale_module.FixedLossScale, + loss_scale_module.DynamicLossScale], + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + use_tf_function=[True, False] + )) + def test_basic_tapes(self, loss_scale, strategy_fn, use_tf_function): + loss_scale = loss_scale(32) + def run_fn(): x = constant_op.constant(3.0) with lsgt.LossScaleGradientTape(loss_scale) as g: g.watch(x) y = x * x return g.gradient(y, x) - self.assertEqual(self.evaluate(_inner_test()), 6.0) + dy_dx_list = self._run_with_strategy(run_fn, strategy_fn(), use_tf_function) + self.assertEqual(loss_scale(), 32) + for dy_dx in dy_dx_list: + self.assertEqual(dy_dx, 6.0) - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) - def test_nested_tapes(self, loss_scale): - x = constant_op.constant(3.0) - with lsgt.LossScaleGradientTape(loss_scale(32)) as g: - g.watch(x) - with lsgt.LossScaleGradientTape(loss_scale(32)) as gg: - gg.watch(x) + @test_combinations.generate(test_combinations.combine( + loss_scale=[loss_scale_module.FixedLossScale, + loss_scale_module.DynamicLossScale], + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + use_tf_function=[True, False] + )) + def test_output_gradients(self, loss_scale, strategy_fn, use_tf_function): + loss_scale = loss_scale(32) + def run_fn(): + x = constant_op.constant(3.0) + with lsgt.LossScaleGradientTape(loss_scale) as g: + g.watch(x) y = x * x - dy_dx = gg.gradient(y, x) - self.assertEqual(self.evaluate(dy_dx), 6.0) - d2y_dx2 = g.gradient(dy_dx, x) - self.assertEqual(self.evaluate(d2y_dx2), 2.0) - - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) - def test_non_persistent_tapes_error(self, loss_scale): + return g.gradient(y, x, output_gradients=constant_op.constant(2.0)) + dy_dx_list = self._run_with_strategy(run_fn, strategy_fn(), use_tf_function) + self.assertEqual(loss_scale(), 32) + for dy_dx in dy_dx_list: + self.assertEqual(dy_dx, 12.0) + + @test_combinations.generate(test_combinations.combine( + loss_scale=[loss_scale_module.FixedLossScale, + loss_scale_module.DynamicLossScale], + strategy_fn=[default_strategy_fn], + use_tf_function=[True, False] + )) + def test_nested_tapes(self, loss_scale, strategy_fn, use_tf_function): + # TODO(reedwm): Support nested tapes with mirrored strategy. Currently this + # does not work, as the set of active gradient tapes is a thread-local + # variable. Mirrored strategy spawns new threads, making the outer gradient + # tape non-active when using the inner gradient tape. + outer_loss_scale = loss_scale(32) + inner_loss_scale = loss_scale(32) + def run_fn(): + x = constant_op.constant(3.0) + with lsgt.LossScaleGradientTape(outer_loss_scale) as g: + g.watch(x) + with lsgt.LossScaleGradientTape(inner_loss_scale) as gg: + gg.watch(x) + y = x * x + dy_dx = gg.gradient(y, x) + d2y_dx2 = g.gradient(dy_dx, x) + return dy_dx, d2y_dx2 + + dy_dx_list, d2y_dx2_list = self._run_with_strategy(run_fn, strategy_fn(), + use_tf_function) + self.assertEqual(outer_loss_scale(), 32) + self.assertEqual(inner_loss_scale(), 32) + for dy_dx in dy_dx_list: + self.assertEqual(dy_dx, 6.0) + for d2y_dx2 in d2y_dx2_list: + self.assertEqual(d2y_dx2, 2.0) + + def test_non_persistent_tapes_error(self): x = constant_op.constant(3.0) - with lsgt.LossScaleGradientTape(loss_scale(32), persistent=False) as g: + with lsgt.LossScaleGradientTape(loss_scale_module.FixedLossScale(32), + persistent=False) as g: g.watch(x) y = x * x z = y * y @@ -79,21 +158,36 @@ def test_non_persistent_tapes_error(self, loss_scale): with self.assertRaisesRegexp(RuntimeError, 'persistent'): g.gradient(y, x) - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) - def test_persistent_tapes(self, loss_scale): - x = constant_op.constant(3.0) - with lsgt.LossScaleGradientTape(loss_scale(32), persistent=True) as g: - g.watch(x) - y = x * x - z = y * y - dz_dx = g.gradient(z, x) - self.assertEqual(self.evaluate(dz_dx), 108.0) - dy_dx = g.gradient(y, x) - self.assertEqual(self.evaluate(dy_dx), 6.0) + @test_combinations.generate(test_combinations.combine( + loss_scale=[loss_scale_module.FixedLossScale, + loss_scale_module.DynamicLossScale], + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + use_tf_function=[True, False] + )) + def test_persistent_tapes(self, loss_scale, strategy_fn, use_tf_function): - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) + ls = loss_scale(32) + def run_fn(): + x = constant_op.constant(3.0) + with lsgt.LossScaleGradientTape(ls, persistent=True) as g: + g.watch(x) + y = x * x + z = y * y + dz_dx = g.gradient(z, x) + dy_dx = g.gradient(y, x) + return dz_dx, dy_dx + + dz_dx_list, dy_dx_list = self._run_with_strategy(run_fn, strategy_fn(), + use_tf_function) + for dz_dx in dz_dx_list: + self.assertEqual(dz_dx, 108.0) + for dy_dx in dy_dx_list: + self.assertEqual(dy_dx, 6.0) + + @test_combinations.generate(test_combinations.combine( + loss_scale=[loss_scale_module.FixedLossScale, + loss_scale_module.DynamicLossScale], + )) def test_nested_sources(self, loss_scale): x = (constant_op.constant(19.0), (constant_op.constant(8.), constant_op.constant(9.))) @@ -103,8 +197,10 @@ def test_nested_sources(self, loss_scale): dy_dx = g.gradient(y, x) self.assertEqual(self.evaluate(dy_dx), (13., (13., 13.))) - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) + @test_combinations.generate(test_combinations.combine( + loss_scale=[loss_scale_module.FixedLossScale, + loss_scale_module.DynamicLossScale], + )) def test_nested_targets(self, loss_scale): w = constant_op.constant(3.0) with lsgt.LossScaleGradientTape(loss_scale(32)) as g: @@ -115,67 +211,130 @@ def test_nested_targets(self, loss_scale): grad = g.gradient([x, (y, z)], w) self.assertEqual(self.evaluate(grad), 23) - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) - def test_scaling_inf_gradient(self, loss_scale): - x = constant_op.constant(1.0) - with lsgt.LossScaleGradientTape(loss_scale(32)) as g: - g.watch(x) - y = x * np.inf - dy_dx = g.gradient(y, x) - self.assertEqual(self.evaluate(dy_dx), np.inf) + @test_combinations.generate(test_combinations.combine( + loss_scale=[loss_scale_module.FixedLossScale, + loss_scale_module.DynamicLossScale], + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + non_finite_term=[np.inf, np.nan], + )) + def test_scaling_non_finite_gradient(self, loss_scale, strategy_fn, + non_finite_term): + loss_scale = loss_scale(32) + def run_fn(): + x = constant_op.constant(1.0) + with lsgt.LossScaleGradientTape(loss_scale) as g: + g.watch(x) + y = x * non_finite_term + return g.gradient(y, x) - @parameterized.parameters(loss_scale_module.FixedLossScale, - loss_scale_module.DynamicLossScale) - def test_scaling_nan_gradient(self, loss_scale): - x = constant_op.constant(1.0) - with lsgt.LossScaleGradientTape(loss_scale(32)) as g: - g.watch(x) - y = x * np.nan - dy_dx = g.gradient(y, x) - self.assertTrue(np.isnan(self.evaluate(dy_dx))) + dy_dx_list = self._run_with_strategy(run_fn, strategy_fn()) + check_fn = np.isposinf if non_finite_term == np.inf else np.isnan + for dy_dx in dy_dx_list: + self.assertTrue(check_fn(dy_dx)) - @parameterized.parameters(np.inf, np.nan) - def test_dynamic_scale_to_one_on_non_finite_gradient(self, non_finite_term): + @test_combinations.generate(test_combinations.combine( + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + non_finite_term=[np.inf, np.nan], + use_tf_function=[True, False], + )) + def test_dynamic_scale_to_one_on_non_finite_gradient( + self, strategy_fn, non_finite_term, use_tf_function): loss_scale = loss_scale_module.DynamicLossScale(initial_loss_scale=32) - x = constant_op.constant(1.0) - with lsgt.LossScaleGradientTape(loss_scale) as g: - g.watch(x) - y = x * non_finite_term - g.gradient(y, x) + def run_fn(): + x = constant_op.constant(1.0) + with lsgt.LossScaleGradientTape(loss_scale) as g: + g.watch(x) + y = x * non_finite_term + g.gradient(y, x) + + self._run_with_strategy(run_fn, strategy_fn(), use_tf_function) + self.assertEqual(self.evaluate(loss_scale()), 1.0) + + @test_combinations.generate(test_combinations.combine( + use_tf_function=[True, False], + )) + def test_dynamic_scale_to_one_on_non_finite_gradient_on_last_replica( + self, use_tf_function): + if context.num_gpus() < 1: + # Requires the mirrored strategy to have two replicas: one on the CPU and + # one on the GPU + self.skipTest('Test requires at least 1 GPU') + loss_scale = loss_scale_module.DynamicLossScale(initial_loss_scale=32) + def run_fn(): + x = constant_op.constant(1.0) + with lsgt.LossScaleGradientTape(loss_scale) as g: + g.watch(x) + # The gradient will be finite on the first replica, and infinite on the + # second + rep_ctx = distribution_strategy_context.get_replica_context() + if rep_ctx.replica_id_in_sync_group == rep_ctx.num_replicas_in_sync - 1: + y = x * np.inf + else: + y = x * 2 + return g.gradient(y, x) + + replica0_grad, replica1_grad = self._run_with_strategy( + run_fn, create_mirrored_strategy(), use_tf_function) self.assertEqual(self.evaluate(loss_scale()), 1.0) + self.assertEqual(replica0_grad, 2.0) + self.assertEqual(replica1_grad, np.inf) - @parameterized.parameters([np.inf, np.isposinf], [np.nan, np.isnan]) - def test_fixed_scaling_no_change_non_finite_gradient(self, non_finite_term, - is_non_finite): + @test_combinations.generate(test_combinations.combine( + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + non_finite_term=[np.inf, np.nan], + )) + def test_fixed_scaling_no_change_non_finite_gradient(self, strategy_fn, + non_finite_term): loss_scale = loss_scale_module.FixedLossScale(32) - x = constant_op.constant(1.0) - with lsgt.LossScaleGradientTape(loss_scale) as g: - g.watch(x) - y = x * non_finite_term - dy_dx = g.gradient(y, x) - self.assertTrue(is_non_finite(self.evaluate(dy_dx))) + def run_fn(): + x = constant_op.constant(1.0) + with lsgt.LossScaleGradientTape(loss_scale) as g: + g.watch(x) + y = x * non_finite_term + return g.gradient(y, x) + + dy_dx_list = self._run_with_strategy(run_fn, strategy_fn()) + check_fn = np.isposinf if non_finite_term == np.inf else np.isnan + for dy_dx in dy_dx_list: + self.assertTrue(check_fn(self.evaluate(dy_dx))) self.assertEqual(self.evaluate(loss_scale()), 32.0) - def test_dynamic_loss_scaling_down_loop(self): + @test_combinations.generate(test_combinations.combine( + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + use_tf_function=[True, False] + )) + def test_dynamic_loss_scaling_down_loop(self, strategy_fn, use_tf_function): loss_scale = loss_scale_module.DynamicLossScale(initial_loss_scale=32) - x = constant_op.constant(1.0) - with lsgt.LossScaleGradientTape(loss_scale) as g: - g.watch(x) - y = x * (3.0 * (10**37)) # grad will be inf after scaling - dy_dx = g.gradient(y, x) + def run_fn(): + x = constant_op.constant(1.0) + with lsgt.LossScaleGradientTape(loss_scale) as g: + g.watch(x) + y = x * (3.0 * (10**37)) # grad will be inf after scaling + return g.gradient(y, x) + + dy_dx_list = self._run_with_strategy(run_fn, strategy_fn(), use_tf_function) self.assertEqual(self.evaluate(loss_scale()), 8.0) - self.assertAllClose(self.evaluate(dy_dx), (3.0 * (10**37)), atol=1e-06) + for dy_dx in dy_dx_list: + self.assertAllClose(self.evaluate(dy_dx), (3.0 * (10**37)), atol=1e-06) - def test_dynamic_loss_scaling_inf_target_post_scale(self): + @test_combinations.generate(test_combinations.combine( + strategy_fn=[default_strategy_fn, create_mirrored_strategy], + use_tf_function=[True, False] + )) + def test_dynamic_loss_scaling_inf_target_post_scale(self, strategy_fn, + use_tf_function): loss_scale = loss_scale_module.DynamicLossScale(initial_loss_scale=32.0) - x = constant_op.constant(3.0 * (10**37)) - with lsgt.LossScaleGradientTape(loss_scale) as g: - g.watch(x) - y = x * 3.0 # target will be inf after scaling - dy_dx = g.gradient(y, x) - self.assertAllClose(self.evaluate(dy_dx), 3.0) + def run_fn(): + x = constant_op.constant(3.0 * (10**37)) + with lsgt.LossScaleGradientTape(loss_scale) as g: + g.watch(x) + y = x * 3.0 # target will be inf after scaling + return g.gradient(y, x) + + dy_dx_list = self._run_with_strategy(run_fn, strategy_fn(), use_tf_function) self.assertEqual(self.evaluate(loss_scale()), 32.0) + for dy_dx in dy_dx_list: + self.assertAllClose(self.evaluate(dy_dx), 3.0) if __name__ == '__main__': diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 5971d41525fb99..740f24ec4a4afe 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -54,7 +54,7 @@ def register_extension_info(**kwargs): # not contain rc or alpha, only numbers. # Also update tensorflow/core/public/version.h # and tensorflow/tools/pip_package/setup.py -VERSION = "2.0.0" +VERSION = "2.1.0" VERSION_MAJOR = VERSION.split(".")[0] def if_v2(a): @@ -263,6 +263,8 @@ def get_win_copts(is_external = False): # "/EHs-c-", "/wd4577", "/DNOGDI", + # Also see build:windows lines in tensorflow/opensource_only/.bazelrc + # where we set some other options globally. ] if is_external: return WINDOWS_COPTS + ["/UTF_COMPILE_LIBRARY"] @@ -626,6 +628,11 @@ def tf_cc_binary( [ clean_dep("//third_party/mkl:intel_binary_blob"), ], + ) + if_static( + extra_deps = [], + otherwise = [ + clean_dep("//tensorflow:libtensorflow_framework_import_lib"), + ], ), data = depset(data + added_data_deps), linkopts = linkopts + _rpath_linkopts(name_os), @@ -2331,6 +2338,7 @@ def tf_generate_proto_text_sources(name, srcs_relative_dir, srcs, protodeps = [] hdrs = out_hdrs, visibility = visibility, deps = deps, + alwayslink = 1, ) def tf_genrule_cmd_append_to_srcs(to_append): @@ -2362,7 +2370,7 @@ def tf_py_build_info_genrule(name, out, **kwargs): " --is_config_rocm " + if_rocm("True", "False") + " --key_value " + if_cuda(" cuda_version_number=$${TF_CUDA_VERSION:-} cudnn_version_number=$${TF_CUDNN_VERSION:-} ", "") + - if_windows(" msvcp_dll_name=msvcp140.dll ", "") + + if_windows(" msvcp_dll_names=msvcp140.dll,msvcp140_1.dll ", "") + if_windows_cuda(" ".join([ "nvcuda_dll_name=nvcuda.dll", "cudart_dll_name=cudart64_$$(echo $${TF_CUDA_VERSION:-} | sed \"s/\\.//\").dll", @@ -2445,7 +2453,14 @@ def pybind_extension( name = so_file, srcs = srcs + hdrs, data = data, - copts = copts + ["-fexceptions"], + copts = copts + [ + "-fexceptions", + ] + select({ + clean_dep("//tensorflow:windows"): [], + "//conditions:default": [ + "-fvisibility=hidden", + ], + }), linkopts = linkopts + _rpath_linkopts(name) + select({ "@local_config_cuda//cuda:darwin": [ "-Wl,-exported_symbols_list,$(location %s)" % exported_symbols_file, diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt new file mode 100644 index 00000000000000..7f6d81d297a09e --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt @@ -0,0 +1,7 @@ +path: "tensorflow.keras.layers.experimental" +tf_module { + member { + name: "preprocessing" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-normalization.__metaclass__.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-normalization.__metaclass__.pbtxt new file mode 100644 index 00000000000000..20bb9904d18d49 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-normalization.__metaclass__.pbtxt @@ -0,0 +1,14 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.Normalization.__metaclass__" +tf_class { + is_instance: "" + member_method { + name: "__init__" + } + member_method { + name: "mro" + } + member_method { + name: "register" + argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-normalization.pbtxt new file mode 100644 index 00000000000000..59f759886c9015 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-normalization.pbtxt @@ -0,0 +1,225 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.Normalization" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'axis\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'None\'], " + } + member_method { + name: "adapt" + argspec: "args=[\'self\', \'data\', \'reset_state\'], varargs=None, keywords=None, defaults=[\'True\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_spec\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.__metaclass__.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.__metaclass__.pbtxt new file mode 100644 index 00000000000000..ceebb69d16a6b0 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.__metaclass__.pbtxt @@ -0,0 +1,14 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.PreprocessingLayer.__metaclass__" +tf_class { + is_instance: "" + member_method { + name: "__init__" + } + member_method { + name: "mro" + } + member_method { + name: "register" + argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.pbtxt new file mode 100644 index 00000000000000..cf939df37dced8 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.pbtxt @@ -0,0 +1,221 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.PreprocessingLayer" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'trainable\', \'name\', \'dtype\', \'dynamic\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\', \'None\', \'False\'], " + } + member_method { + name: "adapt" + argspec: "args=[\'self\', \'data\', \'reset_state\'], varargs=None, keywords=None, defaults=[\'True\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_signature\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.__metaclass__.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.__metaclass__.pbtxt new file mode 100644 index 00000000000000..fe45a5da03bddc --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.__metaclass__.pbtxt @@ -0,0 +1,14 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.TextVectorization.__metaclass__" +tf_class { + is_instance: "" + member_method { + name: "__init__" + } + member_method { + name: "mro" + } + member_method { + name: "register" + argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.pbtxt new file mode 100644 index 00000000000000..d79b7d712f1266 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.pbtxt @@ -0,0 +1,233 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.TextVectorization" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'max_tokens\', \'standardize\', \'split\', \'ngrams\', \'output_mode\', \'output_sequence_length\', \'pad_to_max_tokens\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'lower_and_strip_punctuation\', \'whitespace\', \'None\', \'int\', \'None\', \'True\'], " + } + member_method { + name: "adapt" + argspec: "args=[\'self\', \'data\', \'reset_state\'], varargs=None, keywords=None, defaults=[\'True\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_spec\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_vocabulary" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_vocabulary" + argspec: "args=[\'self\', \'vocab\', \'df_data\', \'oov_df_value\', \'append\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], " + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.pbtxt new file mode 100644 index 00000000000000..abfd2c682de42c --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.preprocessing.pbtxt @@ -0,0 +1,15 @@ +path: "tensorflow.keras.layers.experimental.preprocessing" +tf_module { + member { + name: "Normalization" + mtype: "" + } + member { + name: "PreprocessingLayer" + mtype: "" + } + member { + name: "TextVectorization" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt index 603803595b6bc8..847cc814e0ff03 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt @@ -416,6 +416,10 @@ tf_module { name: "ZeroPadding3D" mtype: "" } + member { + name: "experimental" + mtype: "" + } member_method { name: "Input" argspec: "args=[\'shape\', \'batch_size\', \'name\', \'dtype\', \'sparse\', \'tensor\', \'ragged\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'False\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt index bf7812a668d830..c904681f633ba1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt @@ -140,6 +140,10 @@ tf_module { name: "erfc" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "erfinv" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "exp" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -280,6 +284,10 @@ tf_module { name: "multiply_no_nan" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "ndtri" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "negative" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index 6c75ecb5fbf4f0..5b9747c1cef0f4 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -1236,10 +1236,6 @@ tf_module { name: "erfc" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "erfinv" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "executing_eagerly" argspec: "args=[], varargs=None, keywords=None, defaults=None" @@ -1716,10 +1712,6 @@ tf_module { name: "multiply" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "ndtri" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "negative" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt new file mode 100644 index 00000000000000..7f6d81d297a09e --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt @@ -0,0 +1,7 @@ +path: "tensorflow.keras.layers.experimental" +tf_module { + member { + name: "preprocessing" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-normalization.__metaclass__.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-normalization.__metaclass__.pbtxt new file mode 100644 index 00000000000000..20bb9904d18d49 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-normalization.__metaclass__.pbtxt @@ -0,0 +1,14 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.Normalization.__metaclass__" +tf_class { + is_instance: "" + member_method { + name: "__init__" + } + member_method { + name: "mro" + } + member_method { + name: "register" + argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-normalization.pbtxt new file mode 100644 index 00000000000000..0efa1a8f5af253 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-normalization.pbtxt @@ -0,0 +1,223 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.Normalization" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'axis\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'None\'], " + } + member_method { + name: "adapt" + argspec: "args=[\'self\', \'data\', \'reset_state\'], varargs=None, keywords=None, defaults=[\'True\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_spec\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.__metaclass__.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.__metaclass__.pbtxt new file mode 100644 index 00000000000000..ceebb69d16a6b0 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.__metaclass__.pbtxt @@ -0,0 +1,14 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.PreprocessingLayer.__metaclass__" +tf_class { + is_instance: "" + member_method { + name: "__init__" + } + member_method { + name: "mro" + } + member_method { + name: "register" + argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.pbtxt new file mode 100644 index 00000000000000..cf939df37dced8 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-preprocessing-layer.pbtxt @@ -0,0 +1,221 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.PreprocessingLayer" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'trainable\', \'name\', \'dtype\', \'dynamic\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\', \'None\', \'False\'], " + } + member_method { + name: "adapt" + argspec: "args=[\'self\', \'data\', \'reset_state\'], varargs=None, keywords=None, defaults=[\'True\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_signature\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.__metaclass__.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.__metaclass__.pbtxt new file mode 100644 index 00000000000000..fe45a5da03bddc --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.__metaclass__.pbtxt @@ -0,0 +1,14 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.TextVectorization.__metaclass__" +tf_class { + is_instance: "" + member_method { + name: "__init__" + } + member_method { + name: "mro" + } + member_method { + name: "register" + argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.pbtxt new file mode 100644 index 00000000000000..85fe8aec94bf8c --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.-text-vectorization.pbtxt @@ -0,0 +1,231 @@ +path: "tensorflow.keras.layers.experimental.preprocessing.TextVectorization" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'max_tokens\', \'standardize\', \'split\', \'ngrams\', \'output_mode\', \'output_sequence_length\', \'pad_to_max_tokens\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'lower_and_strip_punctuation\', \'whitespace\', \'None\', \'int\', \'None\', \'True\'], " + } + member_method { + name: "adapt" + argspec: "args=[\'self\', \'data\', \'reset_state\'], varargs=None, keywords=None, defaults=[\'True\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_spec\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_vocabulary" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_vocabulary" + argspec: "args=[\'self\', \'vocab\', \'df_data\', \'oov_df_value\', \'append\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], " + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.pbtxt new file mode 100644 index 00000000000000..abfd2c682de42c --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.preprocessing.pbtxt @@ -0,0 +1,15 @@ +path: "tensorflow.keras.layers.experimental.preprocessing" +tf_module { + member { + name: "Normalization" + mtype: "" + } + member { + name: "PreprocessingLayer" + mtype: "" + } + member { + name: "TextVectorization" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt index 9f1b0dc41fdfde..5574cc9ca5918a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt @@ -408,6 +408,10 @@ tf_module { name: "ZeroPadding3D" mtype: "" } + member { + name: "experimental" + mtype: "" + } member_method { name: "Input" argspec: "args=[\'shape\', \'batch_size\', \'name\', \'dtype\', \'sparse\', \'tensor\', \'ragged\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'False\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt index 82688f51640b38..2ec2ab27476270 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt @@ -140,6 +140,10 @@ tf_module { name: "erfc" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "erfinv" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "exp" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -280,6 +284,10 @@ tf_module { name: "multiply_no_nan" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "ndtri" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "negative" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.mixed_precision.experimental.-loss-scale-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.mixed_precision.experimental.-loss-scale-gradient-tape.pbtxt deleted file mode 100644 index 7f4715832e20ca..00000000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.mixed_precision.experimental.-loss-scale-gradient-tape.pbtxt +++ /dev/null @@ -1,38 +0,0 @@ -path: "tensorflow.mixed_precision.experimental.LossScaleGradientTape" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - argspec: "args=[\'self\', \'loss_scale\', \'persistent\', \'watch_accessed_variables\'], varargs=None, keywords=None, defaults=[\'False\', \'True\'], " - } - member_method { - name: "batch_jacobian" - argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], " - } - member_method { - name: "gradient" - argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'UnconnectedGradients.NONE\'], " - } - member_method { - name: "jacobian" - argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], " - } - member_method { - name: "reset" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "stop_recording" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "watch" - argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "watched_variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.mixed_precision.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.mixed_precision.experimental.pbtxt index 5abfdcd109d210..61700226fbbfa5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.mixed_precision.experimental.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.mixed_precision.experimental.pbtxt @@ -12,8 +12,4 @@ tf_module { name: "LossScale" mtype: "" } - member { - name: "LossScaleGradientTape" - mtype: "" - } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt index d67870a92b8c17..7cf14d69e49f7a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt @@ -624,10 +624,6 @@ tf_module { name: "equal" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "erfinv" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "executing_eagerly" argspec: "args=[], varargs=None, keywords=None, defaults=None" @@ -812,10 +808,6 @@ tf_module { name: "multiply" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "ndtri" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "negative" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/ci_build/Dockerfile.android b/tensorflow/tools/ci_build/Dockerfile.android index 81e9077cd0ace5..80949ac64ebcdf 100644 --- a/tensorflow/tools/ci_build/Dockerfile.android +++ b/tensorflow/tools/ci_build/Dockerfile.android @@ -29,7 +29,7 @@ RUN mkdir -p ${ANDROID_DEV_HOME} ENV ANDROID_SDK_FILENAME tools_r25.2.5-linux.zip ENV ANDROID_SDK_URL https://dl.google.com/android/repository/${ANDROID_SDK_FILENAME} ENV ANDROID_API_LEVEL 23 -ENV ANDROID_NDK_API_LEVEL 18 +ENV ANDROID_NDK_API_LEVEL 21 # Build Tools Version liable to change. ENV ANDROID_BUILD_TOOLS_VERSION 28.0.0 ENV ANDROID_SDK_HOME ${ANDROID_DEV_HOME}/sdk diff --git a/tensorflow/tools/ci_build/Dockerfile.rocm b/tensorflow/tools/ci_build/Dockerfile.rocm index 191947da7a74d7..a083bc6debd9e6 100644 --- a/tensorflow/tools/ci_build/Dockerfile.rocm +++ b/tensorflow/tools/ci_build/Dockerfile.rocm @@ -3,7 +3,7 @@ FROM ubuntu:xenial MAINTAINER Jeff Poznanovic -ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/2.6/ +ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/2.8.0/ ARG ROCM_PATH=/opt/rocm ENV DEBIAN_FRONTEND noninteractive diff --git a/tensorflow/tools/ci_build/builds/builds_common.sh b/tensorflow/tools/ci_build/builds/builds_common.sh index 55a4ac800f39b3..8b0c065a9e3702 100644 --- a/tensorflow/tools/ci_build/builds/builds_common.sh +++ b/tensorflow/tools/ci_build/builds/builds_common.sh @@ -235,7 +235,7 @@ android_sdk_repository( android_ndk_repository( name="androidndk", path="${ANDROID_NDK_HOME}", - api_level=18) + api_level=21) EOF fi fi diff --git a/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh b/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh index c87ec292471064..3bb8d8b7afa30e 100755 --- a/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh +++ b/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh @@ -22,6 +22,18 @@ pip --version pip install portpicker pip install *.whl +# Make bazel version the same as the env that invokes this script +rm -rf ~/bazel +mkdir ~/bazel +pushd ~/bazel +wget https://github.com/bazelbuild/bazel/releases/download/"${BAZEL_VERSION}"/bazel-"${BAZEL_VERSION}"-installer-linux-x86_64.sh +chmod +x bazel-*.sh +./bazel-"${BAZEL_VERSION}"-installer-linux-x86_64.sh --user +rm bazel-"${BAZEL_VERSION}"-installer-linux-x86_64.sh +PATH="/bazel_pip/bin:$PATH" +popd +bazel version + # Use default configuration yes "" | python configure.py diff --git a/tensorflow/tools/ci_build/builds/docker_test.sh b/tensorflow/tools/ci_build/builds/docker_test.sh index 38891b60e57676..39e119f889537e 100755 --- a/tensorflow/tools/ci_build/builds/docker_test.sh +++ b/tensorflow/tools/ci_build/builds/docker_test.sh @@ -109,7 +109,8 @@ if [ "${IMAGE_TYPE}" == "gpu" ]; then libs=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}') GPU_EXTRA_PARAMS="${devices} ${libs}" elif [ "${IMAGE_TYPE}" == "rocm" ]; then - ROCM_EXTRA_PARAMS="--device=/dev/kfd --device=/dev/dri --group-add video" + ROCM_EXTRA_PARAMS="--device=/dev/kfd --device=/dev/dri --group-add video \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size 16G" else GPU_EXTRA_PARAMS="" ROCM_EXTRA_PARAMS="" diff --git a/tensorflow/tools/ci_build/builds/pip_new.sh b/tensorflow/tools/ci_build/builds/pip_new.sh index 932d3e8f60c55e..2559dacd91551a 100755 --- a/tensorflow/tools/ci_build/builds/pip_new.sh +++ b/tensorflow/tools/ci_build/builds/pip_new.sh @@ -60,7 +60,13 @@ # and tensorflow-gpu pip package. Will # automatically handle adding/removing of _gpu # suffix depending on what project name was -# passed. +# passed. Only work for Ubuntu. +# TF_BUILD_BOTH_CPU_PACKAGES: (1 | 0) +# 1 will build both tensorflow (no gpu support) +# and tensorflow-cpu pip package. Will +# automatically handle adding/removing of _cpu +# suffix depending on what project name was +# passed. Only work for MacOS # # To-be-deprecated variable(s). # GIT_TAG_OVERRIDE: Values for `--git_tag_override`. This flag gets passed @@ -241,11 +247,13 @@ DEFAULT_PIP_TESTS="" # Do not run any tests by default DEFAULT_PROJECT_NAME="tensorflow" DEFAULT_PIP_TEST_ROOT="pip_test" DEFAULT_BUILD_BOTH_GPU_PACKAGES=0 +DEFAULT_BUILD_BOTH_CPU_PACKAGES=0 # Take in optional global variables PIP_TESTS=${TF_PIP_TESTS:-$DEFAULT_PIP_TESTS} PROJECT_NAME=${TF_PROJECT_NAME:-$DEFAULT_PROJECT_NAME} PIP_TEST_ROOT=${TF_PIP_TEST_ROOT:-$DEFAULT_PIP_TEST_ROOT} BUILD_BOTH_GPU_PACKAGES=${TF_BUILD_BOTH_GPU_PACKAGES:-$DEFAULT_BUILD_BOTH_GPU_PACKAGES} +BUILD_BOTH_CPU_PACKAGES=${TF_BUILD_BOTH_CPU_PACKAGES:-$DEFAULT_BUILD_BOTH_CPU_PACKAGES} # Local variables PIP_WHL_DIR="${KOKORO_ARTIFACTS_DIR}/tensorflow/${PIP_TEST_ROOT}/whl" @@ -640,20 +648,38 @@ WHL_DIR=$(dirname "${WHL_PATH}") echo "Size of the PIP wheel file built: $(ls -l ${WHL_PATH} | awk '{print $5}')" # Build the other GPU package. -if [ "$BUILD_BOTH_GPU_PACKAGES" -eq "1" ]; then - echo "====================================="\ - "Building the other GPU pip package." +if [[ "$BUILD_BOTH_GPU_PACKAGES" -eq "1" ]] || [[ "$BUILD_BOTH_CPU_PACKAGES" -eq "1" ]]; then + + if [[ "$BUILD_BOTH_GPU_PACKAGES" -eq "1" ]] && [[ "$BUILD_BOTH_CPU_PACKAGES" -eq "1" ]]; then + die "ERROR: TF_BUILD_BOTH_GPU_PACKAGES and TF_BUILD_BOTH_GPU_PACKAGES cannot both be set. No additional package will be built." + fi + + echo "=====================================" + if [[ "$BUILD_BOTH_GPU_PACKAGES" -eq "1" ]]; then + if ! [[ ${OS_TYPE} == "ubuntu" ]]; then + die "ERROR: pip_new.sh only support building both GPU wheels on ubuntu." + fi + echo "Building the other GPU pip package." + PROJECT_SUFFIX="gpu" + else + if ! [[ ${OS_TYPE} == "macos" ]]; then + die "ERROR: pip_new.sh only support building both CPU wheels on macos." + fi + echo "Building the other CPU pip package." + PROJECT_SUFFIX="cpu" + fi + # Check container type - if ! [[ ${CONTAINER_TYPE} == "gpu" ]]; then - die "Error: CONTAINER_TYPE needs to be `GPU` to build GPU packages. Got "\ + if ! [[ ${CONTAINER_TYPE} == ${PROJECT_SUFFIX} ]]; then + die "Error: CONTAINER_TYPE needs to be \"${PROJECT_SUFFIX}\" to build ${PROJECT_SUFFIX} packages. Got"\ "\"${CONTAINER_TYPE}\" instead." fi - if [[ "$PROJECT_NAME" == *_gpu ]]; then - NEW_PROJECT_NAME=${PROJECT_NAME%"_gpu"} + if [[ "$PROJECT_NAME" == *_${PROJECT_SUFFIX} ]]; then + NEW_PROJECT_NAME=${PROJECT_NAME%"_${PROJECT_SUFFIX}"} else - NEW_PROJECT_NAME="${PROJECT_NAME}_gpu" + NEW_PROJECT_NAME="${PROJECT_NAME}_${PROJECT_SUFFIX}" fi - echo "The given gpu \$PROJECT_NAME is ${PROJECT_NAME}. The additional GPU "\ + echo "The given ${PROJECT_SUFFIX} \$PROJECT_NAME is ${PROJECT_NAME}. The additional ${PROJECT_SUFFIX}"\ "pip package will have project name ${NEW_PROJECT_NAME}." ./bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} ${GPU_FLAG} ${NIGHTLY_FLAG} "--project_name" ${NEW_PROJECT_NAME} || die "build_pip_package FAILED" diff --git a/tensorflow/tools/ci_build/ci_build.sh b/tensorflow/tools/ci_build/ci_build.sh index 079765bd5f9d22..d41972f4e1a9ba 100755 --- a/tensorflow/tools/ci_build/ci_build.sh +++ b/tensorflow/tools/ci_build/ci_build.sh @@ -111,7 +111,8 @@ fi # Add extra params for rocm devices and libraries for ROCm container. if [[ "${CONTAINER_TYPE}" == "rocm" ]]; then - ROCM_EXTRA_PARAMS="--device=/dev/kfd --device=/dev/dri --group-add video" + ROCM_EXTRA_PARAMS="--device=/dev/kfd --device=/dev/dri --group-add video \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size 16G" else ROCM_EXTRA_PARAMS="" fi diff --git a/tensorflow/tools/ci_build/install/install_centos_pip_packages.sh b/tensorflow/tools/ci_build/install/install_centos_pip_packages.sh index 7cd8d9f4418bd9..51c7a77079ac52 100755 --- a/tensorflow/tools/ci_build/install/install_centos_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_centos_pip_packages.sh @@ -57,8 +57,8 @@ pip3 install --upgrade protobuf==3.6.1 pip2 install --upgrade numpy==1.14.5 pip3 install --upgrade numpy==1.14.5 -pip2 install scipy==1.1.0 -pip3 install scipy==1.1.0 +pip2 install scipy==1.2.2 +pip3 install scipy==1.4.1 pip2 install scikit-learn==0.18.1 pip3 install scikit-learn==0.18.1 diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh index bb13c795284e81..170482b45657c7 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh @@ -76,8 +76,8 @@ else pip3 install --upgrade numpy==1.14.5 fi -pip2 install scipy==1.1.0 -pip3 install scipy==1.1.0 +pip2 install scipy==1.2.2 +pip3 install scipy==1.4.1 pip2 install scikit-learn==0.18.1 pip3 install scikit-learn==0.18.1 diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh index 135e8e81addceb..e68b3b24477531 100755 --- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh @@ -64,7 +64,7 @@ rm -rf /usr/lib/python3/dist-packages/six* # This workaround isn't needed for Ubuntu 16.04 or later. pip3.5 install --no-binary=:all: --upgrade numpy==1.14.5 -pip3.5 install scipy==0.18.1 +pip3.5 install scipy==1.4.1 pip3.5 install scikit-learn==0.19.1 diff --git a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh index af62d9efc78e54..e3eaa843412482 100755 --- a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh @@ -76,7 +76,7 @@ rm -rf /usr/lib/python3/dist-packages/six* # This workaround isn't needed for Ubuntu 16.04 or later. pip3 install --no-binary=:all: --upgrade numpy==1.14.5 -pip3 install scipy==0.18.1 +pip3 install scipy==1.4.1 pip3 install scikit-learn==0.19.1 diff --git a/tensorflow/tools/ci_build/presubmit/macos/py2_cc/build.sh b/tensorflow/tools/ci_build/presubmit/macos/py2_cc/build.sh new file mode 100644 index 00000000000000..92acb7ab7fe19b --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/macos/py2_cc/build.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# TODO(mihaimaruseac,hyey,ggadde): Convert to py3 + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function setup_pip () { + install_pip2 + python -m virtualenv tf_build_env --system-site-packages + source tf_build_env/bin/activate + install_macos_pip_deps +} + +function run_build () { + # Run configure. + export TF_NEED_CUDA=0 + export PYTHON_BIN_PATH=$(which python2) + yes "" | $PYTHON_BIN_PATH configure.py + tag_filters="-no_oss,-no_oss_py2,-gpu,-tpu,-benchmark-test,-nomac,-no_mac,-v1only" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + "${BAZEL_WRAPPER_PATH}" \ + test \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" \ + --action_env=PATH \ + --remote_accept_cached=true \ + --spawn_strategy=standalone \ + --remote_local_fallback=false \ + --remote_timeout=600 \ + --strategy=Javac=standalone \ + --strategy=Closure=standalone \ + --genrule_strategy=standalone \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_macos +which bazel +set_bazel_outdir + +setup_pip +run_build diff --git a/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh b/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh new file mode 100644 index 00000000000000..ffc823a6e2eb56 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function setup_pip () { + python3.7 -m virtualenv tf_build_env --system-site-packages + source tf_build_env/bin/activate + install_macos_pip_deps +} + +function run_build () { + # Run configure. + export TF_NEED_CUDA=0 + export PYTHON_BIN_PATH=$(which python3.7) + yes "" | $PYTHON_BIN_PATH configure.py + tag_filters="-no_oss,-no_oss_py2,-gpu,-tpu,-benchmark-test,-nomac,-no_mac,-v1only" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + "${BAZEL_WRAPPER_PATH}" \ + test \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" \ + --action_env=PATH \ + --remote_accept_cached=true \ + --spawn_strategy=standalone \ + --remote_local_fallback=false \ + --remote_timeout=600 \ + --strategy=Javac=standalone \ + --strategy=Closure=standalone \ + --genrule_strategy=standalone \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_macos +which bazel +set_bazel_outdir + +setup_pip +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/android/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/android/build.sh new file mode 100644 index 00000000000000..5fe3c41ae59a10 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/android/build.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function run_build () { + export ANDROID_NDK_HOME="/opt/android-ndk-r17c" + export NDK_HOME=$ANDROID_NDK_HOME + export ANDROID_SDK_HOME="/opt/android-sdk/current" + export ANDROID_API_LEVEL="23" + export ANDROID_BUILD_TOOLS_VERSION="28.0.0" + + ANDROID_OUT=android.out + ANDROID_OUT_TARGET=gen_android_out + + # Run the presubmit android build. + tensorflow/tools/ci_build/builds/android.sh 2>&1 | tee tensorflow/tools/ci_build/builds/${ANDROID_OUT} + RC=${PIPESTATUS[0]} + + # Since we are running the build remotely (rbe), we need to build a bazel + # target that would output the log generated above and return the expected + # error code. + cat << EOF > tensorflow/tools/ci_build/builds/BUILD +package(default_visibility = ["//tensorflow:internal"]) + +sh_test( + name = "${ANDROID_OUT_TARGET}", + srcs = ["${ANDROID_OUT_TARGET}.sh"], + data = ["${ANDROID_OUT}"], + tags = ["local"], +) +EOF + + cat << EOF > tensorflow/tools/ci_build/builds/${ANDROID_OUT_TARGET}.sh +#!/bin/bash +cat tensorflow/tools/ci_build/builds/${ANDROID_OUT} +exit ${RC} +EOF + + # Now trigger the rbe build that outputs the log + chmod +x tensorflow/tools/ci_build/builds/${ANDROID_OUT_TARGET}.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + "${BAZEL_WRAPPER_PATH}" \ + --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ + test \ + --test_output=all \ + tensorflow/tools/ci_build/builds:${ANDROID_OUT_TARGET} + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/cpu_py36_full/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/cpu_py36_full/build.sh new file mode 100644 index 00000000000000..d852ba3796f434 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/cpu_py36_full/build.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function run_build () { + # Build a unique cache silo string. + UBUNTU_VERSION=$(lsb_release -a | grep Release | awk '{print $2}') + IMAGE_VERSION=$(cat /VERSION) + CACHE_SILO_VAL="cpu-py3-ubuntu-16-${UBUNTU_VERSION}-${IMAGE_VERSION}" + + # Run configure. + # Do not run configure.py when doing remote build & test: + # Most things we set with configure.py are not used in a remote build setting, + # as the build will be defined by pre-configured build files that are checked + # in. + # TODO(klimek): Allow using the right set of bazel flags without the need to + # run configure.py; currently we need to carefully copy them, which is brittle. + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_CUDA=0 + export ACTION_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + export PYTHON_BIN_PATH="/usr/bin/python3" + export TF2_BEHAVIOR=1 + tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test""$(maybe_skip_v1)" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + "${BAZEL_WRAPPER_PATH}" \ + test \ + --config=rbe \ + --python_path="${PYTHON_BIN_PATH}" \ + --action_env=PATH="${ACTION_PATH}" \ + --action_env=PYTHON_BIN_PATH="${PYTHON_BIN_PATH}" \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --action_env=TF_PYTHON_CONFIG_REPO=@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/py3 \ + --action_env=TF_ENABLE_XLA=1 \ + --test_tag_filters="${tag_filters}" \ + --build_tag_filters="${tag_filters}" \ + --test_lang_filters=cc,py \ + --define=with_default_optimizations=true \ + --define=framework_shared_object=true \ + --define=with_xla_support=true \ + -c opt \ + --copt="-w" \ + --copt=-mavx \ + --linkopt=-lrt \ + --distinct_host_configuration=false \ + --remote_default_platform_properties="properties:{name:\"build\" value:\"${CACHE_SILO_VAL}\"}" \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010:toolchain \ + --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 \ + --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 \ + --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --extra_toolchains=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010:cc-toolchain-k8 \ + --extra_execution_platforms=@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010 \ + --host_platform=@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010 \ + --remote_timeout=3600 \ + --platforms=@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010 \ + -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/gpu_py36_full/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/gpu_py36_full/build.sh new file mode 100644 index 00000000000000..3fa4d4f1d7da45 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/gpu_py36_full/build.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function run_build () { + # Build a unique cache silo string. + UBUNTU_VERSION=$(lsb_release -a | grep Release | awk '{print $2}') + IMAGE_VERSION=$(cat /VERSION) + CACHE_SILO_VAL="gpu-py3-ubuntu-16-${UBUNTU_VERSION}-${IMAGE_VERSION}" + + # Run configure. + # Do not run configure.py when doing remote build & test: + # Most things we set with configure.py are not used in a remote build setting, + # as the build will be defined by pre-configured build files that are checked + # in. + # TODO(klimek): Allow using the right set of bazel flags without the need to + # run configure.py; currently we need to carefully copy them, which is brittle. + export LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64" + # TODO(klimek): Remove once we don't try to read it while setting up the remote + # config for cuda (we currently don't use it, as it's only used when compiling + # with clang, but we still require it to be set anyway). + export TF_CUDA_COMPUTE_CAPABILITIES=6.0 + export ACTION_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + export PYTHON_BIN_PATH="/usr/bin/python3" + export TF2_BEHAVIOR=1 + tag_filters="gpu,-no_gpu,-nogpu,-benchmark-test,-no_oss,-oss_serial""$(maybe_skip_v1)" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + # TODO(klimek): Re-enable tensorrt tests (with different runtime image) once + # we can build them. + # TODO(klimek): Stop using action_env for things that are only needed during + # setup - we're artificially poisoning the cache. + "${BAZEL_WRAPPER_PATH}" \ + test \ + --config=rbe \ + --python_path="${PYTHON_BIN_PATH}" \ + --action_env=PATH="${ACTION_PATH}" \ + --action_env=PYTHON_BIN_PATH="${PYTHON_BIN_PATH}" \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --action_env=REMOTE_GPU_TESTING=1 \ + --action_env=TF_CUDA_COMPUTE_CAPABILITIES="${TF_CUDA_COMPUTE_CAPABILITIES}" \ + --action_env=TF_CUDA_CONFIG_REPO=@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/cuda10.0-cudnn7 \ + --action_env=TF_CUDA_VERSION=10 \ + --action_env=TF_CUDNN_VERSION=7 \ + --action_env=TF_NEED_TENSORRT=0 \ + --action_env=TF_NEED_CUDA=1 \ + --action_env=TF_PYTHON_CONFIG_REPO=@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/py3 \ + --test_env=LD_LIBRARY_PATH \ + --test_tag_filters="${tag_filters}" \ + --build_tag_filters="${tag_filters}" \ + --test_lang_filters=cc,py \ + --define=with_default_optimizations=true \ + --define=framework_shared_object=true \ + --define=with_xla_support=true \ + --define=using_cuda_nvcc=true \ + --define=use_fast_cpp_protos=true \ + --define=allow_oversize_protos=true \ + --define=grpc_no_ares=true \ + -c opt \ + --copt="-w" \ + --copt=-mavx \ + --linkopt=-lrt \ + --distinct_host_configuration=false \ + --remote_default_platform_properties="properties:{name:\"build\" value:\"${CACHE_SILO_VAL}\"}" \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 \ + --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:jdk8 \ + --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --extra_toolchains=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain-linux-x86_64 \ + --extra_execution_platforms=@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010,@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010-gpu \ + --host_platform=@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010 \ + --local_test_jobs=4 \ + --remote_timeout=3600 \ + --platforms=@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010 \ + -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh new file mode 100644 index 00000000000000..250b0c1253d3ec --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function install_pylint () { + # TODO(gunan): figure out why we get stuck with later versions of pylint. + # TODO(mihaimaruseac): this is used in the release build in the same way, + # maybe extract out to a common? + sudo python2 -m pip install pylint==1.6.4 + sudo python3 -m pip install pylint==1.6.4 +} + +function run_sanity_checks () { + SANITY_OUT=ci_sanity.out + SANITY_OUT_TARGET=gen_ci_sanity_out + + # Run tensorflow sanity checks. + tensorflow/tools/ci_build/ci_sanity.sh 2>&1 | tee tensorflow/tools/ci_build/${SANITY_OUT} + RC=${PIPESTATUS[0]} + + # Since we are running the sanity build remotely (rbe), we need to build a bazel + # target that would output the log generated above and return the expected + # error code. + cat << EOF > tensorflow/tools/ci_build/BUILD +package(default_visibility = ["//tensorflow:internal"]) + +sh_test( + name = "${SANITY_OUT_TARGET}", + srcs = ["${SANITY_OUT_TARGET}.sh"], + data = ["${SANITY_OUT}"], + tags = ["local"], +) +EOF + + cat << EOF > tensorflow/tools/ci_build/${SANITY_OUT_TARGET}.sh +#!/bin/bash +cat tensorflow/tools/ci_build/${SANITY_OUT} +exit ${RC} +EOF + + # Now trigger the rbe build that outputs the log + chmod +x tensorflow/tools/ci_build/${SANITY_OUT_TARGET}.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + "${BAZEL_WRAPPER_PATH}" \ + --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ + test \ + --test_output=all \ + tensorflow/tools/ci_build:${SANITY_OUT_TARGET} + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +install_pylint + +run_sanity_checks diff --git a/tensorflow/tools/ci_build/presubmit/windows/cpu_py36_full/build.bat b/tensorflow/tools/ci_build/presubmit/windows/cpu_py36_full/build.bat new file mode 100644 index 00000000000000..fcc079f7b0e010 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/windows/cpu_py36_full/build.bat @@ -0,0 +1,44 @@ +echo on +setlocal enableextensions enabledelayedexpansion + +@REM This is the path to bazel_wrapper.py, should be set as an argument +set BAZEL_WRAPPER_PATH=%~f1 + +@REM Load common definitions, install bazel +CALL tensorflow\tools\ci_build\release\common_win.bat + +@REM Set up common variables used through the script +set WIN_OUT=win.out +set WIN_OUT_TARGET=gen_win_out +set BUILD_PATH=tensorflow/tools/ci_build/builds +set GEN_SCRIPT=%BUILD_PATH%/%WIN_OUT_TARGET%.sh +set GEN_BUILD=%BUILD_PATH%/BUILD + +@REM Run the presubmit win build. +CALL tensorflow\tools\ci_build\windows\cpu\pip\run.bat --enable_remote_cache %* > %BUILD_PATH%/%WIN_OUT% 2>&1 +set RC=%errorlevel% + +@REM Since we are running the sanity build remotely (rbe), we need to build a bazel +@REM target that would output the log generated above and return the expected +@REM error code. +echo package(default_visibility = ["//visibility:public"]) > %GEN_BUILD% +echo. >> %GEN_BUILD% +echo sh_test( >> %GEN_BUILD% +echo name = "%WIN_OUT_TARGET%", >> %GEN_BUILD% +echo srcs = ["%WIN_OUT_TARGET%.sh"], >> %GEN_BUILD% +echo data = ["%WIN_OUT%"], >> %GEN_BUILD% +echo tags = ["local"], >> %GEN_BUILD% +echo ) >> %GEN_BUILD% + +echo #!/bin/bash > %GEN_SCRIPT% +echo function rlocation() { >> %GEN_SCRIPT% +echo fgrep -m1 "$1 " "$RUNFILES_MANIFEST_FILE" ^| cut -d' ' -f2- >> %GEN_SCRIPT% +echo } >> %GEN_SCRIPT% +echo cat $(rlocation %BUILD_PATH%/%WIN_OUT%) >> %GEN_SCRIPT% +echo exit %RC% >> %GEN_SCRIPT% + +@REM Now trigger the rbe build that outputs the log +chmod +x %GEN_SCRIPT% + +@REM Run bazel test command. +%PY_EXE% %BAZEL_WRAPPER_PATH% --output_user_root=%TMPDIR% --host_jvm_args=-Dbazel.DigestFunction=SHA256 test %BUILD_PATH%:%WIN_OUT_TARGET% --test_output=all diff --git a/tensorflow/tools/ci_build/presubmit/windows/gpu_py36_full/build.bat b/tensorflow/tools/ci_build/presubmit/windows/gpu_py36_full/build.bat new file mode 100644 index 00000000000000..80edefc2bf050e --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/windows/gpu_py36_full/build.bat @@ -0,0 +1,45 @@ +echo on +setlocal enableextensions enabledelayedexpansion + +@REM This is the path to bazel_wrapper.py, should be set as an argument +set BAZEL_WRAPPER_PATH=%~f1 + +@REM Load common definitions, install bazel +CALL tensorflow\tools\ci_build\release\common_win.bat + +@REM Set up common variables used through the script +set WIN_OUT=win.out +set WIN_OUT_TARGET=gen_win_out +set BUILD_PATH=tensorflow/tools/ci_build/builds +set GEN_SCRIPT=%BUILD_PATH%/%WIN_OUT_TARGET%.sh +set GEN_BUILD=%BUILD_PATH%/BUILD + +@REM Run the presubmit win build. +CALL tensorflow\tools\ci_build\windows\gpu\pip\run.bat --enable_remote_cache %* > %BUILD_PATH%/%WIN_OUT% 2>&1 +set RC=%errorlevel% + +@REM Since we are running the sanity build remotely (rbe), we need to build a bazel +@REM target that would output the log generated above and return the expected +@REM error code. +echo package(default_visibility = ["//visibility:public"]) > %GEN_BUILD% +echo. >> %GEN_BUILD% +echo sh_test( >> %GEN_BUILD% +echo name = "%WIN_OUT_TARGET%", >> %GEN_BUILD% +echo srcs = ["%WIN_OUT_TARGET%.sh"], >> %GEN_BUILD% +echo data = ["%WIN_OUT%"], >> %GEN_BUILD% +echo tags = ["local"], >> %GEN_BUILD% +echo ) >> %GEN_BUILD% + +echo #!/bin/bash > %GEN_SCRIPT% +echo function rlocation() { >> %GEN_SCRIPT% +echo fgrep -m1 "$1 " "$RUNFILES_MANIFEST_FILE" ^| cut -d' ' -f2- >> %GEN_SCRIPT% +echo } >> %GEN_SCRIPT% +echo cat $(rlocation %BUILD_PATH%/%WIN_OUT%) >> %GEN_SCRIPT% +echo exit %RC% >> %GEN_SCRIPT% + +@REM Now trigger the rbe build that outputs the log +chmod +x %GEN_SCRIPT% + +@REM Run bazel test command. +%PY_EXE% %BAZEL_WRAPPER_PATH% --output_user_root=%TMPDIR% --host_jvm_args=-Dbazel.DigestFunction=SHA256 test %BUILD_PATH%:%WIN_OUT_TARGET% --test_output=all + diff --git a/tensorflow/tools/ci_build/release/common_win.bat b/tensorflow/tools/ci_build/release/common_win.bat index ad23e3a1ab3475..95b09008c542e0 100644 --- a/tensorflow/tools/ci_build/release/common_win.bat +++ b/tensorflow/tools/ci_build/release/common_win.bat @@ -56,7 +56,7 @@ IF "%PYTHON_DIRECTORY%"=="Python37" ( :: Set cuda related environment variables. If we are not using CUDA, these are not used. IF NOT DEFINED TF_CUDA_VERSION ( - SET TF_CUDA_VERSION=10.0 + SET TF_CUDA_VERSION=10.1 ) SET TF_CUDNN_VERSION=7 SET TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh index b1c27c2f7b3aa8..e3b74060823279 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py2_full/nonpip.sh @@ -24,7 +24,8 @@ bazel version set_bazel_outdir # Pick a more recent version of xcode -sudo xcode-select --switch /Applications/Xcode_9.2.app/Contents/Developer +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" python -m virtualenv tf_build_env --system-site-packages source tf_build_env/bin/activate diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py2_full/pip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py2_full/pip.sh index 624690b4f6554a..3744559a988429 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py2_full/pip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py2_full/pip.sh @@ -23,13 +23,18 @@ which bazel bazel version set_bazel_outdir +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + # Install macos pip dependencies install_macos_pip_deps sudo -# Export required variables for running pip.sh +# Export required variables for running pip_new.sh export OS_TYPE="MACOS" export CONTAINER_TYPE="CPU" export TF_PYTHON_VERSION='python2' +export TF_BUILD_BOTH_CPU_PACKAGES=1 # Run configure. export TF_NEED_CUDA=0 diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh index 8142cdb2019699..d821656ba12efe 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh @@ -24,7 +24,8 @@ bazel version set_bazel_outdir # Pick a more recent version of xcode -sudo xcode-select --switch /Applications/Xcode_9.2.app/Contents/Developer +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" python3.5 -m virtualenv tf_build_env --system-site-packages source tf_build_env/bin/activate diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/pip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/pip.sh index 8e5ea5cee7faf5..4559c1896164eb 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/pip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/pip.sh @@ -23,13 +23,18 @@ which bazel bazel version set_bazel_outdir +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + # Install macos pip dependencies install_macos_pip_deps sudo pip3.5 -# Export required variables for running pip.sh +# Export required variables for running pip_new.sh export OS_TYPE="MACOS" export CONTAINER_TYPE="CPU" export TF_PYTHON_VERSION='python3.5' +export TF_BUILD_BOTH_CPU_PACKAGES=1 # Run configure. export TF_NEED_CUDA=0 diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh index dcc8147fbf882c..93205f8a60d458 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh @@ -24,7 +24,8 @@ bazel version set_bazel_outdir # Pick a more recent version of xcode -sudo xcode-select --switch /Applications/Xcode_9.2.app/Contents/Developer +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" python3.6 -m virtualenv tf_build_env --system-site-packages source tf_build_env/bin/activate diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/pip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/pip.sh index ed6eff65bf39db..0ae2c3b4069667 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/pip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/pip.sh @@ -23,13 +23,18 @@ which bazel bazel version set_bazel_outdir +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + # Install macos pip dependencies install_macos_pip_deps sudo pip3.6 -# Export required variables for running pip.sh +# Export required variables for running pip_new.sh export OS_TYPE="MACOS" export CONTAINER_TYPE="CPU" export TF_PYTHON_VERSION='python3.6' +export TF_BUILD_BOTH_CPU_PACKAGES=1 # Run configure. export TF_NEED_CUDA=0 diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh index b0eff08a45da92..de34e7be8e33e1 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh @@ -24,7 +24,8 @@ bazel version set_bazel_outdir # Pick a more recent version of xcode -sudo xcode-select --switch /Applications/Xcode_9.2.app/Contents/Developer +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" python -m virtualenv tf_build_env --system-site-packages source tf_build_env/bin/activate diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/pip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/pip.sh index 1c1df96171f4b8..2d5fb071913aff 100644 --- a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/pip.sh +++ b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/pip.sh @@ -23,13 +23,18 @@ which bazel bazel version set_bazel_outdir +# Pick a more recent version of xcode +export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" + # Install macos pip dependencies install_macos_pip_deps sudo pip3.7 -# Export required variables for running pip.sh +# Export required variables for running pip_new.sh export OS_TYPE="MACOS" export CONTAINER_TYPE="CPU" export TF_PYTHON_VERSION='python3.7' +export TF_BUILD_BOTH_CPU_PACKAGES=1 # Run configure. export TF_NEED_CUDA=0 diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nightly_release.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nightly_release.sh index b18c20d2c34658..f121d60ca2af7c 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nightly_release.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nightly_release.sh @@ -21,6 +21,8 @@ set_bazel_outdir install_ubuntu_16_pip_deps pip3.6 +update_bazel_linux + python2.7 tensorflow/tools/ci_build/update_version.py --nightly # Run configure. diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_pip_on_cpu/build.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_pip_on_cpu/build.sh index 4b619aa7c540cd..d6c2df745e1f26 100755 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_pip_on_cpu/build.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_pip_on_cpu/build.sh @@ -53,4 +53,4 @@ WHL_PATH=$(ls "${PIP_WHL_DIR}"/*.whl) cp "${WHL_PATH}" "$(pwd)"/. chmod +x tensorflow/tools/ci_build/builds/docker_cpu_pip.sh -docker run -e "CI_BUILD_USER=$(id -u -n)" -e "CI_BUILD_UID=$(id -u)" -e "CI_BUILD_GROUP=$(id -g -n)" -e "CI_BUILD_GID=$(id -g)" -e "CI_BUILD_HOME=/bazel_pip" -v "$(pwd)":/bazel_pip tensorflow/tensorflow:devel-py3 "./bazel_pip/tensorflow/tools/ci_build/builds/with_the_same_user" "./bazel_pip/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh" +docker run -e "BAZEL_VERSION=${BAZEL_VERSION}" -e "CI_BUILD_USER=$(id -u -n)" -e "CI_BUILD_UID=$(id -u)" -e "CI_BUILD_GROUP=$(id -g -n)" -e "CI_BUILD_GID=$(id -g)" -e "CI_BUILD_HOME=/bazel_pip" -v "$(pwd)":/bazel_pip tensorflow/tensorflow:devel-py3 "./bazel_pip/tensorflow/tools/ci_build/builds/with_the_same_user" "./bazel_pip/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh" diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nightly_release.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nightly_release.sh index 0786f157cf22e6..a2d67494dc60b8 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nightly_release.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nightly_release.sh @@ -31,7 +31,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 export TF_NEED_TENSORRT=1 diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh index f2e6a38abfc34d..ae6cb5aea81995 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip.sh @@ -27,7 +27,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip_v1.sh index fb8b1c12ebab06..be52c7ca37cf76 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/nonpip_v1.sh @@ -20,12 +20,14 @@ source tensorflow/tools/ci_build/release/common.sh install_ubuntu_16_pip_deps pip2.7 +update_bazel_linux + # Run configure. export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip.sh index 1d0064b5e4eaba..c77db80087f833 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip_v1.sh index 9218b90638dc9d..4959fcf8c5dba2 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py2_full/pip_v1.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nightly_release.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nightly_release.sh index ea9c9c259ce7fe..04024cb0ce8a3e 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nightly_release.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nightly_release.sh @@ -31,7 +31,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 export TF_NEED_TENSORRT=1 diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh index 609b06afbc33f2..13f6ce837a9717 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh @@ -27,7 +27,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip_v1.sh index e036175bf8f8f7..b45174a7b7fdb6 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip_v1.sh @@ -27,7 +27,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip.sh index 53b0e6d8a1fc87..4fe4edb8d9cad9 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip_v1.sh index efd52c53362393..a03388fd992284 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip_v1.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nightly_release.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nightly_release.sh index dc4636f6576231..21ef6ec1a85877 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nightly_release.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nightly_release.sh @@ -31,7 +31,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 export TF_NEED_TENSORRT=1 diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh index 93d3fcfec359cc..38ce102e990e5b 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh @@ -27,7 +27,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip_v1.sh index ee041d306fb300..888d8106b97123 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip_v1.sh @@ -27,7 +27,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip.sh index 2e23e6edd76e88..e24b9f5019f249 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip_v1.sh index 5c96c542088a14..fde847237c0210 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip_v1.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nightly_release.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nightly_release.sh index 09da2a756ff126..e44cfd0777da1a 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nightly_release.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nightly_release.sh @@ -31,7 +31,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 export TF_NEED_TENSORRT=1 diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh index a3985d24e3efd5..0a7bbb381378aa 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh @@ -27,7 +27,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip_v1.sh index c99e47e791b0e9..506aa3e857faaf 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip_v1.sh @@ -27,7 +27,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip.sh index da1830cdd72391..ff30c1e88af401 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip_v1.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip_v1.sh index 7787c1ee519971..be85dbfc065c97 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip_v1.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip_v1.sh @@ -32,7 +32,7 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 +export TF_CUDA_VERSION=10.1 export TF_CUDNN_VERSION=7 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release.bat b/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release.bat index f10ba0ecc4fedc..bd8c217ddefe77 100644 --- a/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release.bat +++ b/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release.bat @@ -17,4 +17,4 @@ SET PYTHON_DIRECTORY=Python35 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release.bat b/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release.bat index 244e9479eb53e0..0a81a90a43164c 100644 --- a/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release.bat +++ b/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release.bat @@ -17,4 +17,4 @@ SET PYTHON_DIRECTORY=Python36 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release.bat b/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release.bat index 4164c3cc9b5395..9591d7aac343bd 100644 --- a/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release.bat +++ b/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release.bat @@ -17,4 +17,4 @@ SET PYTHON_DIRECTORY=Python37 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release.bat b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release.bat index 71db61889bcff2..cba62225bee4fe 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release.bat +++ b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release.bat @@ -17,4 +17,7 @@ SET PYTHON_DIRECTORY=Python35 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" +bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_pip_rename.sh index 6a868382777791..039f9516d8601d 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_pip_rename.sh +++ b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_pip_rename.sh @@ -19,6 +19,6 @@ set -x source tensorflow/tools/ci_build/release/common.sh # Copy and rename to tensorflow -for f in $(ls py_test_dir/tensorflow_gpu-*cp3*-cp3*m-win_amd64.whl); do - copy_to_new_project_name "${f}" tensorflow +for f in $(ls py_test_dir/tensorflow-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow_gpu done diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_v1.bat b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_v1.bat index abd63888e70c71..55e4e4f57827ec 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_v1.bat +++ b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_v1.bat @@ -17,7 +17,7 @@ SET PYTHON_DIRECTORY=Python35 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --project_name "tensorflow" for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release.bat b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release.bat index ed0c127afe5512..ede8bd35f52f24 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release.bat +++ b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release.bat @@ -17,4 +17,7 @@ SET PYTHON_DIRECTORY=Python36 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" +bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh \ No newline at end of file diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_pip_rename.sh index 6a868382777791..039f9516d8601d 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_pip_rename.sh +++ b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_pip_rename.sh @@ -19,6 +19,6 @@ set -x source tensorflow/tools/ci_build/release/common.sh # Copy and rename to tensorflow -for f in $(ls py_test_dir/tensorflow_gpu-*cp3*-cp3*m-win_amd64.whl); do - copy_to_new_project_name "${f}" tensorflow +for f in $(ls py_test_dir/tensorflow-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow_gpu done diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_v1.bat b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_v1.bat index 7eafdf8af20b0a..a66ca900e47b66 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_v1.bat +++ b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_v1.bat @@ -17,7 +17,7 @@ SET PYTHON_DIRECTORY=Python36 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --project_name "tensorflow" for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release.bat b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release.bat index 00d85ef9119b71..7509270fc43796 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release.bat +++ b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release.bat @@ -17,4 +17,7 @@ SET PYTHON_DIRECTORY=Python37 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" +bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh \ No newline at end of file diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_pip_rename.sh index 6a868382777791..039f9516d8601d 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_pip_rename.sh +++ b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_pip_rename.sh @@ -19,6 +19,6 @@ set -x source tensorflow/tools/ci_build/release/common.sh # Copy and rename to tensorflow -for f in $(ls py_test_dir/tensorflow_gpu-*cp3*-cp3*m-win_amd64.whl); do - copy_to_new_project_name "${f}" tensorflow +for f in $(ls py_test_dir/tensorflow-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow_gpu done diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_v1.bat b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_v1.bat index 261947f58f380b..059e28134c881d 100644 --- a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_v1.bat +++ b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_v1.bat @@ -17,7 +17,7 @@ SET PYTHON_DIRECTORY=Python37 CALL tensorflow\tools\ci_build\release\common_win.bat -call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --project_name "tensorflow" for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh index 1e825580071f0c..a64d5ef9c9c7d8 100644 --- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh @@ -120,6 +120,10 @@ if [[ "$TF_NIGHTLY" == 1 ]]; then else EXTRA_PIP_FLAGS="--project_name ${PROJECT_NAME} --nightly_flag" fi +else + if [[ -v PROJECT_NAME ]]; then + EXTRA_PIP_FLAGS="--project_name ${PROJECT_NAME}" + fi fi # Enable short object file path to avoid long path issue on Windows. @@ -154,7 +158,7 @@ if [[ "$TF_NIGHTLY" == 1 ]]; then fi # Running python tests on Windows needs pip package installed -PIP_NAME=$(ls ${PY_TEST_DIR}/tensorflow-*.whl) +PIP_NAME=$(ls ${PY_TEST_DIR}/tensorflow*.whl) reinstall_tensorflow_pip ${PIP_NAME} # NUMBER_OF_PROCESSORS is predefined on Windows diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh index 99fa086a025e9a..299cbe32260e52 100644 --- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh @@ -120,6 +120,10 @@ if [[ "$TF_NIGHTLY" == 1 ]]; then else EXTRA_PIP_FLAGS="--project_name ${PROJECT_NAME} --nightly_flag" fi +else + if [[ -v PROJECT_NAME ]]; then + EXTRA_PIP_FLAGS="--project_name ${PROJECT_NAME}" + fi fi # Enable short object file path to avoid long path issue on Windows. @@ -154,7 +158,7 @@ if [[ "$TF_NIGHTLY" == 1 ]]; then fi # Running python tests on Windows needs pip package installed -PIP_NAME=$(ls ${PY_TEST_DIR}/tensorflow_gpu-*.whl) +PIP_NAME=$(ls ${PY_TEST_DIR}/tensorflow*.whl) reinstall_tensorflow_pip ${PIP_NAME} TF_GPU_COUNT=${TF_GPU_COUNT:-4} diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 58dca6fccff790..c1cc78e3269300 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -47,7 +47,7 @@ # result for pip. # Also update tensorflow/tensorflow.bzl and # tensorflow/core/public/version.h -_VERSION = '2.0.0' +_VERSION = '2.1.0' REQUIRED_PACKAGES = [ 'absl-py >= 0.7.0', @@ -61,8 +61,8 @@ 'numpy >= 1.16.0, < 2.0', 'opt_einsum >= 2.3.2', 'protobuf >= 3.8.0', - 'tensorboard >= 2.0.0, < 2.1.0', - 'tensorflow_estimator >= 2.0.0, < 2.1.0', + 'tensorboard >= 2.1.0, < 2.2.0', + 'tensorflow_estimator >= 2.1.0rc0, < 2.2.0', 'termcolor >= 1.1.0', 'wrapt >= 1.11.1', # python3 requires wheel 0.26 @@ -73,6 +73,10 @@ # functools comes with python3, need to install the backport for python2 'functools32 >= 3.2.3;python_version<"3"', 'six >= 1.12.0', + # scipy < 1.4.1 causes segfaults due to pybind11 + # Latest scipy pip for py2 is scipy==1.2.2 + 'scipy == 1.4.1;python_version>="3"', + 'scipy == 1.2.2;python_version<"3"', ] if sys.byteorder == 'little': diff --git a/tensorflow/virtual_root_template_v1.__init__.py b/tensorflow/virtual_root_template_v1.__init__.py index 236e9f52258973..9a45bc0355d0b7 100644 --- a/tensorflow/virtual_root_template_v1.__init__.py +++ b/tensorflow/virtual_root_template_v1.__init__.py @@ -132,7 +132,4 @@ def _forward_module(old_name): except NameError: pass -# Manually patch keras and estimator so tf.keras and tf.estimator work -keras = _sys.modules["tensorflow.keras"] -if not _root_estimator: estimator = _sys.modules["tensorflow.estimator"] # LINT.ThenChange(//tensorflow/virtual_root_template_v2.__init__.py.oss) diff --git a/tensorflow/virtual_root_template_v2.__init__.py b/tensorflow/virtual_root_template_v2.__init__.py index 83c020182a8ee9..bd8c903e455db5 100644 --- a/tensorflow/virtual_root_template_v2.__init__.py +++ b/tensorflow/virtual_root_template_v2.__init__.py @@ -126,14 +126,4 @@ def _forward_module(old_name): except NameError: pass -# TODO(mihaimaruseac): Revisit all of this once we release 2.1 -# Manually patch keras and estimator so tf.keras and tf.estimator work -keras = _sys.modules["tensorflow.keras"] -if not _root_estimator: estimator = _sys.modules["tensorflow.estimator"] -# Also import module aliases -try: - from tensorflow_core import losses, metrics, initializers, optimizers -except ImportError: - pass - # LINT.ThenChange(//tensorflow/virtual_root_template_v1.__init__.py.oss) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 47dfc9eb600f17..77e605fe76a6aa 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -171,11 +171,11 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): name = "eigen_archive", build_file = clean_dep("//third_party:eigen.BUILD"), patch_file = clean_dep("//third_party/eigen3:gpu_packet_math.patch"), - sha256 = "091d1a3124ea41ac2e70e30028365d78d43a1c617a26445aef15e140e4fab1dd", - strip_prefix = "eigen-eigen-afc120bc03bd", + sha256 = "65d732985b593b553c20566e1f236f48dcc626730c418aed7b2aa1d0e3f1a0af", + strip_prefix = "eigen-4e696901f873a2347f76d931cf2f701e31e15d05", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/bitbucket.org/eigen/eigen/get/afc120bc03bd.tar.gz", - "https://bitbucket.org/eigen/eigen/get/afc120bc03bd.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/gitlab.com/libeigen/eigen/-/archive/4e696901f873a2347f76d931cf2f701e31e15d05/eigen-4e696901f873a2347f76d931cf2f701e31e15d05.tar.gz", + "https://gitlab.com/libeigen/eigen/-/archive/4e696901f873a2347f76d931cf2f701e31e15d05/eigen-4e696901f873a2347f76d931cf2f701e31e15d05.tar.gz", ], ) diff --git a/third_party/eigen3/gpu_packet_math.patch b/third_party/eigen3/gpu_packet_math.patch index 50ac056df79a68..1b6131abd41bc2 100644 --- a/third_party/eigen3/gpu_packet_math.patch +++ b/third_party/eigen3/gpu_packet_math.patch @@ -22,4 +22,161 @@ return res; } }; - \ No newline at end of file +--- a/unsupported/Eigen/SpecialFunctions ++++ b/unsupported/Eigen/SpecialFunctions +@@ -48,6 +48,9 @@ + } + + #include "src/SpecialFunctions/SpecialFunctionsImpl.h" ++#if defined(EIGEN_HIPCC) ++#include "src/SpecialFunctions/HipVectorCompatibility.h" ++#endif + #include "src/SpecialFunctions/SpecialFunctionsPacketMath.h" + #include "src/SpecialFunctions/SpecialFunctionsHalf.h" + #include "src/SpecialFunctions/SpecialFunctionsFunctors.h" +--- /dev/null ++++ b/unsupported/Eigen/src/SpecialFunctions/HipVectorCompatibility.h +@@ -0,0 +1,143 @@ ++#ifndef HIP_VECTOR_COMPATIBILITY_H ++#define HIP_VECTOR_COMPATIBILITY_H ++ ++namespace hip_impl { ++ template struct Scalar_accessor; ++} // end namespace hip_impl ++ ++namespace Eigen { ++namespace internal { ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct lgamma_impl> : lgamma_impl {}; ++#endif ++ ++template ++struct digamma_impl_maybe_poly> ++ : digamma_impl_maybe_poly {}; ++ ++template ++struct digamma_impl> : digamma_impl {}; ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct erf_impl> : erf_impl {}; ++#endif // EIGEN_HAS_C99_MATH ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct erfc_impl> : erfc_impl {}; ++#endif // EIGEN_HAS_C99_MATH ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct ndtri_impl> : ndtri_impl {}; ++#endif // EIGEN_HAS_C99_MATH ++ ++template ++struct igammac_cf_impl, mode> ++ : igammac_cf_impl {}; ++ ++template ++struct igamma_series_impl, mode> ++ : igamma_series_impl {}; ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct igammac_impl> : igammac_impl {}; ++#endif // EIGEN_HAS_C99_MATH ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct igamma_generic_impl, mode> ++ : igamma_generic_impl {}; ++#endif // EIGEN_HAS_C99_MATH ++ ++template ++struct igamma_impl> : igamma_impl {}; ++ ++template ++struct igamma_der_a_retval> ++ : igamma_der_a_retval {}; ++ ++template ++struct igamma_der_a_impl> ++ : igamma_der_a_impl {}; ++ ++template ++struct gamma_sample_der_alpha_retval> ++ : gamma_sample_der_alpha_retval {}; ++ ++template ++struct gamma_sample_der_alpha_impl> ++ : gamma_sample_der_alpha_impl {}; ++ ++template ++struct zeta_impl_series> ++ : zeta_impl_series {}; ++ ++template ++struct zeta_impl> : zeta_impl {}; ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct polygamma_impl> ++ : polygamma_impl {}; ++#endif // EIGEN_HAS_C99_MATH ++ ++#if EIGEN_HAS_C99_MATH ++template ++struct betainc_impl> : betainc_impl {}; ++ ++template ++struct incbeta_cfe> : incbeta_cfe {}; ++ ++template ++struct betainc_helper> ++ : betainc_helper {}; ++#else ++template ++struct betainc_impl> : betainc_impl {}; ++#endif // EIGEN_HAS_C99_MATH ++ ++template ++struct bessel_i0e_impl> : bessel_i0e_impl {}; ++ ++template ++struct bessel_i0_impl> : bessel_i0_impl {}; ++ ++template ++struct bessel_i1e_impl> : bessel_i1e_impl {}; ++ ++template ++struct bessel_i1_impl> : bessel_i1_impl {}; ++ ++template ++struct bessel_k0e_impl> : bessel_k0e_impl {}; ++ ++template ++struct bessel_k0_impl> : bessel_k0_impl {}; ++ ++template ++struct bessel_k1e_impl> : bessel_k1e_impl {}; ++ ++template ++struct bessel_k1_impl> : bessel_k1_impl {}; ++ ++template ++struct bessel_j0_impl> : bessel_j0_impl {}; ++ ++template ++struct bessel_y0_impl> : bessel_y0_impl {}; ++ ++template ++struct bessel_j1_impl> : bessel_j1_impl {}; ++ ++template ++struct bessel_y1_impl> : bessel_y1_impl {}; ++ ++} // end namespace internal ++} // end namespace Eigen ++ ++#endif // HIP_VECTOR_COMPATIBILITY_H diff --git a/third_party/gpus/crosstool/BUILD.rocm.tpl b/third_party/gpus/crosstool/BUILD.rocm.tpl index be32aa7f808c34..8ca69e117c8486 100644 --- a/third_party/gpus/crosstool/BUILD.rocm.tpl +++ b/third_party/gpus/crosstool/BUILD.rocm.tpl @@ -90,7 +90,7 @@ cc_toolchain_config( "-lm", ], link_libs = [], - opt_link_flags = ["-Wl,--gc-sections"], + opt_link_flags = [], unfiltered_compile_flags = [ "-fno-canonical-system-headers", "-Wno-builtin-macro-redefined", diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc5-rocm/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/gcc5-rocm/BUILD index 00daa6042acd97..e1b16d20dbe397 100755 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc5-rocm/BUILD +++ b/third_party/toolchains/preconfig/ubuntu16.04/gcc5-rocm/BUILD @@ -122,7 +122,7 @@ cc_toolchain_config( "-ffunction-sections", "-fdata-sections", ], - opt_link_flags = ["-Wl,--gc-sections"], + opt_link_flags = [], supports_start_end_lib = True, target_libc = "local", target_system_name = "local", diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1/clang/bin/crosstool_wrapper_driver_is_not_gcc b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1/clang/bin/crosstool_wrapper_driver_is_not_gcc index af878f037734e0..44e745f69e02bd 100755 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1/clang/bin/crosstool_wrapper_driver_is_not_gcc +++ b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1/clang/bin/crosstool_wrapper_driver_is_not_gcc @@ -53,6 +53,11 @@ NVCC_PATH = '/usr/local/cuda-10.1/bin/nvcc' PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH) NVCC_VERSION = '10.1' +# Environment variable for supported TF CUDA Compute Capabilities +# eg. export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 +CUDA_COMPUTE_ENV_VAR = 'TF_CUDA_COMPUTE_CAPABILITIES' +DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,6.0' + def Log(s): print('gpus/crosstool: {0}'.format(s)) @@ -202,7 +207,7 @@ def InvokeNvcc(argv, log=False): srcs = ' '.join(src_files) out = ' -o ' + out_file[0] - supported_cuda_compute_capabilities = [ "3.0", "6.0" ] + supported_cuda_compute_capabilities = os.environ.get(CUDA_COMPUTE_ENV_VAR, DEFAULT_CUDA_COMPUTE_CAPABILITIES).split(',') nvccopts = '-D_FORCE_INLINES ' for capability in supported_cuda_compute_capabilities: capability = capability.replace('.', '')