build: bump to TensorFlow 2.4.0 (tensorflow#1156)

- Remove the 2.3 compatibility path
- Update the install rules to install additional headers for X10
- Update the documentation to point to a 2.4 build of TensorFlow
- Update the include paths for the internal use of TensorFlow
- Update macOS/Windows CI rules to use the 2.4 release build

.github/workflows/macOS.yml

@@ -22,14 +22,14 @@ jobs:
 
       - name: Install X10
         run: |
-          curl -sL https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip -o tensorflow-darwin-x64.zip
+          curl -sL https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip -o tensorflow-darwin-x64.zip
           unzip tensorflow-darwin-x64.zip
-          mv tensorflow-darwin-x64/Library/tensorflow-2.3.0 ~/Library/
+          mv tensorflow-darwin-x64/Library/tensorflow-2.4.0 ~/Library/
 
       - name: Build
         run: |
-          TOOLCHAINS=org.swift.50202012141a swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.3.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.3.0/usr/lib
+          TOOLCHAINS=org.swift.50202012141a swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.4.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.4.0/usr/lib
     # - name: Run tests
     #   run: |
-    #     DYLD_LIBRARY_PATH=${HOME}/Library/tensorflow-2.3.0/usr/lib TOOLCHAINS=org.swift.50202012141a swift test -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.3.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.3.0/usr/lib
+    #     DYLD_LIBRARY_PATH=${HOME}/Library/tensorflow-2.4.0/usr/lib TOOLCHAINS=org.swift.50202012141a swift test -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.4.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.4.0/usr/lib
 

.github/workflows/windows.yml

@@ -32,16 +32,16 @@ jobs:
 
       - name: Install X10
         run: |
-          Invoke-WebRequest -UseBasicParsing -Uri https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip -OutFile tensorflow-windows-x64.zip
+          Invoke-WebRequest -UseBasicParsing -Uri https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip -OutFile tensorflow-windows-x64.zip
           Expand-Archive -Force -Path tensorflow-windows-x64.zip -DestinationPath C:\Library\
-          Move-Item C:\Library\tensorflow-windows-x64\Library\tensorflow-2.3.0 C:\Library
-          echo "C:\Library\tensorflow-2.3.0\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          Move-Item C:\Library\tensorflow-windows-x64\Library\tensorflow-2.4.0 C:\Library
+          echo "C:\Library\tensorflow-2.4.0\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
       - name: Build
         run: |
           # We build twice as the first one currently fails for some unexplained
           # reason.  This should only build once.
-          swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.3.0\usr\include -Xlinker -LC:\Library\tensorflow-2.3.0\usr\lib
-          swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.3.0\usr\include -Xlinker -LC:\Library\tensorflow-2.3.0\usr\lib
+          swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.4.0\usr\include -Xlinker -LC:\Library\tensorflow-2.4.0\usr\lib
+          swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.4.0\usr\include -Xlinker -LC:\Library\tensorflow-2.4.0\usr\lib
     # - name: Run tests
     #   run: swift test -v

CMakeLists.txt

@@ -167,7 +167,7 @@ if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
     GIT_REPOSITORY
       git://github.com/tensorflow/tensorflow
     GIT_TAG
-      r2.3
+      r2.4
     UPDATE_DISCONNECTED
       TRUE
     CONFIGURE_COMMAND
@@ -207,6 +207,7 @@ if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
   set(X10_LIBRARY ${SOURCE_DIR}/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX})
   set(X10_INCLUDE_DIRS
     ${SOURCE_DIR}
+    ${SOURCE_DIR}/tensorflow
     ${SOURCE_DIR}/bazel-bin
     ${SOURCE_DIR}/bazel-libtensorflow/external/com_google_absl
     ${SOURCE_DIR}/bazel-libtensorflow/external/com_google_protobuf/src

Documentation/Development.md

@@ -409,8 +409,8 @@ $ swift test -Xcc -I/Library/tensorflow-2.4.0/usr/include -Xlinker -L/Library/te
 
 [swift]: https://swift.org/download/#snapshots
 [cmake]: https://www.cmake.org/download
-[windows10]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip
-[macOS]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip
+[windows10]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip
+[macOS]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip
 [bazel]: https://docs.bazel.build/versions/master/install.html
 [bazelisk]: https://github.com/bazelbuild/bazelisk
 [configure.py]: https://github.com/tensorflow/tensorflow/blob/master/configure.py

Sources/CTensorFlow/CMakeLists.txt

@@ -12,13 +12,21 @@ list(GET X10_INCLUDE_DIRS 0 X10_SOURCE_DIR)
 
 install(FILES include/module.modulemap include/CTensorFlow.h
   DESTINATION lib/swift/tensorflow)
-install(DIRECTORY ${X10_SOURCE_DIR}/tensorflow/c
-  DESTINATION lib/swift/tensorflow
-  FILES_MATCHING
-    PATTERN c_api.h
-    PATTERN c_api_experimental.h
-    PATTERN tf_attrtype.h
-    PATTERN tf_datatype.h
-    PATTERN tf_status.h
-    PATTERN tf_tensor.h
-    PATTERN eager/c_api.h)
+install(FILES
+  ${X10_SOURCE_DIR}/tensorflow/c/c_api.h
+  ${X10_SOURCE_DIR}/tensorflow/c/c_api_experimental.h
+  ${X10_SOURCE_DIR}/tensorflow/c/c_api_macros.h
+  ${X10_SOURCE_DIR}/tensorflow/c/tf_attrtype.h
+  ${X10_SOURCE_DIR}/tensorflow/c/tf_datatype.h
+  ${X10_SOURCE_DIR}/tensorflow/c/tf_file_statistics.h
+  ${X10_SOURCE_DIR}/tensorflow/c/tf_status.h
+  ${X10_SOURCE_DIR}/tensorflow/c/tf_tensor.h
+  ${X10_SOURCE_DIR}/tensorflow/c/tf_tstring.h
+  DESTINATION lib/swift/tensorflow/c)
+install(FILES
+  ${X10_SOURCE_DIR}/tensorflow/c/eager/c_api.h
+  DESTINATION lib/swift/tensorflow/c/eager)
+install(FILES
+  ${X10_SOURCE_DIR}/tensorflow/core/platform/ctstring.h
+  ${X10_SOURCE_DIR}/tensorflow/core/platform/ctstring_internal.h
+  DESTINATION lib/swift/tensorflow/core/platform)

Sources/TensorFlow/Core/StringTensor.swift

@@ -47,72 +47,23 @@ extension StringTensor {
     // utf8CString is null-terminated. TF APIs want the strings without null-terminators.
     let cStrings = scalars.map { $0.utf8CString.dropLast() }
 
-    // Note: `TENSORFLOW_MASTER` changes below are necessary for the new TensorFlow ABI-stable
-    // unified string tensor design.
-    #if TENSORFLOW_MASTER
-      let byteCount = scalars.count * MemoryLayout<TF_TString>.stride
-
-      let handle = TensorHandle<String>(
-        shape: shape.dimensions,
-        byteCount: byteCount,
-        bufferInitializer: { tensorBuffer in
-          var dataAddr =
-            tensorBuffer.bindMemory(to: TF_TString.self, capacity: scalars.count)
-          for cString in cStrings {
-            TF_TString_Init(dataAddr)
-            cString.withUnsafeBufferPointer { buffer in
-              TF_TString_Copy(dataAddr, buffer.baseAddress, buffer.count)
-            }
-            dataAddr = dataAddr.advanced(by: 1)
-          }
-        })
-      self.init(handle: handle)
-    #else
-      let tfEncodedSizes = cStrings.map { TF_StringEncodedSize($0.count) }
-
-      // Format information copied from tensorflow/c/c_api.h:
-      // The format for TF_STRING tensors is:
-      //   start_offset: array[uint64]
-      //   data:         byte[...]
-      //
-      //   The string length (as a varint), followed by the contents of the string is encoded at
-      //   data[start_offset[i]]].
-      // The size of the "start_offset" region.
-      let startOffsetsByteCount = scalars.count * MemoryLayout<UInt64>.stride
-
-      // The size of the "data" region.
-      let dataByteCount = tfEncodedSizes.reduce(0, +) * MemoryLayout<UInt8>.stride
-
-      let handle = TensorHandle<String>(
-        shape: shape.dimensions,
-        byteCount: startOffsetsByteCount + dataByteCount,
-        bufferInitializer: { tensorBuffer in
-          // Initialize the "start_offset" region.
-          var startOffset: UInt64 = 0
-          var startOffsetAddr =
-            tensorBuffer.bindMemory(to: UInt64.self, capacity: scalars.count)
-          for tfEncodedSize in tfEncodedSizes {
-            startOffsetAddr.initialize(to: startOffset)
-            startOffsetAddr = startOffsetAddr.advanced(by: 1)
-            startOffset = startOffset + UInt64(tfEncodedSize)
-          }
-
-          // Initialize the "data" region.
-          var dataAddr = tensorBuffer.advanced(by: startOffsetsByteCount)
-            .bindMemory(to: Int8.self, capacity: dataByteCount)
-          let status = TF_NewStatus()
-          for (cString, tfEncodedSize) in zip(cStrings, tfEncodedSizes) {
-            _ = cString.withUnsafeBufferPointer { buffer in
-              TF_StringEncode(
-                buffer.baseAddress, buffer.count, dataAddr, tfEncodedSize, status)
-            }
-            checkOk(status)
-            dataAddr = dataAddr.advanced(by: tfEncodedSize)
+    let byteCount = scalars.count * MemoryLayout<TF_TString>.stride
+
+    let handle = TensorHandle<String>(
+      shape: shape.dimensions,
+      byteCount: byteCount,
+      bufferInitializer: { tensorBuffer in
+        var dataAddr =
+          tensorBuffer.bindMemory(to: TF_TString.self, capacity: scalars.count)
+        for cString in cStrings {
+          TF_TString_Init(dataAddr)
+          cString.withUnsafeBufferPointer { buffer in
+            TF_TString_Copy(dataAddr, buffer.baseAddress, buffer.count)
           }
-          TF_DeleteStatus(status)
-        })
-      self.init(handle: handle)
-    #endif
+          dataAddr = dataAddr.advanced(by: 1)
+        }
+      })
+    self.init(handle: handle)
   }
 
   /// Creates a 0-D `StringTensor` from a scalar value.