feat: Upgrade to protobuf 27.0 and remove py_proto_library (#1933)

Protobuf team is taking ownership of `py_proto_library` and the
implementation was moved to protobuf repository.

Remove py_proto_library from rules_python, to prevent divergent
implementations.

Make a redirect with a deprecation warning, so that this doesn't break
any users.

Expected side effect of this change is also that the protobuf version is
sufficiently updated that there is no more use of legacy struct
providers.

Closes #1935 
Closes #1924 
Closes #1925 
Closes #1703 
Closes #1707 
Closes #1597 
Closes #1293 
Closes #1080
Fixes #1438

Author: comius

.bazelci/presubmit.yml

@@ -147,6 +147,14 @@ tasks:
     <<: *reusable_config
     name: "Default: Debian"
     platform: debian11
+    build_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
+    test_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
   macos:
     <<: *reusable_config
     name: "Default: MacOS"
@@ -169,19 +177,39 @@ tasks:
       # build kite cc toolchain.
       - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
       - "--build_tag_filters=-docs"
+    build_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
     test_flags:
       - "--test_tag_filters=-integration-test,-acceptance-test,-docs"
       # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1,
       # which prevents cc toolchain autodetection from working correctly
       # on Bazel 5.4 and earlier. To workaround this, manually specify the
       # build kite cc toolchain.
       - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
+    test_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
   rbe:
     <<: *reusable_config
     name: "RBE: Ubuntu"
     platform: rbe_ubuntu1604
+    build_flags:
+      - "--build_tag_filters=-docs"
+    build_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
+      - '-//docs/...'
     test_flags:
-      - "--test_tag_filters=-integration-test,-acceptance-test"
+      - "--test_tag_filters=-integration-test,-acceptance-test,-docs"
+    test_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
+      - '-//docs/...'
 
   integration_test_build_file_generation_ubuntu_minimum_supported_workspace:
     <<: *minimum_supported_version
@@ -234,6 +262,21 @@ tasks:
     name: "examples/bzlmod: Debian"
     working_directory: examples/bzlmod
     platform: debian11
+    build_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
+    test_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
+    coverage_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
   integration_test_bzlmod_macos:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod
@@ -395,6 +438,14 @@ tasks:
     name: "examples/py_proto_library: Debian, workspace"
     working_directory: examples/py_proto_library
     platform: debian11
+    build_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
+    test_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
   integration_test_py_proto_library_macos_workspace:
     <<: *reusable_build_test_all
     <<: *common_workspace_flags

CHANGELOG.md

@@ -25,6 +25,9 @@ A brief description of the categories of changes:
 [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x
 
 ### Changed
+* (rules) `py_proto_library` is deprecated in favour of the
+  implementation in https://github.com/protocolbuffers/protobuf. It will be
+  removed in the future release.
 * (deps) Upgrade the `pip_install` dependencies to pick up a new version of pip.
 * (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and
   `py_library` now depend on the `//python:exec_tools_toolchain_type` for build

MODULE.bazel

@@ -9,9 +9,9 @@ bazel_dep(name = "bazel_skylib", version = "1.6.1")
 bazel_dep(name = "rules_cc", version = "0.0.9")
 bazel_dep(name = "platforms", version = "0.0.4")
 
-# Those are loaded only when using py_proto_library
-bazel_dep(name = "rules_proto", version = "6.0.0-rc1")
-bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
+# For backwards compatibility, those are loaded only when using py_proto_library
+# Use py_proto_library directly from protobuf repository
+bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf")
 
 internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps")
 use_repo(

WORKSPACE

@@ -148,9 +148,3 @@ http_file(
         "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
     ],
 )
-
-# rules_proto expects //external:python_headers to point at the python headers.
-bind(
-    name = "python_headers",
-    actual = "//python/cc:current_py_cc_headers",
-)

WORKSPACE.bzlmod

@@ -55,8 +55,3 @@ http_file(
     ],
 )
 
-# rules_proto expects //external:python_headers to point at the python headers.
-bind(
-    name = "python_headers",
-    actual = "//python/cc:current_py_cc_headers",
-)

examples/bzlmod/MODULE.bazel

@@ -11,11 +11,8 @@ local_path_override(
     path = "../..",
 )
 
-# (py_proto_library specific) We are using rules_proto to define rules_proto targets to be consumed by py_proto_library.
-bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
-
 # (py_proto_library specific) Add the protobuf library for well-known types (e.g. `Any`, `Timestamp`, etc)
-bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
+bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf")
 
 # We next initialize the python toolchain using the extension.
 # You can set different Python versions in this block.

examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "message_proto_py_pb2",

examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "pricetag_proto_py_pb2",

examples/py_proto_library/WORKSPACE

@@ -24,24 +24,11 @@ python_register_toolchains(
 # Then we need to setup dependencies in order to use py_proto_library
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-http_archive(
-    name = "rules_proto",
-    sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c",
-    strip_prefix = "rules_proto-6.0.0-rc1",
-    url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz",
-)
-
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
-
-rules_proto_dependencies()
-
-rules_proto_toolchains()
-
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "4fc5ff1b2c339fb86cd3a25f0b5311478ab081e65ad258c6789359cd84d421f8",
-    strip_prefix = "protobuf-26.1",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v26.1.tar.gz"],
+    sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa",
+    strip_prefix = "protobuf-27.0",
+    urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz"],
 )
 
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")

examples/py_proto_library/example.com/another_proto/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "message_proto_py_pb2",

examples/py_proto_library/example.com/proto/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "pricetag_proto_py_pb2",

internal_deps.bzl

@@ -164,20 +164,12 @@ def rules_python_internal_deps():
         ],
     )
 
-    http_archive(
-        name = "rules_proto",
-        sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c",
-        strip_prefix = "rules_proto-6.0.0-rc1",
-        url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz",
-    )
-
     http_archive(
         name = "com_google_protobuf",
-        sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae",
-        strip_prefix = "protobuf-21.7",
+        sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa",
+        strip_prefix = "protobuf-27.0",
         urls = [
-            "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
-            "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
+            "https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz",
         ],
     )
 

internal_setup.bzl

@@ -20,7 +20,6 @@ load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies")
 load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries")
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
 load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
 load("//python/private:internal_config_repo.bzl", "internal_config_repo")  # buildifier: disable=bzl-visibility
@@ -35,9 +34,6 @@ def rules_python_internal_setup():
 
     bazel_skylib_workspace()
 
-    rules_proto_dependencies()
-    rules_proto_toolchains()
-
     protobuf_deps()
 
     bazel_integration_test_rules_dependencies()

python/BUILD.bazel

@@ -113,7 +113,7 @@ bzl_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        "//python/private/proto:py_proto_library_bzl",
+        "@com_google_protobuf//bazel:py_proto_library_bzl",
     ],
 )
 

python/private/BUILD.bazel

@@ -29,7 +29,6 @@ filegroup(
     srcs = glob(["**"]) + [
         "//python/private/bzlmod:distribution",
         "//python/private/common:distribution",
-        "//python/private/proto:distribution",
         "//python/private/whl_filegroup:distribution",
         "//tools/build_defs/python/private:distribution",
     ],