Update protobuf library version

As of Sept, upb has moved into protobuf; update to support that.

This reworks how protos are generated to feel a bit closer to the aspect-like code generation of Bazel. Now each proto generator is registered, and each proto_library() rule will invoke all registered aspects; this will allow consistent and reliable invocation of protoc across all sub-repositories.

The change permits removing duplicate .proto files from local_proto_mirror, and simplifies the boostrapping process.

Also note, on windows the updated protobuf is more likely to trigger file not found issues due to path length; if this happens, add the bazel startup option:

  startup --output_user_root=C:/_out

See also:

bazelbuild/bazel#18683
protocolbuffers/protobuf#12947
grpc/grpc#33986

PiperOrigin-RevId: 588606143
Change-Id: Iafa5f16b0a9bbc1058a876c0cc739d82743f96a4

Author: laramiel

.bazelversion

@@ -1 +1 @@
-6.1.0
+6.4.0

CMakeLists.txt

@@ -16,7 +16,7 @@
 # build.
 
 cmake_minimum_required(VERSION 3.24)
-project(tensorstore LANGUAGES CXX)
+project(tensorstore LANGUAGES C CXX)
 
 # Enable CMP0077 (option honors variables) for subprojects.
 set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
@@ -92,6 +92,7 @@ bazel_to_cmake(
   --ignore-library "//docs:doctest.bzl"
   --ignore-library "//bazel:non_compile.bzl"
   --ignore-library "@com_google_protobuf//:protobuf.bzl"
+  --ignore-library "@rules_python//python:packaging.bzl"
   --bazelrc .bazelrc
   --module bazel_to_cmake.bzl_library.grpc_generate_cc
   --module bazel_to_cmake.bzl_library.local_mirror

WORKSPACE

@@ -7,3 +7,8 @@ load("//:external.bzl", "tensorstore_dependencies")
 tensorstore_dependencies()
 
 register_toolchains("@local_config_python//:py_toolchain")
+
+# Register proto toolchains.
+load("@rules_proto//proto:repositories.bzl", "rules_proto_toolchains")
+
+rules_proto_toolchains()

third_party/com_github_cncf_udpa/cmake_extra.BUILD.bazel

@@ -12,7 +12,7 @@
 # patching the repository.
 
 load(
-    "@com_google_protobuf_upb//bazel:upb_proto_library.bzl",
+    "@com_google_protobuf//bazel:upb_proto_library.bzl",
     "upb_proto_library",
     "upb_proto_reflection_library",
 )

third_party/com_github_grpc_grpc/patches/update_build_system.diff

@@ -15,30 +15,114 @@
 # NOTE: The second patch to includes has no change, but is intended to signal
 # whether we still need a workaround in bazel_to_cmake/emit_cc.py
 #
-diff -u -Nur a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
---- a/bazel/grpc_build_system.bzl	2023-09-13 20:33:23.000000000 +0000
-+++ b/bazel/grpc_build_system.bzl	2023-10-04 01:16:42.011957527 +0000
-@@ -32,8 +32,14 @@
+diff -Nur a/BUILD b/BUILD
+--- a/BUILD	2023-09-13 13:33:23
++++ b/BUILD	2023-12-04 22:12:55
+@@ -194,6 +194,13 @@
+ )
+
+ config_setting(
++    name = "mingw-gcc",
++    flag_values = {
++        "@bazel_tools//tools/cpp:compiler": "mingw-gcc",
++    },
++)
++
++config_setting(
+     name = "mac_x86_64",
+     values = {"cpu": "darwin_x86_64"},
+ )
+@@ -1116,6 +1123,7 @@
+         "//src/core:slice_refcount",
+         "//src/core:status_helper",
+         "//src/core:transport_fwd",
++        "//src/core:notification",
+     ],
+ )
+
+@@ -1842,6 +1850,7 @@
+     deps = [
+         "gpr",
+         "grpc_trace",
++        "grpc_public_hdrs",
+     ],
+ )
+
+@@ -1924,6 +1933,8 @@
+         "absl/memory",
+         "absl/types:optional",
+         "upb_lib",
++        "upb_base",
++        "upb_mem",
+         "protobuf_headers",
+         "absl/container:inlined_vector",
+     ],
+@@ -2075,6 +2086,7 @@
+     external_deps = [
+         "protobuf_headers",
+         "protobuf",
++        "absl/status",
+     ],
+     language = "c++",
+     public_hdrs = [
+@@ -2738,6 +2750,7 @@
+     external_deps = [
+         "absl/strings",
+         "absl/strings:str_format",
++        "absl/types:optional",
+     ],
+     tags = ["nofixdeps"],
+     visibility = ["@grpc:iomgr_buffer_list"],
+@@ -3016,6 +3029,7 @@
+         "absl/types:variant",
+         "upb_collections_lib",
+         "upb_lib",
++        "upb_base",
+     ],
+     language = "c++",
+     visibility = ["@grpc:client_channel"],
+@@ -3620,6 +3634,8 @@
+         "upb_textformat_lib",
+         "upb_json_lib",
+         "upb_reflection",
++        "upb_mem",
++        "upb_base",
+     ],
+     language = "c++",
+     tags = ["nofixdeps"],
+@@ -3647,6 +3663,7 @@
+         "protobuf_struct_upb",
+         "protobuf_timestamp_upb",
+         "ref_counted_ptr",
++        ":server_address",
+         "uri_parser",
+         "work_serializer",
+         "//src/core:default_event_engine",
+diff -Nur a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
+--- a/bazel/grpc_build_system.bzl	2023-09-13 13:33:23
++++ b/bazel/grpc_build_system.bzl	2023-12-04 22:07:56
+@@ -32,9 +32,15 @@
  load("//bazel:experiments.bzl", "EXPERIMENTS")
  load("//bazel:test_experiments.bzl", "TEST_EXPERIMENTS")
  load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library", "upb_proto_reflection_library")
 -load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test")
 -load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner")
 +# load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test")
 +# load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner")
-+
+
 +def ios_unit_test(**kwargs):
 +    pass
 +
 +def ios_test_runner(**kwargs):
 +    pass
- 
++
  # The set of pollers to test against if a test exercises polling
  POLLERS = ["epoll1", "poll"]
+
 @@ -55,6 +61,12 @@
          "//conditions:default": [],
      })
- 
+
 +def if_mingw(a):
 +    return select({
 +        "//:mingw-gcc": a,
@@ -75,71 +159,67 @@ diff -u -Nur a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
          "size": size,
          "timeout": timeout,
          "exec_compatible_with": exec_compatible_with,
-diff -u -Nur a/BUILD b/BUILD
---- a/BUILD	2023-09-13 20:33:23.000000000 +0000
-+++ b/BUILD	2023-10-04 01:20:07.264833917 +0000
-@@ -194,6 +194,13 @@
- )
- 
- config_setting(
-+    name = "mingw-gcc",
-+    flag_values = {
-+        "@bazel_tools//tools/cpp:compiler": "mingw-gcc",
-+    },
-+)
-+
-+config_setting(
-     name = "mac_x86_64",
-     values = {"cpu": "darwin_x86_64"},
- )
-@@ -1116,6 +1123,7 @@
-         "//src/core:slice_refcount",
-         "//src/core:status_helper",
-         "//src/core:transport_fwd",
-+        "//src/core:notification",
+diff -Nur a/src/core/BUILD b/src/core/BUILD
+--- a/src/core/BUILD	2023-09-13 13:33:23
++++ b/src/core/BUILD	2023-12-05 09:20:07
+@@ -271,6 +271,8 @@
+         "absl/time",
+         "absl/types:optional",
+         "upb_lib",
++        "upb_base",
++        "upb_mem",
      ],
- )
- 
-@@ -1842,6 +1850,7 @@
+     language = "c++",
      deps = [
-         "gpr",
-         "grpc_trace",
-+        "grpc_public_hdrs",
+@@ -3125,6 +3127,7 @@
+         "//:exec_ctx",
+         "//:gpr",
+         "//:tsi_base",
++        "//:grpc_public_hdrs",
      ],
  )
- 
-@@ -2075,6 +2084,7 @@
-     external_deps = [
-         "protobuf_headers",
-         "protobuf",
-+        "absl/status",
+
+@@ -3993,6 +3996,8 @@
+         "absl/types:optional",
+         "absl/types:variant",
+         "upb_lib",
++        "upb_mem",
++        "upb_base",
      ],
      language = "c++",
-     public_hdrs = [
-@@ -2738,6 +2748,7 @@
-     external_deps = [
-         "absl/strings",
+     deps = [
+@@ -4097,6 +4102,7 @@
          "absl/strings:str_format",
-+        "absl/types:optional",
+         "absl/types:optional",
+         "upb_lib",
++        "upb_base",
+     ],
+     language = "c++",
+     deps = [
+@@ -4220,6 +4226,8 @@
+         "re2",
+         "upb_reflection",
+         "upb_collections_lib",
++        "upb_base",
++        "upb_mem",
      ],
+     language = "c++",
      tags = ["nofixdeps"],
-     visibility = ["@grpc:iomgr_buffer_list"],
-@@ -3647,6 +3658,7 @@
-         "protobuf_struct_upb",
-         "protobuf_timestamp_upb",
-         "ref_counted_ptr",
-+        ":server_address",
-         "uri_parser",
-         "work_serializer",
-         "//src/core:default_event_engine",
-diff -u -Nur a/src/core/BUILD b/src/core/BUILD
---- a/src/core/BUILD	2023-09-13 20:33:23.000000000 +0000
-+++ b/src/core/BUILD	2023-10-04 01:16:42.015957934 +0000
-@@ -3125,6 +3125,7 @@
-         "//:exec_ctx",
-         "//:gpr",
-         "//:tsi_base",
-+        "//:grpc_public_hdrs",
+@@ -4696,6 +4704,8 @@
+         "absl/strings",
+         "absl/types:optional",
+         "upb_lib",
++        "upb_base",
++        "upb_mem",
      ],
- )
- 
+     language = "c++",
+     deps = [
+@@ -5187,6 +5197,8 @@
+         "absl/strings",
+         "absl/types:optional",
+         "upb_lib",
++        "upb_base",
++        "upb_mem",
+     ],
+     language = "c++",
+     deps = [

third_party/com_github_grpc_grpc/workspace.bzl

@@ -12,10 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load(
-    "//third_party:repo.bzl",
-    "third_party_http_archive",
-)
+load("//third_party:repo.bzl", "third_party_http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
 # NOTE: Switch back to a tagged release with darwin-arm64
@@ -36,7 +33,7 @@ def repo():
         ],
         patch_args = ["-p1"],
         repo_mapping = {
-            "@upb": "@com_google_protobuf_upb",
+            "@upb": "@com_google_protobuf",
             "@com_googlesource_code_re2": "@com_google_re2",
             "@com_github_google_benchmark": "@com_google_benchmark",
             "@io_bazel_rules_go": "@local_proto_mirror",
@@ -96,13 +93,15 @@ GRPC_NATIVE_BINDINGS = {
     "protocol_compiler": "@com_google_protobuf//:protoc",
 
     # upb mappings.
-    "upb_json_lib": "@com_google_protobuf_upb//:json",
-    "upb_lib": "@com_google_protobuf_upb//:upb",
-    "upb_lib_descriptor": "@com_google_protobuf_upb//:cmake_descriptor_upb",
-    "upb_lib_descriptor_reflection": "@com_google_protobuf_upb//:cmake_descriptor_upbdefs",
-    "upb_reflection": "@com_google_protobuf_upb//:reflection",
-    "upb_textformat_lib": "@com_google_protobuf_upb//:textformat",
-    "upb_collections_lib": "@com_google_protobuf_upb//:collections",
+    "upb_json_lib": "@com_google_protobuf//upb:json",
+    "upb_lib": "@com_google_protobuf//upb:upb",
+    "upb_lib_descriptor": "@com_google_protobuf//upb:cmake_descriptor_upb",
+    "upb_lib_descriptor_reflection": "@com_google_protobuf//upb:cmake_descriptor_upbdefs",
+    "upb_reflection": "@com_google_protobuf//upb:reflection",
+    "upb_textformat_lib": "@com_google_protobuf//upb:text",
+    "upb_collections_lib": "@com_google_protobuf//upb:collections",
+    "upb_base": "@com_google_protobuf//upb:base",
+    "upb_mem": "@com_google_protobuf//upb:mem",
 
     # These exist to be used by grpc_build_system.bzl
     "benchmark": "@com_google_benchmark//:benchmark",

third_party/com_google_absl/patches/mingw.diff

@@ -0,0 +1,61 @@
+From 2f77684e8dc473a48dbc19167ffe69c40ce8ada4 Mon Sep 17 00:00:00 2001
+From: Derek Mauro <[email protected]>
+Date: Wed, 8 Nov 2023 09:55:31 -0800
+Subject: [PATCH] Avoid using both Win32Waiter and PthreadWaiter on MinGW, and
+ use StdcppWaiter instead.
+
+There are various flavors of MinGW, some of which support pthread,
+and some of which support Win32. Instead of figuring out which
+platform is being used, just use the generic implementation.
+
+PiperOrigin-RevId: 580565507
+Change-Id: Ia85fd7496f1e6ebdeadb95202f0039e844826118
+---
+ absl/synchronization/internal/pthread_waiter.h | 4 ++--
+ absl/synchronization/internal/win32_waiter.h   | 6 ++++--
+ 2 files changed, 6 insertions(+), 4 deletions(-)
+
+diff --git a/absl/synchronization/internal/pthread_waiter.h b/absl/synchronization/internal/pthread_waiter.h
+index 206aefa47c0..23db76ad442 100644
+--- a/absl/synchronization/internal/pthread_waiter.h
++++ b/absl/synchronization/internal/pthread_waiter.h
+@@ -16,7 +16,7 @@
+ #ifndef ABSL_SYNCHRONIZATION_INTERNAL_PTHREAD_WAITER_H_
+ #define ABSL_SYNCHRONIZATION_INTERNAL_PTHREAD_WAITER_H_
+ 
+-#ifndef _WIN32
++#if !defined(_WIN32) && !defined(__MINGW32__)
+ #include <pthread.h>
+ 
+ #include "absl/base/config.h"
+@@ -55,6 +55,6 @@ class PthreadWaiter : public WaiterCrtp<PthreadWaiter> {
+ ABSL_NAMESPACE_END
+ }  // namespace absl
+ 
+-#endif  // ndef _WIN32
++#endif  // !defined(_WIN32) && !defined(__MINGW32__)
+ 
+ #endif  // ABSL_SYNCHRONIZATION_INTERNAL_PTHREAD_WAITER_H_
+diff --git a/absl/synchronization/internal/win32_waiter.h b/absl/synchronization/internal/win32_waiter.h
+index 87eb617cbd1..fdab264e8cb 100644
+--- a/absl/synchronization/internal/win32_waiter.h
++++ b/absl/synchronization/internal/win32_waiter.h
+@@ -20,7 +20,8 @@
+ #include <sdkddkver.h>
+ #endif
+ 
+-#if defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA
++#if defined(_WIN32) && !defined(__MINGW32__) && \
++    _WIN32_WINNT >= _WIN32_WINNT_VISTA
+ 
+ #include "absl/base/config.h"
+ #include "absl/synchronization/internal/kernel_timeout.h"
+@@ -65,6 +66,7 @@ class Win32Waiter : public WaiterCrtp<Win32Waiter> {
+ ABSL_NAMESPACE_END
+ }  // namespace absl
+ 
+-#endif  // defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA
++#endif  // defined(_WIN32) && !defined(__MINGW32__) &&
++        // _WIN32_WINNT >= _WIN32_WINNT_VISTA
+ 
+ #endif  // ABSL_SYNCHRONIZATION_INTERNAL_WIN32_WAITER_H_