Skip to content

Commit d3735bc

Browse files
mkruskal-googlecopybara-github
mkruskal-google
authored and
copybara-github
committed
Support fetching CMake dependencies via -Dprotobuf_FETCH_DEPENDENCIES=ON
This ports upb's WORKSPACE scraping logic to protobuf, and allows us to dynamically fetch our dependencies at the exact same pinned version as in Bazel via protobuf_FETCH_DEPENDENCIES=ON. This is mostly for development purposes, and is preferable to git submodules. In a later cl we will flip the default behavior to "package" #test-continuous PiperOrigin-RevId: 686265348
1 parent 2b0a414 commit d3735bc

14 files changed

+266
-603
lines changed

‎.github/workflows/test_cpp.yml

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,6 @@ jobs:
107107
uses: protocolbuffers/protobuf-ci/checkout@v3
108108
with:
109109
ref: ${{ inputs.safe-checkout }}
110-
submodules: recursive
111110
- name: Cross compile protoc for ${{ matrix.arch }}
112111
id: cross-compile
113112
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3
@@ -131,7 +130,7 @@ jobs:
131130
sccache -z;
132131
cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
133132
-Dprotobuf_BUILD_LIBUPB=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14
134-
-Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }};
133+
-Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }};
135134
cmake --build . --parallel 20;
136135
ctest --parallel 20;
137136
sccache -s"
@@ -141,18 +140,19 @@ jobs:
141140
fail-fast: false # Don't cancel all jobs if one fails.
142141
matrix:
143142
include:
144-
- flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14
143+
- flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
145144
- name: Ninja
146-
flags: -G Ninja -DCMAKE_CXX_STANDARD=14
145+
flags: -G Ninja -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
147146
continuous-only: true
148147
- name: Shared
149-
flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14
148+
flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
150149
continuous-only: true
151150
- name: C++17
152-
flags: -DCMAKE_CXX_STANDARD=17
153-
# TODO Re-enable this.
154-
#- name: C++20
155-
# flags: -DCMAKE_CXX_STANDARD=20
151+
flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
152+
- name: C++20
153+
flags: -DCMAKE_CXX_STANDARD=20 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
154+
- name: Fetch
155+
flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_FETCH_DEPENDENCIES=ON
156156

157157
name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux CMake ${{ matrix.name}}
158158
runs-on: ubuntu-latest
@@ -174,45 +174,59 @@ jobs:
174174
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
175175
uses: protocolbuffers/protobuf-ci/docker@v3
176176
with:
177-
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
177+
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
178178
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
179179
command: >-
180180
/test.sh ${{ matrix.flags}} ${{ env.SCCACHE_CMAKE_FLAGS }}
181-
-Dprotobuf_BUILD_TESTS=ON -Dprotobuf_USE_EXTERNAL_GTEST=ON
182-
-Dprotobuf_ABSL_PROVIDER=package
181+
-Dprotobuf_BUILD_TESTS=ON ${{ matrix.package_flags }}
183182
184183
linux-cmake-install:
185-
name: Linux CMake Install
184+
strategy:
185+
fail-fast: false # Don't cancel all jobs if one fails.
186+
matrix:
187+
type: [package, fetch]
188+
include:
189+
# Set defaults
190+
- type: package
191+
name: Install
192+
flags: -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
193+
- type: fetch
194+
name: Install (Fetch)
195+
flags: -Dprotobuf_FETCH_DEPENDENCIES=ON
196+
continuous-only: true
197+
name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }}Linux CMake ${{ matrix.name }})
186198
runs-on: ubuntu-latest
187199
steps:
188200
- name: Checkout pending changes
189201
uses: protocolbuffers/protobuf-ci/checkout@v3
202+
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
190203
with:
191204
ref: ${{ inputs.safe-checkout }}
192-
submodules: recursive
193205

194206
- name: Setup sccache
195207
uses: protocolbuffers/protobuf-ci/sccache@v3
208+
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
196209
with:
197210
cache-prefix: linux-cmake-install
198211
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
199212

200213
- name: Run tests
201214
uses: protocolbuffers/protobuf-ci/docker@v3
215+
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
202216
with:
203-
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
217+
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
204218
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
205219
command: >-
206220
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}
207-
-Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package
221+
${{ matrix.flags }}
208222
-Dprotobuf_BUILD_SHARED_LIBS=ON \&\&
209223
/test.sh
210224
${{ env.SCCACHE_CMAKE_FLAGS }}
211225
-Dprotobuf_REMOVE_INSTALLED_HEADERS=ON
212226
-Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
213227
-Dprotobuf_BUILD_CONFORMANCE=ON
214228
-DCMAKE_CXX_STANDARD=14
215-
-Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package
229+
${{ matrix.flags }}
216230
217231
# This test should always be skipped on presubmit
218232
linux-cmake-examples:
@@ -236,7 +250,7 @@ jobs:
236250
if: ${{ inputs.continuous-run }}
237251
uses: protocolbuffers/protobuf-ci/docker@v3
238252
with:
239-
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
253+
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
240254
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
241255
command: >-
242256
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}
@@ -268,7 +282,6 @@ jobs:
268282
uses: protocolbuffers/protobuf-ci/checkout@v3
269283
with:
270284
ref: ${{ inputs.safe-checkout }}
271-
submodules: recursive
272285

273286
- name: Setup sccache
274287
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
@@ -288,7 +301,7 @@ jobs:
288301
-c 'set -ex;
289302
cd /workspace;
290303
sccache -z;
291-
cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }};
304+
cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_FETCH_DEPENDENCIES=ON;
292305
cmake --build . --parallel 20;
293306
ctest --verbose --parallel 20;
294307
sccache -s'
@@ -312,7 +325,7 @@ jobs:
312325
- name: Run tests
313326
uses: protocolbuffers/protobuf-ci/docker@v3
314327
with:
315-
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
328+
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
316329
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
317330
command: >-
318331
/test.sh ${{ env.SCCACHE_CMAKE_FLAGS }}
@@ -326,7 +339,6 @@ jobs:
326339
uses: protocolbuffers/protobuf-ci/checkout@v3
327340
with:
328341
ref: ${{ inputs.safe-checkout }}
329-
submodules: recursive
330342

331343
- name: Setup sccache
332344
uses: protocolbuffers/protobuf-ci/sccache@v3
@@ -337,14 +349,14 @@ jobs:
337349
- name: Run tests
338350
uses: protocolbuffers/protobuf-ci/docker@v3
339351
with:
340-
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:429f924aec315704b4233adcbe4b29006116f27769db98acd176b9eb69c31299
352+
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:56548bef786201330017eae685cc3d2fdb564fd2ca3b88e30e28d84572e4c5dd
341353
platform: linux/386
342354
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
343355
command: >-
344356
/bin/bash -cex '
345357
cd /workspace;
346358
sccache -z;
347-
cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }};
359+
cmake . -DCMAKE_CXX_STANDARD=14 -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }};
348360
cmake --build . --parallel 20;
349361
ctest --verbose --parallel 20;
350362
sccache -s'
@@ -460,7 +472,6 @@ jobs:
460472
uses: protocolbuffers/protobuf-ci/checkout@v3
461473
with:
462474
ref: ${{ inputs.safe-checkout }}
463-
submodules: recursive
464475

465476
- name: Setup MSVC
466477
if: ${{ runner.os == 'Windows' && (!matrix.continuous-only || inputs.continuous-run) }}
@@ -499,7 +510,7 @@ jobs:
499510
uses: protocolbuffers/protobuf-ci/bash@v3
500511
with:
501512
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
502-
command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
513+
command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON
503514
- name: Build for install
504515
if: ${{ matrix.install-flags && (!matrix.continuous-only || inputs.continuous-run) }}
505516
shell: bash
@@ -522,7 +533,7 @@ jobs:
522533
uses: protocolbuffers/protobuf-ci/bash@v3
523534
with:
524535
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
525-
command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
536+
command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON
526537

527538
- name: Build
528539
if: ${{ !matrix.continuous-only || inputs.continuous-run }}

‎BUILD.bazel

Lines changed: 5 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ license(
2424
license_text = ":LICENSE",
2525
)
2626

27+
exports_files(
28+
["MODULE.bazel"],
29+
visibility = ["//cmake:__pkg__"],
30+
)
31+
2732
################################################################################
2833
# Well Known Types Proto Library Rules
2934
#
@@ -642,53 +647,3 @@ filegroup(
642647
srcs = glob(["**/*.bzl"]),
643648
visibility = ["//visibility:public"],
644649
)
645-
646-
################################################################################
647-
# Packaging rules
648-
################################################################################
649-
650-
# Files included in all source distributions
651-
pkg_files(
652-
name = "common_dist_files",
653-
srcs = glob(
654-
[
655-
"*.bzl",
656-
"cmake/*.cmake",
657-
"cmake/*.in",
658-
"editors/*",
659-
],
660-
allow_empty = True,
661-
) + [
662-
"BUILD.bazel",
663-
"CMakeLists.txt",
664-
"CONTRIBUTORS.txt",
665-
"LICENSE",
666-
"README.md",
667-
"WORKSPACE",
668-
"cmake/README.md",
669-
"generate_descriptor_proto.sh",
670-
"maven_install.json",
671-
"//third_party:BUILD.bazel",
672-
"//third_party:zlib.BUILD",
673-
],
674-
strip_prefix = strip_prefix.from_root(""),
675-
visibility = ["//pkg:__pkg__"],
676-
)
677-
678-
# Additional files for C#
679-
pkg_files(
680-
name = "csharp_dist_files",
681-
srcs = [
682-
"global.json",
683-
],
684-
visibility = ["//pkg:__pkg__"],
685-
)
686-
687-
# Additional files for ObjC
688-
pkg_files(
689-
name = "objectivec_dist_files",
690-
srcs = [
691-
"Protobuf.podspec",
692-
],
693-
visibility = ["//pkg:__pkg__"],
694-
)

‎CMakeLists.txt

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -153,14 +153,6 @@ file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/cmaketest.map)
153153

154154
find_package(Threads REQUIRED)
155155

156-
# We can install dependencies from submodules if we're running
157-
# CMake v3.13 or newer.
158-
if(CMAKE_VERSION VERSION_LESS 3.13)
159-
set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE OFF)
160-
else()
161-
set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE ON)
162-
endif()
163-
164156
set(_protobuf_FIND_ZLIB)
165157
if (protobuf_WITH_ZLIB)
166158
find_package(ZLIB)
@@ -289,11 +281,13 @@ include_directories(
289281
${protobuf_BINARY_DIR}/src
290282
${protobuf_SOURCE_DIR}/src)
291283

292-
set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library")
293-
set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package")
284+
set(protobuf_FETCH_DEPENDENCIES OFF CACHE BOOL "Download dependencies from GitHub. If this option is not set, the dependency must be available locally, either as a sub-module or an installed package.")
285+
286+
set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library. `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub")
287+
set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package" "fetch")
294288

295-
set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library")
296-
set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package")
289+
set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library. `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub")
290+
set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package" "fetch")
297291

298292
if (protobuf_BUILD_TESTS)
299293
include(${protobuf_SOURCE_DIR}/cmake/gtest.cmake)

‎MODULE.bazel

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ module(
1212
# Bzlmod follows MVS:
1313
# https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution
1414
# Thus the highest version in their module graph is resolved.
15-
bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl")
15+
bazel_dep(name = "abseil-cpp", version = "20230802.1", repo_name = "com_google_absl")
1616
bazel_dep(name = "bazel_skylib", version = "1.7.0")
1717
bazel_dep(name = "jsoncpp", version = "1.9.5")
1818
bazel_dep(name = "rules_cc", version = "0.0.13")
@@ -27,10 +27,9 @@ bazel_dep(name = "rules_rust", version = "0.51.0")
2727
bazel_dep(name = "platforms", version = "0.0.8")
2828
bazel_dep(name = "zlib", version = "1.3.1")
2929
bazel_dep(name = "bazel_features", version = "1.17.0", repo_name = "proto_bazel_features")
30-
3130
bazel_dep(
3231
name = "rules_shell",
33-
version = "0.2.0"
32+
version = "0.2.0",
3433
)
3534

3635
# Proto toolchains

‎cmake/BUILD.bazel

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
load("@rules_python//python:defs.bzl", "py_binary")
2+
load("//upb/cmake:build_defs.bzl", "staleness_test")
3+
4+
py_binary(
5+
name = "dependencies_generator",
6+
srcs = ["dependencies_generator.py"],
7+
)
8+
9+
genrule(
10+
name = "generate_dependencies",
11+
srcs = ["//:MODULE.bazel"],
12+
outs = ["generated-in/dependencies.cmake"],
13+
cmd = "$(location :dependencies_generator) " +
14+
"$(location //:MODULE.bazel) $@",
15+
tools = [":dependencies_generator"],
16+
)
17+
18+
staleness_test(
19+
name = "test_dependencies_staleness",
20+
outs = [
21+
"dependencies.cmake",
22+
],
23+
generated_pattern = "generated-in/%s",
24+
tags = ["manual"],
25+
)

‎cmake/abseil-cpp.cmake

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,20 @@ endif()
1313
if(TARGET absl::strings)
1414
# If Abseil is included already, skip including it.
1515
# (https://github.com/protocolbuffers/protobuf/issues/10435)
16+
elseif (protobuf_FETCH_DEPENDENCIES OR protobuf_ABSL_PROVIDER STREQUAL "fetch")
17+
include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake)
18+
include(FetchContent)
19+
FetchContent_Declare(
20+
absl
21+
GIT_REPOSITORY "https://github.com/abseil/abseil-cpp.git"
22+
GIT_TAG "${abseil-cpp-version}"
23+
)
24+
if(protobuf_INSTALL)
25+
# When protobuf_INSTALL is enabled and Abseil will be built as a module,
26+
# Abseil will be installed along with protobuf for convenience.
27+
set(ABSL_ENABLE_INSTALL ON)
28+
endif()
29+
FetchContent_MakeAvailable(absl)
1630
elseif(protobuf_ABSL_PROVIDER STREQUAL "module")
1731
if(NOT ABSL_ROOT_DIR)
1832
set(ABSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp)
@@ -27,10 +41,6 @@ elseif(protobuf_ABSL_PROVIDER STREQUAL "module")
2741
else()
2842
message(WARNING "protobuf_ABSL_PROVIDER is \"module\" but ABSL_ROOT_DIR is wrong")
2943
endif()
30-
if(protobuf_INSTALL AND NOT _protobuf_INSTALL_SUPPORTED_FROM_MODULE)
31-
message(WARNING "protobuf_INSTALL will be forced to FALSE because protobuf_ABSL_PROVIDER is \"module\" and CMake version (${CMAKE_VERSION}) is less than 3.13.")
32-
set(protobuf_INSTALL FALSE)
33-
endif()
3444
elseif(protobuf_ABSL_PROVIDER STREQUAL "package")
3545
# Use "CONFIG" as there is no built-in cmake module for absl.
3646
find_package(absl REQUIRED CONFIG)

0 commit comments

Comments
 (0)