Reland "Update to protobuf 3.20.0"
This reverts commit d8c5cddc77e0e0aed839a336cefbb9aa914ec7d7.
With the new release, we can retire some patches:
- 0003-remove-static-initializers.patch: Upstream protobuf no longer has
variables like Status::OK but constructor functions like OkStatus(),
so there is no more static initializer.
https://github.com/protocolbuffers/protobuf/commit/3a7bd9c236fda465c16b90567f6b8706915da6cb
- 0004-fix-integer-types-and-shared-library-exports.patch: Half of this
patch is no longer needed. Upstream protobuf no longer has
PROTOBUF_ULONGLONG, etc.
- 0009-uninline-arenastring.patch: No longer applies, function in
question seems to be gone.
- 0023-fix-delimited-message-parsing.patch: Merged upstream
- 0024-fix-thread-priority-on-protobuf-initialization.patch: No longer
needed. Protobuf now uses constant initialization. See cl/351672714
We can also remove the instructions to copy in third_party/six.
protobuf no longer uses it.
Also adds some new patches to fix a static initializer and other
compile failures. See go/protobuf-init-chrome (internal) for details
on that saga. In addition adds a patch to clean up descriptor allocation
code that incorrectly tried to work around a CFI check.
This CL should probably followed up with an update to 3.20.1, which
was released in the time it took to put this together. However,
we cannot update to 21.1 (new version scheme) for now because
upstream moved the JavaScript compiler to a new repo and haven't yet
made a release from that repo.
[email protected] was the one that prepared this roll, and this
reland CL just adds a minor workaround for the CFI issue.
Original CL: https://crrev.com/c/3594212
Revert CL: https://crrev.com/c/3718597
Bug: 1294200
Change-Id: If5fdc6e716684f914acf93dce0f2b81f94e3d89b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3733827
Reviewed-by: Peter Kasting <[email protected]>
Reviewed-by: Robbie Iannucci <[email protected]>
Commit-Queue: Peter Kasting <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1019285}
diff --git a/third_party/protobuf/tests.sh b/third_party/protobuf/tests.sh
index 5113e4a..73460fd 100755
--- a/third_party/protobuf/tests.sh
+++ b/third_party/protobuf/tests.sh
@@ -62,7 +62,7 @@
# List all files that should be included in the distribution package.
git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\
- grep -v ".gitignore" | grep -v "java/compatibility_tests" | grep -v "java/lite/proguard.pgcfg" |\
+ grep -v ".gitignore" | grep -v "java/lite/proguard.pgcfg" |\
grep -v "python/compatibility_tests" | grep -v "python/docs" | grep -v "python/.repo-metadata.json" |\
grep -v "python/protobuf_distutils" | grep -v "csharp/compatibility_tests" > dist.lst
# Unzip the dist tar file.
@@ -88,6 +88,18 @@
}
build_dist_install() {
+ # Create a symlink pointing to python2 and put it at the beginning of $PATH.
+ # This is necessary because the googletest build system involves a Python
+ # script that is not compatible with Python 3. More recent googletest
+ # versions have fixed this, but they have also removed the autotools build
+ # system support that we rely on. This is a temporary workaround to keep the
+ # googletest build working when the default python binary is Python 3.
+ mkdir tmp || true
+ pushd tmp
+ ln -s /usr/bin/python2 ./python
+ popd
+ PATH=$PWD/tmp:$PATH
+
# Initialize any submodules.
git submodule update --init --recursive
./autogen.sh
@@ -104,16 +116,16 @@
# Try to install Java
pushd java
- use_java jdk7
+ use_java jdk11
$MVN install
popd
# Try to install Python
- virtualenv --no-site-packages venv
+ python3 -m venv venv
source venv/bin/activate
pushd python
- python setup.py clean build sdist
- pip install dist/protobuf-*.tar.gz
+ python3 setup.py clean build sdist
+ pip3 install dist/protobuf-*.tar.gz
popd
deactivate
rm -rf python/venv
@@ -177,6 +189,10 @@
use_java() {
version=$1
case "$version" in
+ jdk11)
+ export PATH=/usr/lib/jvm/java-11-openjdk-amd64/bin:$PATH
+ export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+ ;;
jdk8)
export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin:$PATH
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
@@ -192,7 +208,7 @@
esac
MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository
- MVN="$MVN -e -X -Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY"
+ MVN="$MVN -e --quiet -Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY"
which java
java -version
@@ -202,13 +218,26 @@
# --batch-mode suppresses download progress output that spams the logs.
MVN="mvn --batch-mode"
-build_java() {
+internal_build_java() {
version=$1
dir=java_$version
# Java build needs `protoc`.
internal_build_cpp
cp -r java $dir
- cd $dir && $MVN clean && $MVN test
+ cd $dir && $MVN clean
+ # Skip tests here - callers will decide what tests they want to run
+ $MVN install -Dmaven.test.skip=true
+}
+
+build_java() {
+ version=$1
+ internal_build_java $version
+ # Skip the Kotlin tests on Oracle 7
+ if [ "$version" == "oracle7" ]; then
+ $MVN test -pl bom,lite,core,util
+ else
+ $MVN test
+ fi
cd ../..
}
@@ -220,8 +249,13 @@
# This local installation avoids the problem caused by a new version not yet in Maven Central
cd java/bom && $MVN install
cd ../..
- cd java && $MVN test && $MVN install
- cd util && $MVN package assembly:single
+ cd java/core && $MVN test && $MVN install
+ cd ../lite && $MVN test && $MVN install
+ cd ../util && $MVN test && $MVN install && $MVN package assembly:single
+ if [ "$version" == "jdk8" ]; then
+ cd ../kotlin && $MVN test && $MVN install
+ cd ../kotlin-lite && $MVN test && $MVN install
+ fi
cd ../..
cd conformance && make test_java && cd ..
}
@@ -234,22 +268,11 @@
use_java oracle7
build_java oracle7
}
-build_java_compatibility() {
- use_java jdk7
- internal_build_cpp
- # Use the unit-tests extracted from 2.5.0 to test the compatibility between
- # 3.0.0-beta-4 and the current version.
- cd java/compatibility_tests/v2.5.0
- ./test.sh 3.0.0-beta-4
-
- # Test the last released and current version.
- ./test.sh $LAST_RELEASED
-}
build_java_linkage_monitor() {
# Linkage Monitor checks compatibility with other Google libraries
# https://github.com/GoogleCloudPlatform/cloud-opensource-java/tree/master/linkage-monitor
- use_java jdk8
+ use_java jdk11
internal_build_cpp
# Linkage Monitor uses $HOME/.m2 local repository
@@ -311,12 +334,7 @@
build_python() {
internal_build_cpp
cd python
- if [ $(uname -s) == "Linux" ]; then
- envlist=py\{27,33,34,35,36\}-python
- else
- envlist=py\{27,36\}-python
- fi
- python -m tox -e $envlist
+ tox --skip-missing-interpreters
cd ..
}
@@ -324,30 +342,10 @@
internal_build_cpp
cd python
envlist=$1
- python -m tox -e $envlist
+ tox -e $envlist
cd ..
}
-build_python27() {
- build_python_version py27-python
-}
-
-build_python33() {
- build_python_version py33-python
-}
-
-build_python34() {
- build_python_version py34-python
-}
-
-build_python35() {
- build_python_version py35-python
-}
-
-build_python36() {
- build_python_version py36-python
-}
-
build_python37() {
build_python_version py37-python
}
@@ -356,17 +354,20 @@
build_python_version py38-python
}
+build_python39() {
+ build_python_version py39-python
+}
+
+build_python310() {
+ build_python_version py310-python
+}
+
build_python_cpp() {
internal_build_cpp
export LD_LIBRARY_PATH=../src/.libs # for Linux
export DYLD_LIBRARY_PATH=../src/.libs # for OS X
cd python
- if [ $(uname -s) == "Linux" ]; then
- envlist=py\{27,33,34,35,36\}-cpp
- else
- envlist=py\{27,36\}-cpp
- fi
- tox -e $envlist
+ tox --skip-missing-interpreters
cd ..
}
@@ -380,26 +381,6 @@
cd ..
}
-build_python27_cpp() {
- build_python_cpp_version py27-cpp
-}
-
-build_python33_cpp() {
- build_python_cpp_version py33-cpp
-}
-
-build_python34_cpp() {
- build_python_cpp_version py34-cpp
-}
-
-build_python35_cpp() {
- build_python_cpp_version py35-cpp
-}
-
-build_python36_cpp() {
- build_python_cpp_version py36-cpp
-}
-
build_python37_cpp() {
build_python_cpp_version py37-cpp
}
@@ -408,19 +389,15 @@
build_python_cpp_version py38-cpp
}
-build_python_compatibility() {
- internal_build_cpp
- # Use the unit-tests extracted from 2.5.0 to test the compatibility.
- cd python/compatibility_tests/v2.5.0
- # Test between 2.5.0 and the current version.
- ./test.sh 2.5.0
- # Test between 3.0.0-beta-1 and the current version.
- ./test.sh 3.0.0-beta-1
-
- # Test between last released and current version.
- ./test.sh $LAST_RELEASED
+build_python39_cpp() {
+ build_python_cpp_version py39-cpp
}
+build_python310_cpp() {
+ build_python_cpp_version py310-cpp
+}
+
+
build_ruby23() {
internal_build_cpp # For conformance tests.
cd ruby && bash travis-test.sh ruby-2.3.8 && cd ..
@@ -441,6 +418,26 @@
internal_build_cpp # For conformance tests.
cd ruby && bash travis-test.sh ruby-2.7.0 && cd ..
}
+build_ruby30() {
+ internal_build_cpp # For conformance tests.
+ cd ruby && bash travis-test.sh ruby-3.0.2 && cd ..
+}
+build_ruby31() {
+ internal_build_cpp # For conformance tests.
+ cd ruby && bash travis-test.sh ruby-3.1.0 && cd ..
+}
+
+build_jruby92() {
+ internal_build_cpp # For conformance tests.
+ internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes
+ cd ruby && bash travis-test.sh jruby-9.2.20.1 && cd ..
+}
+
+build_jruby93() {
+ internal_build_cpp # For conformance tests.
+ internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes
+ cd ruby && bash travis-test.sh jruby-9.3.3.0 && cd ..
+}
build_javascript() {
internal_build_cpp
@@ -461,59 +458,32 @@
internal_build_cpp
}
-use_php_zts() {
- VERSION=$1
- export PATH=/usr/local/php-${VERSION}-zts/bin:$PATH
- internal_build_cpp
-}
-
-build_php7.0() {
- use_php 7.0
+build_php() {
+ use_php $1
pushd php
rm -rf vendor
+ php -v
+ php -m
composer update
composer test
popd
(cd conformance && make test_php)
}
-build_php7.0_c() {
- IS_64BIT=$1
- use_php 7.0
- php/tests/test.sh
- pushd conformance
- if [ "$IS_64BIT" = "true" ]
- then
- make test_php_c
- else
- make test_php_c_32
- fi
- popd
-}
-
-build_php7.0_mixed() {
- use_php 7.0
+test_php_c() {
pushd php
rm -rf vendor
+ php -v
+ php -m
composer update
- tests/compile_extension.sh
- tests/generate_protos.sh
- php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
+ composer test_c
popd
+ (cd conformance && make test_php_c)
}
-build_php7.0_zts_c() {
- IS_64BIT=$1
- use_php_zts 7.0
- php/tests/test.sh
- pushd conformance
- if [ "$IS_64BIT" = "true" ]
- then
- make test_php_c
- else
- make test_php_c_32
- fi
- popd
+build_php_c() {
+ use_php $1
+ test_php_c
}
build_php7.0_mac() {
@@ -524,14 +494,17 @@
test ! -z "$PHP_FOLDER"
export PATH="$PHP_FOLDER/bin:$PATH"
+ # Install Composer
+ wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer
+ chmod a+x /usr/local/bin/composer
+
# Install valgrind
echo "#! /bin/bash" > valgrind
chmod ug+x valgrind
sudo mv valgrind /usr/local/bin/valgrind
# Test
- php/tests/test.sh
- (cd conformance && make test_php_c)
+ test_php_c
}
build_php7.3_mac() {
@@ -544,14 +517,17 @@
test ! -z "$PHP_FOLDER"
export PATH="$PHP_FOLDER/bin:$PATH"
+ # Install Composer
+ wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer
+ chmod a+x /usr/local/bin/composer
+
# Install valgrind
echo "#! /bin/bash" > valgrind
chmod ug+x valgrind
sudo mv valgrind /usr/local/bin/valgrind
# Test
- php/tests/test.sh
- (cd conformance && make test_php_c)
+ test_php_c
}
build_php_compatibility() {
@@ -564,166 +540,27 @@
php/tests/multirequest.sh
}
-build_php7.1() {
- use_php 7.1
- pushd php
- rm -rf vendor
- composer update
- composer test
- popd
- (cd conformance && make test_php)
-}
-
-build_php7.1_c() {
- IS_64BIT=$1
- use_php 7.1
- php/tests/test.sh
- pushd conformance
- if [ "$IS_64BIT" = "true" ]
- then
- make test_php_c
- else
- make test_php_c_32
- fi
- popd
-}
-
-build_php7.1_mixed() {
- use_php 7.1
- pushd php
- rm -rf vendor
- composer update
- tests/compile_extension.sh
- tests/generate_protos.sh
- php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
- popd
-}
-
-build_php7.1_zts_c() {
- IS_64BIT=$1
- use_php_zts 7.1
- php/tests/test.sh
- pushd conformance
- if [ "$IS_64BIT" = "true" ]
- then
- make test_php_c
- else
- make test_php_c_32
- fi
- popd
-}
-
-build_php7.4() {
- use_php 7.4
- pushd php
- rm -rf vendor
- composer update
- composer test
- popd
- (cd conformance && make test_php)
-}
-
-build_php7.4_c() {
- IS_64BIT=$1
- use_php 7.4
- php/tests/test.sh
- pushd conformance
- if [ "$IS_64BIT" = "true" ]
- then
- make test_php_c
- else
- make test_php_c_32
- fi
- popd
-}
-
-build_php7.4_mixed() {
- use_php 7.4
- pushd php
- rm -rf vendor
- composer update
- tests/compile_extension.sh
- tests/generate_protos.sh
- php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
- popd
-}
-
-build_php7.4_zts_c() {
- IS_64BIT=$1
- use_php_zts 7.4
- php/tests/test.sh
- pushd conformance
- if [ "$IS_64BIT" = "true" ]
- then
- make test_php_c
- else
- make test_php_c_32
- fi
- popd
-}
-
-build_php8.0() {
- use_php 8.0
- pushd php
- rm -rf vendor
- composer update
- composer test
- popd
- (cd conformance && make test_php)
-}
-
-build_php8.0_c() {
- IS_64BIT=$1
- use_php 8.0
- php/tests/test.sh
- pushd conformance
- if [ "$IS_64BIT" = "true" ]
- then
- make test_php_c
- else
- make test_php_c_32
- fi
- popd
-}
-
-build_php8.0_c_64() {
- build_php8.0_c true
-}
-
-build_php8.0_mixed() {
- use_php 8.0
- pushd php
- rm -rf vendor
- composer update
- tests/compile_extension.sh
- tests/generate_protos.sh
- php -dextension=./ext/google/protobuf/modules/protobuf.so ./vendor/bin/phpunit
- popd
-}
-
build_php8.0_all() {
- build_php8.0
- build_php8.0_c_64
- build_php8.0_mixed
+ build_php 8.0
+ build_php 8.1
+ build_php_c 8.0
+ build_php_c 8.1
}
build_php_all_32() {
- build_php7.0
- build_php7.1
- build_php7.4
- build_php7.0_c $1
- build_php7.1_c $1
- build_php7.4_c $1
- build_php7.0_mixed
- build_php7.1_mixed
- build_php7.4_mixed
- build_php7.0_zts_c $1
- build_php7.1_zts_c $1
- build_php7.4_zts_c $1
+ build_php 7.0
+ build_php 7.1
+ build_php 7.4
+ build_php_c 7.0
+ build_php_c 7.1
+ build_php_c 7.4
+ build_php_c 7.1-zts
+ build_php_c 7.2-zts
+ build_php_c 7.5-zts
}
build_php_all() {
- build_php_all_32 true
+ build_php_all_32
build_php_multirequest
build_php_compatibility
}
@@ -742,7 +579,6 @@
csharp |
java_jdk7 |
java_oracle7 |
- java_compatibility |
java_linkage_monitor |
objectivec_ios |
objectivec_ios_debug |
@@ -760,14 +596,15 @@
ruby25 |
ruby26 |
ruby27 |
- jruby |
+ ruby30 |
+ ruby31 |
+ jruby92 |
+ jruby93 |
ruby_all |
- php7.0 |
- php7.0_c |
- php_compatibility |
- php7.1 |
- php7.1_c |
php_all |
+ php_all_32 |
+ php7.0_mac |
+ php7.3_mac |
dist_install |
benchmark)
"
