[Reland] third_party/protobuf: Update to 040f56e
Rebase protobuf to current HEAD to pull the 4 following changes:
* 126082c Add std:: namespace prefix to set and map
* 4c5d3ed Fix integer overflow in FastUInt32ToBufferLeft
* 4587a3f [arm/gcc] Don't rely on KUSER_HELPERS feature for atomics
* a5a2c1d generic atomicops: Use strong compare_exchange
This change has been made following steps described in
third_party/protobuf/README.chromium file, please refer to it
for further information.
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.linux:linux_chromium_cfi_rel_ng;master.tryserver.chromium.linux:linux_chromium_archive_rel_ng;master.tryserver.chromium.linux:linux_chromium_ubsan_rel_ng
BUG=599051, 673488
[email protected]
[email protected]
Change-Id: I3231744158280b49e20f112812617d7685ea994c
Reviewed-on: https://chromium-review.googlesource.com/588107
Reviewed-by: Thomas Anderson <[email protected]>
Reviewed-by: Peter Kasting <[email protected]>
Commit-Queue: Thomas Anderson <[email protected]>
Cr-Commit-Position: refs/heads/master@{#490005}diff --git a/third_party/protobuf/tests.sh b/third_party/protobuf/tests.sh
index 6a9439a5..c6eda93 100755
--- a/third_party/protobuf/tests.sh
+++ b/third_party/protobuf/tests.sh
@@ -28,7 +28,8 @@
fi
./autogen.sh
- ./configure
+ ./configure CXXFLAGS="-fPIC" # -fPIC is needed for python cpp test.
+ # See python/setup.py for more details
make -j2
}
@@ -37,13 +38,51 @@
make check -j2
cd conformance && make test_cpp && cd ..
- # Verify benchmarking code can build successfully.
- cd benchmarks && make && ./generate-datasets && cd ..
+ # The benchmark code depends on cmake, so test if it is installed before
+ # trying to do the build.
+ # NOTE: The travis macOS images say they have cmake, but the xcode8.1 image
+ # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
+ if [[ $(type cmake 2>/dev/null) ]]; then
+ # Verify benchmarking code can build successfully.
+ git submodule init
+ git submodule update
+ cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
+ cd benchmarks && make && ./generate-datasets && cd ..
+ else
+ echo ""
+ echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed."
+ echo ""
+ fi
}
build_cpp_distcheck() {
./autogen.sh
./configure
+ make dist
+
+ # List all files that should be included in the distribution package.
+ git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|php\|cmake\|examples\)" |\
+ grep -v ".gitignore" | grep -v "java/compatibility_tests" |\
+ grep -v "python/compatibility_tests" | grep -v "csharp/compatibility_tests" > dist.lst
+ # Unzip the dist tar file.
+ DIST=`ls *.tar.gz`
+ tar -xf $DIST
+ cd ${DIST//.tar.gz}
+ # Check if every file exists in the dist tar file.
+ FILES_MISSING=""
+ for FILE in $(<../dist.lst); do
+ if ! file $FILE &>/dev/null; then
+ echo "$FILE is not found!"
+ FILES_MISSING="$FILE $FILES_MISSING"
+ fi
+ done
+ cd ..
+ if [ ! -z "$FILES_MISSING" ]; then
+ echo "Missing files in EXTRA_DIST: $FILES_MISSING"
+ exit 1
+ fi
+
+ # Do the regular dist-check for C++.
make distcheck -j2
}
@@ -54,20 +93,22 @@
internal_build_cpp
NUGET=/usr/local/bin/nuget.exe
- if [ "$TRAVIS" == "true" ]; then
- # Install latest version of Mono
- sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
- echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
- echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list
- sudo apt-get update -qq
- sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit
- wget www.nuget.org/NuGet.exe -O nuget.exe
- NUGET=../../nuget.exe
- fi
+ # Perform "dotnet new" once to get the setup preprocessing out of the
+ # way. That spews a lot of output (including backspaces) into logs
+ # otherwise, and can cause problems. It doesn't matter if this step
+ # is performed multiple times; it's cheap after the first time anyway.
+ # (It also doesn't matter if it's unnecessary, which it will be on some
+ # systems. It's necessary on Jenkins in order to avoid unprintable
+ # characters appearing in the JUnit output.)
+ mkdir dotnettmp
+ (cd dotnettmp; dotnet new > /dev/null)
+ rm -rf dotnettmp
- (cd csharp/src; mono $NUGET restore)
csharp/buildall.sh
cd conformance && make test_csharp && cd ..
+
+ # Run csharp compatibility test between 3.0.0 and the current version.
+ csharp/compatibility_tests/v3.0.0/test.sh 3.0.0
}
build_golang() {
@@ -77,10 +118,12 @@
export PATH="`pwd`/src:$PATH"
# Install Go and the Go protobuf compiler plugin.
- sudo apt-get update -qq
- sudo apt-get install -qq golang
+ on_travis sudo apt-get update -qq
+ on_travis sudo apt-get install -qq golang
+
export GOPATH="$HOME/gocode"
mkdir -p "$GOPATH/src/github.com/google"
+ rm -f "$GOPATH/src/github.com/google/protobuf"
ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf"
export PATH="$GOPATH/bin:$PATH"
go get github.com/golang/protobuf/protoc-gen-go
@@ -91,13 +134,10 @@
use_java() {
version=$1
case "$version" in
- jdk6)
- on_travis sudo apt-get install openjdk-6-jdk
- export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH
- ;;
jdk7)
on_travis sudo apt-get install openjdk-7-jdk
export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
+ export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
;;
oracle7)
if [ "$TRAVIS" == "true" ]; then
@@ -108,6 +148,7 @@
yes | sudo apt-get install oracle-java7-installer
fi;
export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH
+ export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
;;
esac
@@ -118,6 +159,7 @@
which java
java -version
+ $MVN -version
}
# --batch-mode supresses download progress output that spams the logs.
@@ -150,10 +192,6 @@
cd javanano && $MVN test && cd ..
}
-build_java_jdk6() {
- use_java jdk6
- build_java jdk6
-}
build_java_jdk7() {
use_java jdk7
build_java_with_conformance_tests
@@ -162,11 +200,15 @@
use_java oracle7
build_java oracle7
}
-
-build_javanano_jdk6() {
- use_java jdk6
- build_javanano
+build_java_compatibility() {
+ use_java jdk7
+ internal_build_cpp
+ # Use the unit-tests extraced from 2.5.0 to test the compatibilty between
+ # 3.0.0-beta-4 and the current version.
+ cd java/compatibility_tests/v2.5.0
+ ./test.sh 3.0.0-beta-4
}
+
build_javanano_jdk7() {
use_java jdk7
build_javanano
@@ -197,59 +239,36 @@
fi
}
-internal_objectivec_common () {
- # Make sure xctool is up to date. Adapted from
- # http://docs.travis-ci.com/user/osx-ci-environment/
- # We don't use a before_install because we test multiple OSes.
- brew update
- brew outdated xctool || brew upgrade xctool
- # Reused the build script that takes care of configuring and ensuring things
- # are up to date. Xcode and conformance tests will be directly invoked.
- objectivec/DevTools/full_mac_build.sh \
- --core-only --skip-xcode --skip-objc-conformance
-}
-
-internal_xctool_debug_and_release() {
- # Always use -reporter plain to avoid escape codes in output (makes travis
- # logs easier to read).
- xctool -reporter plain -configuration Debug "$@"
- xctool -reporter plain -configuration Release "$@"
-}
-
build_objectivec_ios() {
- internal_objectivec_common
- # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool
- # doesn't support >1 destination, so we have to build first and then run the
- # tests one destination at a time.
- internal_xctool_debug_and_release \
- -project objectivec/ProtocolBuffers_iOS.xcodeproj \
- -scheme ProtocolBuffers \
- -sdk iphonesimulator \
- build-tests
- IOS_DESTINATIONS=(
- "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
- "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
- "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
- "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
- )
- for i in "${IOS_DESTINATIONS[@]}" ; do
- internal_xctool_debug_and_release \
- -project objectivec/ProtocolBuffers_iOS.xcodeproj \
- -scheme ProtocolBuffers \
- -sdk iphonesimulator \
- -destination "${i}" \
- run-tests
- done
+ # Reused the build script that takes care of configuring and ensuring things
+ # are up to date. The OS X test runs the objc conformance test, so skip it
+ # here.
+ # Note: travis has xctool installed, and we've looked at using it in the past
+ # but it has ended up proving unreliable (bugs), an they are removing build
+ # support in favor of xcbuild (or just xcodebuild).
+ objectivec/DevTools/full_mac_build.sh \
+ --core-only --skip-xcode-osx --skip-objc-conformance "$@"
+}
+
+build_objectivec_ios_debug() {
+ build_objectivec_ios --skip-xcode-release
+}
+
+build_objectivec_ios_release() {
+ build_objectivec_ios --skip-xcode-debug
}
build_objectivec_osx() {
- internal_objectivec_common
- internal_xctool_debug_and_release \
- -project objectivec/ProtocolBuffers_OSX.xcodeproj \
- -scheme ProtocolBuffers \
- -destination "platform=OS X,arch=x86_64" \
- test
- cd conformance && make test_objc && cd ..
+ # Reused the build script that takes care of configuring and ensuring things
+ # are up to date.
+ objectivec/DevTools/full_mac_build.sh \
+ --core-only --skip-xcode-ios
+}
+
+build_objectivec_cocoapods_integration() {
+ # Update pod to the latest version.
+ gem install cocoapods --no-ri --no-rdoc
+ objectivec/Tests/CocoaPods/run_tests.sh
}
build_python() {
@@ -283,14 +302,16 @@
cd ..
}
-build_ruby19() {
- internal_build_cpp # For conformance tests.
- cd ruby && bash travis-test.sh ruby-1.9 && cd ..
+build_python_compatibility() {
+ internal_build_cpp
+ # Use the unit-tests extraced from 2.5.0 to test the compatibilty.
+ cd python/compatibility_tests/v2.5.0
+ # Test between 2.5.0 and the current version.
+ ./test.sh 2.5.0
+ # Test between 3.0.0-beta-1 and the current version.
+ ./test.sh 3.0.0-beta-1
}
-build_ruby20() {
- internal_build_cpp # For conformance tests.
- cd ruby && bash travis-test.sh ruby-2.0 && cd ..
-}
+
build_ruby21() {
internal_build_cpp # For conformance tests.
cd ruby && bash travis-test.sh ruby-2.1 && cd ..
@@ -301,12 +322,278 @@
}
build_jruby() {
internal_build_cpp # For conformance tests.
- cd ruby && bash travis-test.sh jruby && cd ..
+ # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be
+ # fixed.
+ cd ruby && bash travis-test.sh jruby-1.7 && cd ..
+}
+build_ruby_all() {
+ build_ruby21
+ build_ruby22
+ # TODO(teboring): Disable jruby test temperarily for it randomly fails.
+ # https://grpc-testing.appspot.com/job/protobuf_pull_request/735/consoleFull.
+ # build_jruby
}
build_javascript() {
internal_build_cpp
cd js && npm install && npm test && cd ..
+ cd conformance && make test_nodejs && cd ..
+}
+
+generate_php_test_proto() {
+ internal_build_cpp
+ pushd php/tests
+ # Generate test file
+ rm -rf generated
+ mkdir generated
+ ../../src/protoc --php_out=generated proto/test.proto proto/test_include.proto proto/test_no_namespace.proto proto/test_prefix.proto proto/test_php_namespace.proto proto/test_empty_php_namespace.proto proto/test_service.proto proto/test_service_namespace.proto
+ pushd ../../src
+ ./protoc --php_out=../php/tests/generated google/protobuf/empty.proto
+ ./protoc --php_out=../php/tests/generated -I../php/tests -I. ../php/tests/proto/test_import_descriptor_proto.proto
+ popd
+ popd
+}
+
+use_php() {
+ VERSION=$1
+ PHP=`which php`
+ PHP_CONFIG=`which php-config`
+ PHPIZE=`which phpize`
+ ln -sfn "/usr/local/php-${VERSION}/bin/php" $PHP
+ ln -sfn "/usr/local/php-${VERSION}/bin/php-config" $PHP_CONFIG
+ ln -sfn "/usr/local/php-${VERSION}/bin/phpize" $PHPIZE
+ generate_php_test_proto
+}
+
+use_php_zts() {
+ VERSION=$1
+ PHP=`which php`
+ PHP_CONFIG=`which php-config`
+ PHPIZE=`which phpize`
+ ln -sfn "/usr/local/php-${VERSION}-zts/bin/php" $PHP
+ ln -sfn "/usr/local/php-${VERSION}-zts/bin/php-config" $PHP_CONFIG
+ ln -sfn "/usr/local/php-${VERSION}-zts/bin/phpize" $PHPIZE
+ generate_php_test_proto
+}
+
+use_php_bc() {
+ VERSION=$1
+ PHP=`which php`
+ PHP_CONFIG=`which php-config`
+ PHPIZE=`which phpize`
+ ln -sfn "/usr/local/php-${VERSION}-bc/bin/php" $PHP
+ ln -sfn "/usr/local/php-${VERSION}-bc/bin/php-config" $PHP_CONFIG
+ ln -sfn "/usr/local/php-${VERSION}-bc/bin/phpize" $PHPIZE
+ generate_php_test_proto
+}
+
+build_php5.5() {
+ use_php 5.5
+
+ pushd php
+ rm -rf vendor
+ cp -r /usr/local/vendor-5.5 vendor
+ wget https://phar.phpunit.de/phpunit-4.8.0.phar -O /usr/bin/phpunit
+ phpunit
+ popd
+ pushd conformance
+ make test_php
+ popd
+}
+
+build_php5.5_c() {
+ use_php 5.5
+ wget https://phar.phpunit.de/phpunit-4.8.0.phar -O /usr/bin/phpunit
+ pushd php/tests
+ /bin/bash ./test.sh
+ popd
+ # TODO(teboring): Add it back
+ # pushd conformance
+ # make test_php_c
+ # popd
+}
+
+build_php5.5_zts_c() {
+ use_php_zts 5.5
+ wget https://phar.phpunit.de/phpunit-4.8.0.phar -O /usr/bin/phpunit
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ # TODO(teboring): Add it back
+ # pushd conformance
+ # make test_php_zts_c
+ # popd
+}
+
+build_php5.6() {
+ use_php 5.6
+ pushd php
+ rm -rf vendor
+ cp -r /usr/local/vendor-5.6 vendor
+ wget https://phar.phpunit.de/phpunit-5.7.0.phar -O /usr/bin/phpunit
+ phpunit
+ popd
+ pushd conformance
+ make test_php
+ popd
+}
+
+build_php5.6_c() {
+ use_php 5.6
+ wget https://phar.phpunit.de/phpunit-5.7.0.phar -O /usr/bin/phpunit
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ # TODO(teboring): Add it back
+ # pushd conformance
+ # make test_php_c
+ # popd
+}
+
+build_php5.6_zts_c() {
+ use_php_zts 5.6
+ wget https://phar.phpunit.de/phpunit-5.7.0.phar -O /usr/bin/phpunit
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ # TODO(teboring): Add it back
+ # pushd conformance
+ # make test_php_zts_c
+ # popd
+}
+
+build_php5.6_mac() {
+ generate_php_test_proto
+ # Install PHP
+ curl -s https://php-osx.liip.ch/install.sh | bash -s 5.6
+ PHP_FOLDER=`find /usr/local -type d -name "php5-5.6*"` # The folder name may change upon time
+ export PATH="$PHP_FOLDER/bin:$PATH"
+
+ # Install phpunit
+ curl https://phar.phpunit.de/phpunit-5.6.10.phar -L -o phpunit.phar
+ chmod +x phpunit.phar
+ sudo mv phpunit.phar /usr/local/bin/phpunit
+
+ # Install valgrind
+ echo "#! /bin/bash" > valgrind
+ chmod ug+x valgrind
+ sudo mv valgrind /usr/local/bin/valgrind
+
+ # Test
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ # TODO(teboring): Add it back
+ # pushd conformance
+ # make test_php_c
+ # popd
+}
+
+build_php7.0() {
+ use_php 7.0
+ pushd php
+ rm -rf vendor
+ cp -r /usr/local/vendor-7.0 vendor
+ wget https://phar.phpunit.de/phpunit-5.6.0.phar -O /usr/bin/phpunit
+ phpunit
+ popd
+ pushd conformance
+ make test_php
+ popd
+}
+
+build_php7.0_c() {
+ use_php 7.0
+ wget https://phar.phpunit.de/phpunit-5.6.0.phar -O /usr/bin/phpunit
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ # TODO(teboring): Add it back
+ # pushd conformance
+ # make test_php_c
+ # popd
+}
+
+build_php7.0_zts_c() {
+ use_php_zts 7.0
+ wget https://phar.phpunit.de/phpunit-5.6.0.phar -O /usr/bin/phpunit
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ # TODO(teboring): Add it back.
+ # pushd conformance
+ # make test_php_zts_c
+ # popd
+}
+
+build_php7.0_mac() {
+ generate_php_test_proto
+ # Install PHP
+ curl -s https://php-osx.liip.ch/install.sh | bash -s 7.0
+ PHP_FOLDER=`find /usr/local -type d -name "php7-7.0*"` # The folder name may change upon time
+ export PATH="$PHP_FOLDER/bin:$PATH"
+
+ # Install phpunit
+ curl https://phar.phpunit.de/phpunit-5.6.0.phar -L -o phpunit.phar
+ chmod +x phpunit.phar
+ sudo mv phpunit.phar /usr/local/bin/phpunit
+
+ # Install valgrind
+ echo "#! /bin/bash" > valgrind
+ chmod ug+x valgrind
+ sudo mv valgrind /usr/local/bin/valgrind
+
+ # Test
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ # TODO(teboring): Add it back
+ # pushd conformance
+ # make test_php_c
+ # popd
+}
+
+build_php_compatibility() {
+ internal_build_cpp
+ php/tests/compatibility_test.sh
+}
+
+build_php7.1() {
+ use_php 7.1
+ pushd php
+ rm -rf vendor
+ cp -r /usr/local/vendor-7.1 vendor
+ wget https://phar.phpunit.de/phpunit-5.6.0.phar -O /usr/bin/phpunit
+ phpunit
+ popd
+ pushd conformance
+ # TODO(teboring): Add it back
+ # make test_php
+ popd
+}
+
+build_php7.1_c() {
+ use_php 7.1
+ wget https://phar.phpunit.de/phpunit-5.6.0.phar -O /usr/bin/phpunit
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ pushd conformance
+ # make test_php_c
+ popd
+}
+
+build_php7.1_zts_c() {
+ use_php_zts 7.1
+ wget https://phar.phpunit.de/phpunit-5.6.0.phar -O /usr/bin/phpunit
+ cd php/tests && /bin/bash ./test.sh && cd ../..
+ pushd conformance
+ # make test_php_c
+ popd
+}
+
+build_php_all_32() {
+ build_php5.5
+ build_php5.6
+ build_php7.0
+ build_php7.1
+ build_php5.5_c
+ build_php5.6_c
+ build_php7.0_c
+ build_php7.1_c
+ build_php5.5_zts_c
+ build_php5.6_zts_c
+ build_php7.0_zts_c
+ build_php7.1_zts_c
+}
+
+build_php_all() {
+ build_php_all_32
+ build_php_compatibility
}
# Note: travis currently does not support testing more than one language so the
@@ -322,22 +609,35 @@
if [ "$#" -ne 1 ]; then
echo "
Usage: $0 { cpp |
+ cpp_distcheck |
csharp |
- java_jdk6 |
java_jdk7 |
java_oracle7 |
- javanano_jdk6 |
+ java_compatibility |
javanano_jdk7 |
javanano_oracle7 |
objectivec_ios |
+ objectivec_ios_debug |
+ objectivec_ios_release |
objectivec_osx |
+ objectivec_cocoapods_integration |
python |
python_cpp |
- ruby19 |
- ruby20 |
+ python_compatibility |
ruby21 |
ruby22 |
- jruby }
+ jruby |
+ ruby_all |
+ php5.5 |
+ php5.5_c |
+ php5.6 |
+ php5.6_c |
+ php7.0 |
+ php7.0_c |
+ php_compatibility |
+ php7.1 |
+ php7.1_c |
+ php_all)
"
exit 1
fi
