ARROW-3803: [C++/Python] Merge C++ builds and tests, run Python tests in separate CI entries

.travis.yml

@@ -62,74 +62,67 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-  - name: "C++ & Python w/ gcc 4.9"
+  - name: "C++ unit tests, code coverage with gcc 4.8"
     compiler: gcc
     language: cpp
     os: linux
     jdk: openjdk8
     env:
-    - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_TRAVIS_VALGRIND=1
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_ORC=1
     - ARROW_TRAVIS_COVERAGE=1
     - ARROW_TRAVIS_PARQUET=1
-    - ARROW_TRAVIS_PYTHON_DOCS=1
+    - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_GANDIVA_JAVA=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    - ARROW_TRAVIS_PYTHON_JVM=1
-    - ARROW_TRAVIS_JAVA_BUILD_ONLY=1
-    - ARROW_TRAVIS_PYTHON_GANDIVA=1
-    # ARROW-2999 Benchmarks are disabled in Travis CI for the time being
-    # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
-    - MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9"
     before_script:
-    # (ARROW_CI_CPP_AFFECTED implies ARROW_CI_PYTHON_AFFECTED)
-    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
+    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     # If either C++ or Python changed, we must install the C++ libraries
     - git submodule update --init
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
-    # All test steps are required for accurate C++ coverage info
-    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-    # Build Arrow Java to test the pyarrow<->JVM in-process bridge
-    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
-    # Only run Plasma tests with valgrind in one of the Python builds because
-    # they are slow
-    - export PLASMA_VALGRIND=0
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
-    - export PLASMA_VALGRIND=1
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
-    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
-  - name: "Gandiva C++ w/ gcc 4.9 and Java"
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
+    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
+    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
+  - name: "Python 2.7 and 3.6 unit tests, coverage with gcc 4.8"
     compiler: gcc
     language: cpp
     os: linux
     jdk: openjdk8
     env:
-    - ARROW_TRAVIS_GANDIVA=1
-    - ARROW_TRAVIS_GANDIVA_JAVA=1
-    - ARROW_TRAVIS_GANDIVA_TESTS=1
-    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
-    - ARROW_CPP_BUILD_TARGETS="gandiva-all"
-    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    # Valgrind is needed for the Plasma store tests
     - ARROW_TRAVIS_VALGRIND=1
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_TRAVIS_COVERAGE=1
+    - ARROW_TRAVIS_PYTHON_DOCS=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    - MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9"
+    - ARROW_TRAVIS_PYTHON_JVM=1
+    - ARROW_TRAVIS_PYTHON_GANDIVA=1
+    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
+    # TODO(wesm): Run the benchmarks outside of Travis
+    # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
     before_script:
-    # Run if something changed in CPP or Java.
-    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
+    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
-  - name: "[OS X] C++ & Python w/ XCode 6.4"
+    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+    # Only run Plasma tests with valgrind in one of the Python builds because
+    # they are slow
+    - export PLASMA_VALGRIND=0
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+    - export PLASMA_VALGRIND=1
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
+    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
+  - name: "[OS X] C++ w/ XCode 8.3"
     compiler: clang
     language: cpp
-    osx_image: xcode6.4
+    osx_image: xcode8.3
     os: osx
     cache:
     addons:
@@ -138,39 +131,41 @@ matrix:
     - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_ORC=1
     - ARROW_TRAVIS_PARQUET=1
+    - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_GANDIVA_JAVA=1
+    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib
+    # which can get loaded before the toolchain Boost libraries. These seem to
+    # get loaded even though we are modifying LD_LIBRARY_PATH. We build our own
+    # Boost and statically link to get around the issue until this can be
+    # investigated further
+    - ARROW_TRAVIS_VENDORED_BOOST=1
     before_script:
-    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
+    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
     # If either C++ or Python changed, we must install the C++ libraries
     - git submodule update --init
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
-    - if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
-  - name: "[OS X] Gandiva C++ w/ XCode 8.3 & Java"
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
+  - name: "[OS X] Python w/ XCode 6.4"
     compiler: clang
     language: cpp
-    # xcode 7.3 has a bug in strptime.
-    osx_image: xcode8.3
+    osx_image: xcode6.4
     os: osx
     cache:
     addons:
     env:
-    - ARROW_TRAVIS_GANDIVA=1
-    - ARROW_TRAVIS_GANDIVA_JAVA=1
-    - ARROW_TRAVIS_GANDIVA_TESTS=1
-    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
-    - ARROW_CPP_BUILD_TARGETS="gandiva-all"
     - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
     before_script:
-    # Run if something changed in CPP or Java.
-    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_cpp.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
+    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   - name: "[manylinux1] Python"
     language: cpp
     before_script:

ci/conda_env_cpp.yml

@@ -15,7 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-boost-cpp
+# ARROW-4056: The conda-forge boost 1.69.0 seems to break the Parquet unit
+# tests with Xcode 8.3. Root cause not yet determined
+boost-cpp=1.68.0
 brotli
 bzip2
 cmake

ci/travis_before_script_cpp.sh

@@ -40,6 +40,14 @@ if [ "$only_library_mode" == "no" ]; then
   source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 fi
 
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
+  # Set up C++ toolchain from conda-forge packages for faster builds
+  source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
+fi
+
+mkdir -p $ARROW_CPP_BUILD_DIR
+pushd $ARROW_CPP_BUILD_DIR
+
 CMAKE_COMMON_FLAGS="\
 -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
 -DARROW_NO_DEPRECATED_API=ON \
@@ -48,15 +56,10 @@ CMAKE_LINUX_FLAGS=""
 CMAKE_OSX_FLAGS=""
 
 if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
-  # Set up C++ toolchain from conda-forge packages for faster builds
-  source $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
   CMAKE_COMMON_FLAGS="${CMAKE_COMMON_FLAGS} -DARROW_JEMALLOC=ON"
   CMAKE_COMMON_FLAGS="${CMAKE_COMMON_FLAGS} -DARROW_WITH_BZ2=ON"
 fi
 
-mkdir -p $ARROW_CPP_BUILD_DIR
-pushd $ARROW_CPP_BUILD_DIR
-
 if [ $only_library_mode == "yes" ]; then
   CMAKE_COMMON_FLAGS="\
 $CMAKE_COMMON_FLAGS \
@@ -115,10 +118,14 @@ if [ $ARROW_TRAVIS_VERBOSE == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
 fi
 
-if [ $ARROW_TRAVIS_USE_VENDORED_BOOST == "1" ]; then
+if [ $ARROW_TRAVIS_VENDORED_BOOST == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BOOST_VENDORED=ON"
 fi
 
+if [ $ARROW_TRAVIS_STATIC_BOOST == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BOOST_USE_SHARED=OFF"
+fi
+
 if [ $ARROW_TRAVIS_OPTIONAL_INSTALL == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_OPTIONAL_INSTALL=ON"
 fi

ci/travis_env_common.sh

@@ -33,6 +33,8 @@ export ARROW_RUBY_DIR=$TRAVIS_BUILD_DIR/ruby
 export ARROW_RUST_DIR=${TRAVIS_BUILD_DIR}/rust
 export ARROW_R_DIR=${TRAVIS_BUILD_DIR}/r
 
+export ARROW_TRAVIS_COVERAGE=${ARROW_TRAVIS_COVERAGE:=0}
+
 if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then
     export ARROW_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/coverage.info
     export ARROW_PYTHON_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/.coverage

ci/travis_script_python.sh

@@ -87,7 +87,7 @@ rm -rf *
 # XXX Can we simply reuse CMAKE_COMMON_FLAGS from travis_before_script_cpp.sh?
 CMAKE_COMMON_FLAGS="-DARROW_EXTRA_ERROR_CONTEXT=ON"
 
-PYTHON_CPP_BUILD_TARGETS="arrow_python-all plasma"
+PYTHON_CPP_BUILD_TARGETS="arrow_python-all plasma parquet"
 
 if [ $ARROW_TRAVIS_COVERAGE == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON"
@@ -103,6 +103,7 @@ cmake -GNinja \
       -DARROW_BUILD_TESTS=ON \
       -DARROW_BUILD_UTILITIES=OFF \
       -DARROW_OPTIONAL_INSTALL=ON \
+      -DARROW_PARQUET=on \
       -DARROW_PLASMA=on \
       -DARROW_TENSORFLOW=on \
       -DARROW_PYTHON=on \
@@ -176,12 +177,11 @@ if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then
     coverage report -i --include="*/_parquet.pyx"
     # Generate XML file for CodeCov
     coverage xml -i -o $TRAVIS_BUILD_DIR/coverage.xml
-    # Capture C++ coverage info and combine with previous coverage file
+    # Capture C++ coverage info
     pushd $TRAVIS_BUILD_DIR
     lcov --quiet --directory . --capture --no-external --output-file coverage-python-tests.info \
         2>&1 | grep -v "WARNING: no data found for /usr/include"
     lcov --add-tracefile coverage-python-tests.info \
-        --add-tracefile $ARROW_CPP_COVERAGE_FILE \
         --output-file $ARROW_CPP_COVERAGE_FILE
     rm coverage-python-tests.info
     popd   # $TRAVIS_BUILD_DIR

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -346,6 +346,8 @@ if (MSVC AND ARROW_USE_STATIC_CRT)
   set(Boost_USE_STATIC_RUNTIME ON)
 endif()
 set(Boost_ADDITIONAL_VERSIONS
+  "1.70.0" "1.70"
+  "1.69.0" "1.69"
   "1.68.0" "1.68"
   "1.67.0" "1.67"
   "1.66.0" "1.66"

cpp/src/plasma/CMakeLists.txt

@@ -96,6 +96,8 @@ ADD_ARROW_LIB(plasma
   SHARED_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} ${PLASMA_LINK_LIBS}
   STATIC_LINK_LIBS ${FLATBUFFERS_STATIC_LIB} ${CMAKE_THREAD_LIBS_INIT} ${PLASMA_STATIC_LINK_LIBS})
 
+add_dependencies(plasma ${PLASMA_LIBRARIES})
+
 foreach(LIB_TARGET ${PLASMA_LIBRARIES})
   target_compile_definitions(${LIB_TARGET}
     PRIVATE ARROW_EXPORTING)

java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/MicroBenchmarkTest.java

@@ -26,10 +26,12 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 
+@Ignore
 public class MicroBenchmarkTest extends BaseEvaluatorTest {
 
   private double toleranceRatio = 4.0;

python/CMakeLists.txt

@@ -138,6 +138,7 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-constant-logical-operand")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-declarations")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sometimes-uninitialized")
 
   # We have public Cython APIs which return C++ types, which are in an extern
   # "C" blog (no symbol mangling) and clang doesn't like this