PARQUET-512: Add Google benchmark for performance testing

.travis.yml

@@ -28,7 +28,7 @@ matrix:
     os: linux
     before_script:
     - source $TRAVIS_BUILD_DIR/ci/before_script_travis.sh
-    - cmake -DCMAKE_CXX_FLAGS="-Werror" -DPARQUET_TEST_MEMCHECK=ON -DPARQUET_GENERATE_COVERAGE=1 $TRAVIS_BUILD_DIR
+    - cmake -DCMAKE_CXX_FLAGS="-Werror" -DPARQUET_TEST_MEMCHECK=ON -DPARQUET_BUILD_BENCHMARKS=ON -DPARQUET_GENERATE_COVERAGE=1 $TRAVIS_BUILD_DIR
     - export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
   - compiler: clang
     os: linux

CMakeLists.txt

@@ -74,6 +74,9 @@ if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   option(PARQUET_USE_SSE
 	"Build with SSE4 optimizations"
 	OFF)
+  option(PARQUET_BUILD_BENCHMARKS
+	"Build the libparquet benchmark suite"
+    OFF)
   option(PARQUET_BUILD_TESTS
 	"Build the libparquet test suite"
 	ON)
@@ -102,6 +105,60 @@ else()
   set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}")
 endif()
 
+############################################################
+# Benchmarking
+############################################################
+# Add a new micro benchmark, with or without an executable that should be built.
+# If benchmarks are enabled then they will be run along side unit tests with ctest.
+# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests,
+# respectively.
+#
+# REL_BENCHMARK_NAME is the name of the benchmark app. It may be a single component
+# (e.g. monotime-benchmark) or contain additional components (e.g.
+# net/net_util-benchmark). Either way, the last component must be a globally
+# unique name.
+
+# The benchmark will registered as unit test with ctest with a label
+# of 'benchmark'.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_PARQUET_BENCHMARK REL_BENCHMARK_NAME)
+    if(NOT PARQUET_BUILD_BENCHMARKS)
+    return()
+  endif()
+  get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
+    # This benchmark has a corresponding .cc file, set it up as an executable.
+    set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
+    add_executable(${BENCHMARK_NAME} "${REL_BENCHMARK_NAME}.cc")
+    target_link_libraries(${BENCHMARK_NAME} ${PARQUET_BENCHMARK_LINK_LIBS})
+    add_dependencies(runbenchmark ${BENCHMARK_NAME})
+    set(NO_COLOR "--color_print=false")
+  else()
+    # No executable, just invoke the benchmark (probably a script) directly.
+    set(BENCHMARK_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME})
+    set(NO_COLOR "")
+  endif()
+
+  add_test(${BENCHMARK_NAME}
+    ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} benchmark ${BENCHMARK_PATH} ${NO_COLOR})
+  set_tests_properties(${BENCHMARK_NAME} PROPERTIES LABELS "benchmark")
+  if(ARGN)
+    set_tests_properties(${BENCHMARK_NAME} PROPERTIES ${ARGN})
+  endif()
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_BENCHMARKS.
+function(ADD_PARQUET_BENCHMARK_DEPENDENCIES REL_BENCHMARK_NAME)
+  if(NOT PARQUET_BUILD_BENCHMARKS)
+    return()
+  endif()
+  get_filename_component(BENCMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+
+  add_dependencies(${BENCHMARK_NAME} ${ARGN})
+endfunction()
+
 ############################################################
 # Testing
 ############################################################
@@ -113,6 +170,9 @@ endif()
 # net/net_util-test). Either way, the last component must be a globally
 # unique name.
 #
+# The unit test is added with a label of "unittest" to support filtering with
+# ctest.
+#
 # Arguments after the test name will be passed to set_tests_properties().
 function(ADD_PARQUET_TEST REL_TEST_NAME)
   if(NOT PARQUET_BUILD_TESTS)
@@ -124,6 +184,7 @@ function(ADD_PARQUET_TEST REL_TEST_NAME)
     # This test has a corresponding .cc file, set it up as an executable.
     set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
     add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
+    add_dependencies(unittest ${TEST_NAME})
 
 	if(APPLE)
 	  # On OS X / Thrift >= 0.9.2, tr1/tuple.h is not in libc++
@@ -149,8 +210,9 @@ function(ADD_PARQUET_TEST REL_TEST_NAME)
 	  valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
   else()
 	add_test(${TEST_NAME}
-      ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
+        ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
   endif()
+  set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
   if(ARGN)
     set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
   endif()
@@ -213,11 +275,26 @@ add_library(zlibstatic STATIC IMPORTED)
 set_target_properties(zlibstatic PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB})
 
 ## GTest
+add_custom_target(unittest ctest -L unittest)
 find_package(GTest REQUIRED)
 include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
 add_library(gtest STATIC IMPORTED)
 set_target_properties(gtest PROPERTIES IMPORTED_LOCATION ${GTEST_STATIC_LIB})
 
+## Google Benchmark
+if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
+  set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
+endif()
+
+if(PARQUET_BUILD_BENCHMARKS)
+  add_custom_target(runbenchmark ctest -L benchmark)
+  find_package(GBenchmark REQUIRED)
+  include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR})
+  message(${GBENCHMARK_STATIC_LIB})
+  add_library(gbenchmark STATIC IMPORTED)
+  set_target_properties(gbenchmark PROPERTIES IMPORTED_LOCATION ${GBENCHMARK_STATIC_LIB})
+endif()
+
 # Thrift requires these definitions for some types that we use
 add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DHAVE_NETDB_H)
 add_definitions(-fPIC)
@@ -331,6 +408,11 @@ set(PARQUET_MIN_TEST_LIBS
   parquet)
 set(PARQUET_TEST_LINK_LIBS ${PARQUET_MIN_TEST_LIBS})
 
+#############################################################
+# Benchmark linking
+
+set(PARQUET_BENCHMARK_LINK_LIBS parquet parquet_benchmark_main)
+
 #############################################################
 # Code coverage
 

README.md

@@ -24,6 +24,7 @@
 - zlib
 - thrift 0.7+ [install instructions](https://thrift.apache.org/docs/install/)
 - googletest 1.7.0 (cannot be installed with package managers)
+- Google Benchmark (only required if building benchmarks)
 
 You can install these dependencies using a package manager or using the
 `thirdparty/` scripts in this repository. On Homebrew, you can run:
@@ -87,7 +88,7 @@ This library uses Google's `googletest` unit test framework. After building
 with `make`, you can run the test suite by running
 
 ```
-ctest
+make unittest
 ```
 
 The test suite relies on an environment variable `PARQUET_TEST_DATA` pointing
@@ -107,6 +108,19 @@ you can use valgrind with ctest to look for memory leaks:
 valgrind --tool=memcheck --leak-check=yes ctest
 ```
 
+## Building/Running benchmarks
+
+Follow the directions for simple build except run cmake
+with the `--PARQUET_BUILD_BENCHMARKS` parameter set correctly:
+
+    cmake -DPARQUET_BUILD_BENCHMARKS=ON ..
+
+and instead of make unittest run either `make; ctest` to run both unit tests
+and benchmarks or `make runbenchmark` to run only the benchmark tests.
+
+Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
+
+
 ## Out-of-source builds
 
 parquet-cpp supports out of source builds. For example:
@@ -116,7 +130,7 @@ mkdir test-build
 cd test-build
 cmake ..
 make
-ctest
+ctest -L unittest
 ```
 
 By using out-of-source builds you can preserve your current build state in case
@@ -172,7 +186,7 @@ mkdir coverage-build
 cd coverage-build
 cmake -DPARQUET_GENERATE_COVERAGE=1
 make -j$PARALLEL
-ctest
+ctest -L unittest
 ```
 
 The `gcov` artifacts are not located in a place that works well with either
@@ -205,4 +219,4 @@ coveralls -t $PARQUET_CPP_COVERAGE_TOKEN --gcov-options '\-l' -r $PARQUET_ROOT -
 
 
 Note that `gcov` throws off artifacts from the STL, so I excluded my toolchain
-root stored in `$NATIVE_TOOLCHAIN` to avoid a cluttered coverage report.
+root stored in `$NATIVE_TOOLCHAIN` to avoid a cluttered coverage report.