ARROW-202: Integrate with appveyor ci for windows

LICENSE.txt

@@ -200,3 +200,15 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Kudu.
+
+ * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
+
+Copyright: 2016 The Apache Software Foundation.
+Home page: https://kudu.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------

NOTICE.txt

@@ -41,3 +41,17 @@ This product includes software from the CMake project
 
 This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
  * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Kudu, which includes the following in
+its NOTICE file:
+
+  Apache Kudu
+  Copyright 2016 The Apache Software Foundation
+
+  This product includes software developed at
+  The Apache Software Foundation (http://www.apache.org/).
+
+  Portions of this software were developed at
+  Cloudera, Inc (http://www.cloudera.com/).

appveyor.yml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Operating system (build VM template)
+os: Visual Studio 2015
+
+environment:
+  matrix:
+    - GENERATOR: Visual Studio 14 2015 Win64
+    # - GENERATOR: Visual Studio 14 2015
+  MSVC_DEFAULT_OPTIONS: ON
+  BOOST_ROOT: C:\Libraries\boost_1_59_0
+  BOOST_LIBRARYDIR: C:\Libraries\boost_1_59_0\lib64-msvc-14.0
+
+build_script:
+ - cd cpp
+ - mkdir build
+ - cd build
+ # A lot of features are still deactivated as they do not build on Windows
+ #  * gbenchmark doesn't build with MSVC
+ - cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_IPC=OFF -DARROW_HDFS=OFF -DARROW_BUILD_BENCHMARKS=OFF ..
+ - cmake --build . --config Debug
+
+# test_script:
+#  - ctest -VV

cpp/CMakeLists.txt

@@ -141,9 +141,11 @@ endif()
 # For CMAKE_BUILD_TYPE=Release
 #   -O3: Enable all compiler optimizations
 #   -g: Enable symbols for profiler tools (TODO: remove for shipping)
-set(CXX_FLAGS_DEBUG "-ggdb -O0")
-set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
-set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+if (NOT MSVC)
+  set(CXX_FLAGS_DEBUG "-ggdb -O0")
+  set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
+  set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
+endif()
 
 set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")
 set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use")
@@ -347,6 +349,8 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
       COMPILE_FLAGS " -DARROW_VALGRIND")
     add_test(${TEST_NAME}
       valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
+  elseif(MSVC)
+    add_test(${TEST_NAME} ${TEST_PATH})
   else()
     add_test(${TEST_NAME}
       ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
@@ -431,40 +435,44 @@ endfunction()
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
-# find boost headers and libs
+# Find static boost headers and libs
+# TODO Differentiate here between release and debug builds
 set(Boost_DEBUG TRUE)
 set(Boost_USE_MULTITHREADED ON)
 set(Boost_USE_STATIC_LIBS ON)
 find_package(Boost COMPONENTS system filesystem REQUIRED)
-include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-set(BOOST_STATIC_LIBS ${Boost_LIBRARIES})
-list(LENGTH BOOST_STATIC_LIBS BOOST_STATIC_LIBS_LEN)
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+  set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+else()
+  set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+  set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+endif()
 
-# Find Boost shared libraries.
+# Find shared Boost libraries.
 set(Boost_USE_STATIC_LIBS OFF)
 find_package(Boost COMPONENTS system filesystem REQUIRED)
-set(BOOST_SHARED_LIBS ${Boost_LIBRARIES})
-list(LENGTH BOOST_SHARED_LIBS BOOST_SHARED_LIBS_LEN)
-list(SORT BOOST_SHARED_LIBS)
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+  set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
+else()
+  set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+  set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
+endif()
 
 message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
 message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
 
-math(EXPR LAST_IDX "${BOOST_STATIC_LIBS_LEN} - 1")
-foreach(IDX RANGE ${LAST_IDX})
-  list(GET BOOST_STATIC_LIBS ${IDX} BOOST_STATIC_LIB)
-  list(GET BOOST_SHARED_LIBS ${IDX} BOOST_SHARED_LIB)
+ADD_THIRDPARTY_LIB(boost_system
+    STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
+    SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
+
+ADD_THIRDPARTY_LIB(boost_filesystem
+    STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
+    SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
+
+SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
 
-  # Remove the prefix/suffix from the library name.
-  #
-  # e.g. libboost_system-mt --> boost_system
-  get_filename_component(LIB_NAME ${BOOST_STATIC_LIB} NAME_WE)
-  string(REGEX REPLACE "lib([^-]*)(-mt)?" "\\1" LIB_NAME_NO_PREFIX_SUFFIX ${LIB_NAME})
-  ADD_THIRDPARTY_LIB(${LIB_NAME_NO_PREFIX_SUFFIX}
-    STATIC_LIB "${BOOST_STATIC_LIB}"
-    SHARED_LIB "${BOOST_SHARED_LIB}")
-  list(APPEND ARROW_BOOST_LIBS ${LIB_NAME_NO_PREFIX_SUFFIX})
-endforeach()
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 
 # ----------------------------------------------------------------------
@@ -482,7 +490,7 @@ if(ARROW_BUILD_TESTS)
 
     ExternalProject_Add(googletest_ep
       URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
-      CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
+      CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
       # googletest doesn't define install rules, so just build in the
       # source dir and don't try to install.  See its README for
       # details.
@@ -491,7 +499,7 @@ if(ARROW_BUILD_TESTS)
 
     set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
-    set(GTEST_STATIC_LIB "${GTEST_PREFIX}/libgtest.a")
+    set(GTEST_STATIC_LIB "${GTEST_PREFIX}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GTEST_VENDORED 1)
   else()
     find_package(GTest REQUIRED)
@@ -571,7 +579,7 @@ if(ARROW_BUILD_BENCHMARKS)
         "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
 
     set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
-    set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/libbenchmark.a")
+    set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GBENCHMARK_VENDORED 1)
   else()
     find_package(GBenchmark REQUIRED)

cpp/cmake_modules/CompilerInfo.cmake

@@ -1,25 +1,32 @@
-# Copyright 2013 Cloudera, Inc.
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 # Sets COMPILER_FAMILY to 'clang' or 'gcc'
 # Sets COMPILER_VERSION to the version
 execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
                 ERROR_VARIABLE COMPILER_VERSION_FULL)
 message(INFO " ${COMPILER_VERSION_FULL}")
+message(INFO " ${CMAKE_CXX_COMPILER_ID}")
+
+if(MSVC)
+  set(COMPILER_FAMILY "msvc")
 
 # clang on Linux and Mac OS X before 10.9
-if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
   set(COMPILER_FAMILY "clang")
   string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
@@ -29,10 +36,15 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
   string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
 
-# clang on Mac OS X, XCode 7+. No version replacement is done
-# because Apple no longer advertises the upstream LLVM version.
-elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-.*")
+# clang on Mac OS X, XCode 7.
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-7")
+  set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "3.7.0svn")
+
+# clang on Mac OS X, XCode 8.
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-8")
   set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "3.8.0svn")
 
 # gcc
 elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*")

cpp/src/arrow/array-test.cc

@@ -18,6 +18,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <memory>
+#include <numeric>
 #include <vector>
 
 #include "gtest/gtest.h"

cpp/src/arrow/io/CMakeLists.txt

@@ -18,10 +18,16 @@
 # ----------------------------------------------------------------------
 # arrow_io : Arrow IO interfaces
 
-set(ARROW_IO_LINK_LIBS
-  arrow_shared
-  dl
-)
+if (MSVC)
+  set(ARROW_IO_LINK_LIBS
+    arrow_shared
+  )
+else()
+  set(ARROW_IO_LINK_LIBS
+    arrow_shared
+    dl
+  )
+endif()
 
 if (ARROW_BOOST_USE_SHARED)
   set(ARROW_IO_PRIVATE_LINK_LIBS

cpp/src/arrow/io/io-file-test.cc

@@ -18,7 +18,9 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
-#include <fcntl.h>
+#ifndef _MSC_VER
+# include <fcntl.h>
+#endif
 #include <fstream>
 #include <memory>
 #include <sstream>
@@ -38,7 +40,12 @@ static bool FileExists(const std::string& path) {
 }
 
 static bool FileIsClosed(int fd) {
+#ifdef _MSC_VER
+  // Close file a second time, this should set errno to EBADF
+  close(fd);
+#else
   if (-1 != fcntl(fd, F_GETFD)) { return false; }
+#endif
   return errno == EBADF;
 }
 

cpp/src/arrow/io/memory.cc

@@ -17,7 +17,18 @@
 
 #include "arrow/io/memory.h"
 
-#include <sys/mman.h>  // For memory-mapping
+// sys/mman.h not present in Visual Studio or Cygwin
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include "arrow/io/mman.h"
+#undef Realloc
+#undef Free
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
 
 #include <algorithm>
 #include <cerrno>

cpp/src/arrow/io/mman.h

@@ -76,7 +76,7 @@ static DWORD __map_mmap_prot_file(const int prot) {
   return desiredAccess;
 }
 
-void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
+static void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
   HANDLE fm, h;
 
   void* map = MAP_FAILED;
@@ -143,15 +143,15 @@ void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
   return map;
 }
 
-int munmap(void* addr, size_t len) {
+static int munmap(void* addr, size_t len) {
   if (UnmapViewOfFile(addr)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);
 
   return -1;
 }
 
-int mprotect(void* addr, size_t len, int prot) {
+static int mprotect(void* addr, size_t len, int prot) {
   DWORD newProtect = __map_mmap_prot_page(prot);
   DWORD oldProtect = 0;
 
@@ -162,23 +162,23 @@ int mprotect(void* addr, size_t len, int prot) {
   return -1;
 }
 
-int msync(void* addr, size_t len, int flags) {
+static int msync(void* addr, size_t len, int flags) {
   if (FlushViewOfFile(addr, len)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);
 
   return -1;
 }
 
-int mlock(const void* addr, size_t len) {
+static int mlock(const void* addr, size_t len) {
   if (VirtualLock((LPVOID)addr, len)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);
 
   return -1;
 }
 
-int munlock(const void* addr, size_t len) {
+static int munlock(const void* addr, size_t len) {
   if (VirtualUnlock((LPVOID)addr, len)) return 0;
 
   errno = __map_mman_error(GetLastError(), EPERM);