ARROW-14421: [C++] Implement Flight SQL

ci/docker/conda-cpp.dockerfile

@@ -41,6 +41,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=CONDA \
     ARROW_FLIGHT=ON \
+    ARROW_FLIGHT_SQL=ON \
     ARROW_GANDIVA=ON \
     ARROW_HOME=$CONDA_PREFIX \
     ARROW_ORC=ON \

ci/scripts/cpp_build.sh

@@ -70,6 +70,7 @@ cmake \
   -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \
   -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \
   -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \
+  -DARROW_FLIGHT_SQL=${ARROW_FLIGHT_SQL:-OFF} \
   -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \
   -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
   -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \

cpp/CMakeLists.txt

@@ -334,6 +334,10 @@ if(ARROW_GANDIVA)
   set(ARROW_WITH_RE2 ON)
 endif()
 
+if(ARROW_FLIGHT_SQL)
+  set(ARROW_FLIGHT ON)
+endif()
+
 if(ARROW_CUDA
    OR ARROW_FLIGHT
    OR ARROW_PARQUET

cpp/cmake_modules/DefineOptions.cmake

@@ -226,6 +226,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   define_option(ARROW_FLIGHT
                 "Build the Arrow Flight RPC System (requires GRPC, Protocol Buffers)" OFF)
 
+  define_option(ARROW_FLIGHT_SQL "Build the Arrow Flight SQL extension" OFF)
+
   define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)
 
   define_option(ARROW_GCS

cpp/cmake_modules/FindArrowFlightSql.cmake

@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find Arrow Flight SQL
+#
+# This module requires Arrow from which it uses
+#  arrow_find_package()
+#
+# This module defines
+#  ARROW_FLIGHT_SQL_FOUND, whether Flight has been found
+#  ARROW_FLIGHT_SQL_IMPORT_LIB,
+#    path to libarrow_flight's import library (Windows only)
+#  ARROW_FLIGHT_SQL_INCLUDE_DIR, directory containing headers
+#  ARROW_FLIGHT_SQL_LIBS, deprecated. Use ARROW_FLIGHT_SQL_LIB_DIR instead
+#  ARROW_FLIGHT_SQL_LIB_DIR, directory containing Flight libraries
+#  ARROW_FLIGHT_SQL_SHARED_IMP_LIB, deprecated. Use ARROW_FLIGHT_SQL_IMPORT_LIB instead
+#  ARROW_FLIGHT_SQL_SHARED_LIB, path to libarrow_flight's shared library
+#  ARROW_FLIGHT_SQL_STATIC_LIB, path to libarrow_flight.a
+
+if(DEFINED ARROW_FLIGHT_SQL_FOUND)
+  return()
+endif()
+
+set(find_package_arguments)
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION)
+  list(APPEND find_package_arguments "${${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION}")
+endif()
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
+  list(APPEND find_package_arguments REQUIRED)
+endif()
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
+  list(APPEND find_package_arguments QUIET)
+endif()
+find_package(Arrow ${find_package_arguments})
+
+if(ARROW_FOUND)
+  arrow_find_package(ARROW_FLIGHT_SQL
+                     "${ARROW_HOME}"
+                     arrow_flight_sql
+                     arrow/flight/sql/api.h
+                     ArrowFlightSql
+                     arrow-flight-sql)
+  if(NOT ARROW_FLIGHT_SQL_VERSION)
+    set(ARROW_FLIGHT_SQL_VERSION "${ARROW_VERSION}")
+  endif()
+endif()
+
+if("${ARROW_FLIGHT_SQL_VERSION}" VERSION_EQUAL "${ARROW_VERSION}")
+  set(ARROW_FLIGHT_SQL_VERSION_MATCH TRUE)
+else()
+  set(ARROW_FLIGHT_SQL_VERSION_MATCH FALSE)
+endif()
+
+mark_as_advanced(ARROW_FLIGHT_SQL_IMPORT_LIB
+                 ARROW_FLIGHT_SQL_INCLUDE_DIR
+                 ARROW_FLIGHT_SQL_LIBS
+                 ARROW_FLIGHT_SQL_LIB_DIR
+                 ARROW_FLIGHT_SQL_SHARED_IMP_LIB
+                 ARROW_FLIGHT_SQL_SHARED_LIB
+                 ARROW_FLIGHT_SQL_STATIC_LIB
+                 ARROW_FLIGHT_SQL_VERSION
+                 ARROW_FLIGHT_SQL_VERSION_MATCH)
+
+find_package_handle_standard_args(
+  ArrowFlightSql
+  REQUIRED_VARS ARROW_FLIGHT_SQL_INCLUDE_DIR ARROW_FLIGHT_SQL_LIB_DIR
+                ARROW_FLIGHT_SQL_VERSION_MATCH
+  VERSION_VAR ARROW_FLIGHT_SQL_VERSION)
+set(ARROW_FLIGHT_SQL_FOUND ${ArrowFlightSql_FOUND})
+
+if(ArrowFlightSql_FOUND AND NOT ArrowFlightSql_FIND_QUIETLY)
+  message(STATUS "Found the Arrow Flight SQL by ${ARROW_FLIGHT_SQL_FIND_APPROACH}")
+  message(STATUS "Found the Arrow Flight SQL shared library: ${ARROW_FLIGHT_SQL_SHARED_LIB}"
+  )
+  message(STATUS "Found the Arrow Flight SQL import library: ${ARROW_FLIGHT_SQL_IMPORT_LIB}"
+  )
+  message(STATUS "Found the Arrow Flight SQL static library: ${ARROW_FLIGHT_SQL_STATIC_LIB}"
+  )
+endif()

cpp/cmake_modules/FindSQLite3Alt.cmake

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Once done this will define
+# - FindSQLite3Alt
+#
+# This module will set the following variables if found:
+#  SQLite3_INCLUDE_DIRS  - SQLite3 include dir.
+#  SQLite3_LIBRARIES     - List of libraries when using SQLite3.
+#  SQLite3_FOUND         - True if SQLite3 found.
+#
+# Usage of this module as follows:
+# find_package(SQLite3Alt)
+
+find_path(SQLite3_INCLUDE_DIR sqlite3.h)
+find_library(SQLite3_LIBRARY NAMES sqlite3)
+
+# handle the QUIETLY and REQUIRED arguments and set SQLite3_FOUND to TRUE if
+# all listed variables are TRUE
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(SQLite3Alt REQUIRED_VARS SQLite3_LIBRARY
+                                                           SQLite3_INCLUDE_DIR)
+
+mark_as_advanced(SQLite3_LIBRARY SQLite3_INCLUDE_DIR)
+
+if(SQLite3Alt_FOUND)
+  set(SQLite3_INCLUDE_DIRS ${SQLite3_INCLUDE_DIR})
+  set(SQLite3_LIBRARIES ${SQLite3_LIBRARY})
+endif()

cpp/src/arrow/CMakeLists.txt

@@ -732,6 +732,10 @@ if(ARROW_FLIGHT)
   add_subdirectory(flight)
 endif()
 
+if(ARROW_FLIGHT_SQL)
+  add_subdirectory(flight/sql)
+endif()
+
 if(ARROW_HIVESERVER2)
   add_subdirectory(dbi/hiveserver2)
 endif()

cpp/src/arrow/flight/CMakeLists.txt

@@ -25,7 +25,23 @@ if(WIN32)
   list(APPEND ARROW_FLIGHT_LINK_LIBS ws2_32.lib)
 endif()
 
-if(ARROW_TEST_LINKAGE STREQUAL "static")
+set(ARROW_FLIGHT_TEST_LINKAGE
+    "${ARROW_TEST_LINKAGE}"
+    PARENT_SCOPE)
+if(Protobuf_USE_STATIC_LIBS)
+  message(STATUS "Linking Arrow Flight tests statically due to static Protobuf")
+  set(ARROW_FLIGHT_TEST_LINKAGE
+      "static"
+      PARENT_SCOPE)
+endif()
+if(NOT ARROW_GRPC_USE_SHARED)
+  message(STATUS "Linking Arrow Flight tests statically due to static gRPC")
+  set(ARROW_FLIGHT_TEST_LINKAGE
+      "static"
+      PARENT_SCOPE)
+endif()
+
+if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static")
   set(ARROW_FLIGHT_TEST_LINK_LIBS
       arrow_flight_static arrow_flight_testing_static ${ARROW_FLIGHT_STATIC_LINK_LIBS}
       ${ARROW_TEST_LINK_LIBS})

cpp/src/arrow/flight/sql/ArrowFlightSqlConfig.cmake.in

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This config sets the following variables in your project::
+#
+#   ArrowFlightSql_FOUND - true if Arrow Flight SQL found on the system
+#
+# This config sets the following targets in your project::
+#
+#   arrow_flight_sql_shared - for linked as shared library if shared library is built
+#   arrow_flight_sql_static - for linked as static library if static library is built
+
+@PACKAGE_INIT@
+
+include(CMakeFindDependencyMacro)
+find_dependency(ArrowFlight)
+
+# Load targets only once. If we load targets multiple times, CMake reports
+# already existent target error.
+if(NOT (TARGET arrow_flight_sql_shared OR TARGET arrow_flight_sql_static))
+  include("${CMAKE_CURRENT_LIST_DIR}/ArrowFlightSqlTargets.cmake")
+endif()

cpp/src/arrow/flight/sql/CMakeLists.txt

@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+add_custom_target(arrow_flight_sql)
+
+arrow_install_all_headers("arrow/flight/sql")
+
+set(FLIGHT_SQL_PROTO_PATH "${ARROW_SOURCE_DIR}/../format")
+set(FLIGHT_SQL_PROTO ${ARROW_SOURCE_DIR}/../format/FlightSql.proto)
+
+set(FLIGHT_SQL_GENERATED_PROTO_FILES "${CMAKE_CURRENT_BINARY_DIR}/FlightSql.pb.cc"
+                                     "${CMAKE_CURRENT_BINARY_DIR}/FlightSql.pb.h")
+
+set(PROTO_DEPENDS ${FLIGHT_SQL_PROTO} ${ARROW_PROTOBUF_LIBPROTOBUF})
+
+add_custom_command(OUTPUT ${FLIGHT_SQL_GENERATED_PROTO_FILES}
+                   COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${FLIGHT_SQL_PROTO_PATH}"
+                           "--cpp_out=${CMAKE_CURRENT_BINARY_DIR}" "${FLIGHT_SQL_PROTO}"
+                   DEPENDS ${PROTO_DEPENDS})
+
+set_source_files_properties(${FLIGHT_SQL_GENERATED_PROTO_FILES} PROPERTIES GENERATED TRUE)
+
+add_custom_target(flight_sql_protobuf_gen ALL DEPENDS ${FLIGHT_SQL_GENERATED_PROTO_FILES})
+
+set(ARROW_FLIGHT_SQL_SRCS server.cc sql_info_internal.cc client.cc
+                          "${CMAKE_CURRENT_BINARY_DIR}/FlightSql.pb.cc")
+
+add_arrow_lib(arrow_flight_sql
+              CMAKE_PACKAGE_NAME
+              ArrowFlightSql
+              PKG_CONFIG_NAME
+              arrow-flight-sql
+              OUTPUTS
+              ARROW_FLIGHT_SQL_LIBRARIES
+              SOURCES
+              ${ARROW_FLIGHT_SQL_SRCS}
+              DEPENDENCIES
+              flight_sql_protobuf_gen
+              SHARED_LINK_FLAGS
+              ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in cpp/arrow/CMakeLists.txt
+              SHARED_LINK_LIBS
+              arrow_flight_shared
+              STATIC_LINK_LIBS
+              arrow_flight_static)
+
+if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static")
+  set(ARROW_FLIGHT_SQL_TEST_LINK_LIBS
+      arrow_flight_sql_static arrow_flight_testing_static
+      ${ARROW_FLIGHT_STATIC_LINK_LIBS} ${ARROW_TEST_LINK_LIBS})
+else()
+  set(ARROW_FLIGHT_SQL_TEST_LINK_LIBS arrow_flight_sql_shared arrow_flight_testing_shared
+                                      ${ARROW_TEST_LINK_LIBS})
+endif()
+
+# Build test server for unit tests
+if(ARROW_BUILD_TESTS OR ARROW_BUILD_EXAMPLES)
+  find_package(SQLite3Alt REQUIRED)
+
+  set(ARROW_FLIGHT_SQL_TEST_SERVER_SRCS
+      example/sqlite_sql_info.cc
+      example/sqlite_statement.cc
+      example/sqlite_statement_batch_reader.cc
+      example/sqlite_server.cc
+      example/sqlite_tables_schema_batch_reader.cc)
+
+  add_arrow_test(flight_sql_test
+                 SOURCES
+                 client_test.cc
+                 server_test.cc
+                 ${ARROW_FLIGHT_SQL_TEST_SERVER_SRCS}
+                 STATIC_LINK_LIBS
+                 ${ARROW_FLIGHT_SQL_TEST_LINK_LIBS}
+                 ${SQLite3_LIBRARIES}
+                 LABELS
+                 "arrow_flight_sql")
+
+  add_executable(flight_sql_test_server test_server_cli.cc
+                                        ${ARROW_FLIGHT_SQL_TEST_SERVER_SRCS})
+  target_link_libraries(flight_sql_test_server
+                        PRIVATE ${ARROW_FLIGHT_SQL_TEST_LINK_LIBS} ${GFLAGS_LIBRARIES}
+                                ${SQLite3_LIBRARIES})
+
+  add_executable(flight_sql_test_app test_app_cli.cc)
+  target_link_libraries(flight_sql_test_app PRIVATE ${ARROW_FLIGHT_SQL_TEST_LINK_LIBS}
+                                                    ${GFLAGS_LIBRARIES})
+endif()

cpp/src/arrow/flight/sql/api.h

@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "arrow/flight/sql/client.h"

cpp/src/arrow/flight/sql/arrow-flight-sql.pc.in

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
+
+Name: Apache Arrow Flight SQL
+Description: Apache Arrow Flight SQL extension
+Version: @ARROW_VERSION@
+Requires: arrow-flight
+Libs: -L${libdir} -larrow_flight_sql