ARROW-37: [C++ / Python] Implement BooleanArray and BooleanBuilder. Handle Python built-in bool

.travis.yml

@@ -20,10 +20,10 @@ matrix:
     language: cpp
     os: linux
     before_script:
-    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
-    script:
     - export CC="gcc-4.9"
     - export CXX="g++-4.9"
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
   - compiler: clang

cpp/src/arrow/api.h

@@ -27,7 +27,6 @@
 #include "arrow/table.h"
 #include "arrow/type.h"
 
-#include "arrow/types/boolean.h"
 #include "arrow/types/construct.h"
 #include "arrow/types/list.h"
 #include "arrow/types/primitive.h"

cpp/src/arrow/array-test.cc

@@ -71,8 +71,7 @@ TEST_F(TestArray, TestIsNull) {
     if (x == 0) ++null_count;
   }
 
-  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap.data(),
-      null_bitmap.size());
+  std::shared_ptr<Buffer> null_buf = test::bytes_to_null_buffer(null_bitmap);
   std::unique_ptr<Array> arr;
   arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_count, null_buf));
 
@@ -82,7 +81,7 @@ TEST_F(TestArray, TestIsNull) {
   ASSERT_TRUE(arr->null_bitmap()->Equals(*null_buf.get()));
 
   for (size_t i = 0; i < null_bitmap.size(); ++i) {
-    EXPECT_EQ(static_cast<bool>(null_bitmap[i]), !arr->IsNull(i)) << i;
+    EXPECT_EQ(null_bitmap[i], !arr->IsNull(i)) << i;
   }
 }
 

cpp/src/arrow/builder.cc

@@ -54,5 +54,12 @@ Status ArrayBuilder::Advance(int32_t elements) {
   return Status::OK();
 }
 
+Status ArrayBuilder::Reserve(int32_t elements) {
+  if (length_ + elements > capacity_) {
+    int32_t new_capacity = util::next_power2(length_ + elements);
+    return Resize(new_capacity);
+  }
+  return Status::OK();
+}
 
 } // namespace arrow

cpp/src/arrow/builder.h

@@ -69,6 +69,8 @@ class ArrayBuilder {
   // Resizes the null_bitmap array
   Status Resize(int32_t new_bits);
 
+  Status Reserve(int32_t extra_bits);
+
   // For cases where raw data was memcpy'd into the internal buffers, allows us
   // to advance the length of the builder. It is your responsibility to use
   // this function responsibly.

cpp/src/arrow/test-util.h

@@ -98,28 +98,27 @@ void randint(int64_t N, T lower, T upper, std::vector<T>* out) {
 }
 
 
+template <typename T>
+void random_real(int n, uint32_t seed, T min_value, T max_value,
+    std::vector<T>* out) {
+  std::mt19937 gen(seed);
+  std::uniform_real_distribution<T> d(min_value, max_value);
+  for (int i = 0; i < n; ++i) {
+    out->push_back(d(gen));
+  }
+}
+
+
 template <typename T>
 std::shared_ptr<Buffer> to_buffer(const std::vector<T>& values) {
   return std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(values.data()),
       values.size() * sizeof(T));
 }
 
-void random_null_bitmap(int64_t n, double pct_null, std::vector<uint8_t>* null_bitmap) {
-  Random rng(random_seed());
-  for (int i = 0; i < n; ++i) {
-    if (rng.NextDoubleFraction() > pct_null) {
-      null_bitmap->push_back(1);
-    } else {
-      // null
-      null_bitmap->push_back(0);
-    }
-  }
-}
-
-void random_null_bitmap(int64_t n, double pct_null, std::vector<bool>* null_bitmap) {
+void random_null_bitmap(int64_t n, double pct_null, uint8_t* null_bitmap) {
   Random rng(random_seed());
   for (int i = 0; i < n; ++i) {
-    null_bitmap->push_back(rng.NextDoubleFraction() > pct_null);
+    null_bitmap[i] = rng.NextDoubleFraction() > pct_null;
   }
 }
 
@@ -160,11 +159,11 @@ static inline int null_count(const std::vector<uint8_t>& valid_bytes) {
   return result;
 }
 
-std::shared_ptr<Buffer> bytes_to_null_buffer(uint8_t* bytes, int length) {
+std::shared_ptr<Buffer> bytes_to_null_buffer(const std::vector<uint8_t>& bytes) {
   std::shared_ptr<Buffer> out;
 
   // TODO(wesm): error checking
-  util::bytes_to_bits(bytes, length, &out);
+  util::bytes_to_bits(bytes, &out);
   return out;
 }
 

cpp/src/arrow/type.h

@@ -132,6 +132,10 @@ struct DataType {
     return children_.size();
   }
 
+  virtual int value_size() const {
+    return -1;
+  }
+
   virtual std::string ToString() const = 0;
 };
 
@@ -191,11 +195,14 @@ inline std::string PrimitiveType<Derived>::ToString() const {
 #define PRIMITIVE_DECL(TYPENAME, C_TYPE, ENUM, SIZE, NAME)  \
   typedef C_TYPE c_type;                                    \
   static constexpr Type::type type_enum = Type::ENUM;       \
-  static constexpr int size = SIZE;                         \
                                                             \
   TYPENAME()                                                \
       : PrimitiveType<TYPENAME>() {}                        \
                                                             \
+  virtual int value_size() const {                          \
+    return SIZE;                                            \
+  }                                                         \
+                                                            \
   static const char* name() {                               \
     return NAME;                                            \
   }
@@ -295,6 +302,12 @@ struct StructType : public DataType {
   std::string ToString() const override;
 };
 
+// These will be defined elsewhere
+template <typename T>
+struct type_traits {
+};
+
+
 } // namespace arrow
 
 #endif  // ARROW_TYPE_H

cpp/src/arrow/types/CMakeLists.txt

@@ -21,7 +21,6 @@
 
 # Headers: top level
 install(FILES
-  boolean.h
   collection.h
   construct.h
   datetime.h

cpp/src/arrow/types/construct.cc

@@ -51,7 +51,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
     BUILDER_CASE(UINT64, UInt64Builder);
     BUILDER_CASE(INT64, Int64Builder);
 
-    // BUILDER_CASE(BOOL, BooleanBuilder);
+    BUILDER_CASE(BOOL, BooleanBuilder);
 
     BUILDER_CASE(FLOAT, FloatBuilder);
     BUILDER_CASE(DOUBLE, DoubleBuilder);
@@ -83,6 +83,7 @@ Status MakePrimitiveArray(const std::shared_ptr<DataType>& type,
     int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap,
     std::shared_ptr<Array>* out) {
   switch (type->type) {
+    MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
     MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
     MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);

cpp/src/arrow/types/list-test.cc

@@ -116,11 +116,14 @@ TEST_F(TestListBuilder, TestBasics) {
 
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
 
+  EXPECT_OK(builder_->Reserve(lengths.size()));
+  EXPECT_OK(vb->Reserve(values.size()));
+
   int pos = 0;
   for (size_t i = 0; i < lengths.size(); ++i) {
     ASSERT_OK(builder_->Append(is_null[i] > 0));
     for (int j = 0; j < lengths[i]; ++j) {
-      ASSERT_OK(vb->Append(values[pos++]));
+      vb->Append(values[pos++]);
     }
   }
 

cpp/src/arrow/types/list.h

@@ -116,7 +116,8 @@ class ListBuilder : public Int32Builder {
       int32_t new_capacity = util::next_power2(length_ + length);
       RETURN_NOT_OK(Resize(new_capacity));
     }
-    memcpy(raw_buffer() + length_, values, length * elsize_);
+    memcpy(raw_data_ + length_, values,
+        type_traits<Int32Type>::bytes_required(length));
 
     if (valid_bytes != nullptr) {
       AppendNulls(valid_bytes, length);
@@ -132,13 +133,13 @@ class ListBuilder : public Int32Builder {
 
     // Add final offset if the length is non-zero
     if (length_) {
-      raw_buffer()[length_] = items->length();
+      raw_data_[length_] = items->length();
     }
 
-    auto result = std::make_shared<Container>(type_, length_, values_, items,
+    auto result = std::make_shared<Container>(type_, length_, data_, items,
         null_count_, null_bitmap_);
 
-    values_ = null_bitmap_ = nullptr;
+    data_ = null_bitmap_ = nullptr;
     capacity_ = length_ = null_count_ = 0;
 
     return result;
@@ -162,7 +163,7 @@ class ListBuilder : public Int32Builder {
     } else {
       util::set_bit(null_bitmap_data_, length_);
     }
-    raw_buffer()[length_++] = value_builder_->length();
+    raw_data_[length_++] = value_builder_->length();
     return Status::OK();
   }