Skip to content

ARROW-371: Handle pandas-nullable types correctly #205

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
xhochy wants to merge 1 commit into from
Closed

ARROW-371: Handle pandas-nullable types correctly #205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion python/pyarrow/tests/test_convert_pandas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ def test_strings(self):
expected = pd.DataFrame({'strings': values * repeats})
self._check_pandas_roundtrip(df, expected)

def test_timestamps_notimezone(self):
def test_timestamps_notimezone_no_nulls(self):
df = pd.DataFrame({
'datetime64': np.array([
'2007-07-13T01:23:34.123',
Expand All @@ -184,6 +184,26 @@ def test_timestamps_notimezone(self):
})
self._check_pandas_roundtrip(df, timestamps_to_ms=False)

def test_timestamps_notimezone_nulls(self):
df = pd.DataFrame({
'datetime64': np.array([
'2007-07-13T01:23:34.123',
None,
'2010-08-13T05:46:57.437'],
dtype='datetime64[ms]')
})
df.info()
self._check_pandas_roundtrip(df, timestamps_to_ms=True)

df = pd.DataFrame({
'datetime64': np.array([
'2007-07-13T01:23:34.123456789',
None,
'2010-08-13T05:46:57.437699912'],
dtype='datetime64[ns]')
})
self._check_pandas_roundtrip(df, timestamps_to_ms=False)

# def test_category(self):
# repeats = 1000
# values = [b'foo', None, u'bar', 'qux', np.nan]
Expand Down
46 changes: 23 additions & 23 deletions python/src/pyarrow/adapters/pandas.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,20 +489,20 @@ struct arrow_traits<arrow::Type::BOOL> {
static constexpr int npy_type = NPY_BOOL;
static constexpr bool supports_nulls = false;
static constexpr bool is_boolean = true;
static constexpr bool is_integer = false;
static constexpr bool is_floating = false;
static constexpr bool is_pandas_numeric_not_nullable = false;
static constexpr bool is_pandas_numeric_nullable = false;
};

#define INT_DECL(TYPE) \
template <> \
struct arrow_traits<arrow::Type::TYPE> { \
static constexpr int npy_type = NPY_##TYPE; \
static constexpr bool supports_nulls = false; \
static constexpr double na_value = NAN; \
static constexpr bool is_boolean = false; \
static constexpr bool is_integer = true; \
static constexpr bool is_floating = false; \
typedef typename npy_traits<NPY_##TYPE>::value_type T; \
#define INT_DECL(TYPE) \
template <> \
struct arrow_traits<arrow::Type::TYPE> { \
static constexpr int npy_type = NPY_##TYPE; \
static constexpr bool supports_nulls = false; \
static constexpr double na_value = NAN; \
static constexpr bool is_boolean = false; \
static constexpr bool is_pandas_numeric_not_nullable = true; \
static constexpr bool is_pandas_numeric_nullable = false; \
typedef typename npy_traits<NPY_##TYPE>::value_type T; \
};

INT_DECL(INT8);
Expand All @@ -520,8 +520,8 @@ struct arrow_traits<arrow::Type::FLOAT> {
static constexpr bool supports_nulls = true;
static constexpr float na_value = NAN;
static constexpr bool is_boolean = false;
static constexpr bool is_integer = false;
static constexpr bool is_floating = true;
static constexpr bool is_pandas_numeric_not_nullable = false;
static constexpr bool is_pandas_numeric_nullable = true;
typedef typename npy_traits<NPY_FLOAT32>::value_type T;
};

Expand All @@ -531,8 +531,8 @@ struct arrow_traits<arrow::Type::DOUBLE> {
static constexpr bool supports_nulls = true;
static constexpr double na_value = NAN;
static constexpr bool is_boolean = false;
static constexpr bool is_integer = false;
static constexpr bool is_floating = true;
static constexpr bool is_pandas_numeric_not_nullable = false;
static constexpr bool is_pandas_numeric_nullable = true;
typedef typename npy_traits<NPY_FLOAT64>::value_type T;
};

Expand All @@ -542,8 +542,8 @@ struct arrow_traits<arrow::Type::TIMESTAMP> {
static constexpr bool supports_nulls = true;
static constexpr int64_t na_value = std::numeric_limits<int64_t>::min();
static constexpr bool is_boolean = false;
static constexpr bool is_integer = true;
static constexpr bool is_floating = false;
static constexpr bool is_pandas_numeric_not_nullable = false;
static constexpr bool is_pandas_numeric_nullable = true;
typedef typename npy_traits<NPY_DATETIME>::value_type T;
};

Expand All @@ -552,8 +552,8 @@ struct arrow_traits<arrow::Type::STRING> {
static constexpr int npy_type = NPY_OBJECT;
static constexpr bool supports_nulls = true;
static constexpr bool is_boolean = false;
static constexpr bool is_integer = false;
static constexpr bool is_floating = false;
static constexpr bool is_pandas_numeric_not_nullable = false;
static constexpr bool is_pandas_numeric_nullable = false;
};


Expand Down Expand Up @@ -655,7 +655,7 @@ class ArrowDeserializer {

template <int T2>
inline typename std::enable_if<
arrow_traits<T2>::is_floating, Status>::type
arrow_traits<T2>::is_pandas_numeric_nullable, Status>::type
ConvertValues(const std::shared_ptr<Array>& arr) {
typedef typename arrow_traits<T2>::T T;

Expand All @@ -668,7 +668,7 @@ class ArrowDeserializer {

T* out_values = reinterpret_cast<T*>(PyArray_DATA(out_));
for (int64_t i = 0; i < arr->length(); ++i) {
out_values[i] = arr->IsNull(i) ? NAN : in_values[i];
out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i];
}
} else {
// Zero-Copy. We can pass the data pointer directly to NumPy.
Expand All @@ -683,7 +683,7 @@ class ArrowDeserializer {
// Integer specialization
template <int T2>
inline typename std::enable_if<
arrow_traits<T2>::is_integer, Status>::type
arrow_traits<T2>::is_pandas_numeric_not_nullable, Status>::type
ConvertValues(const std::shared_ptr<Array>& arr) {
typedef typename arrow_traits<T2>::T T;

Expand Down