Adding operator+[=] and operator/[=] for FloatingArray and IntegerArray.

This includes a design change that obviates the need for an ArrayView.
Instead, every array has an internal offset. Shallow copy is achieved by copy
constructor, though the current set of copy constructors don't yet
support a slice. Deep copy is still achieved through the Copy virtual
function.

More detailed explanation of the changes:

* Adding copy/move constructors for {Floating,Integer,Numeric}Array
* Adding various method for marking/getting nulls (valid bits) in
  integer arrays
* Changing data() and mutable_data() in NumericArray so that they
  return a pointer that starts at the array's offset
* Addition of Addable/Divisable classes (similar to Boost operators)
  for easy support of operator[+/]
* Implementing IntegerArray::operator/, IntegerArray::operator+=
  FloatingArray::operator/=, FloatingArray::operator+=
* Adding meta programming classes TypeList and CartesianProduct.
  The classes are used as scaffolding in array-test to simplify the
  declaration of every permutation of arithmetic operation for all types.

Author: joshuastorck

CMakeLists.txt

@@ -74,7 +74,7 @@ endif()
 #     GCC cannot always verify whether strict aliasing rules are indeed followed due to
 #     fundamental limitations in escape analysis, which can result in subtle bad code generation.
 #     This has a small perf hit but worth it to avoid hard to debug crashes.
-set(CXX_COMMON_FLAGS "-std=c++11 -fno-strict-aliasing -msse4.2 -Wall -Wno-sign-compare -Wno-deprecated -pthread -D__STDC_FORMAT_MACROS")
+set(CXX_COMMON_FLAGS "-std=c++1y -fno-strict-aliasing -msse4.2 -Wall -Wno-sign-compare -Wno-deprecated -pthread -D__STDC_FORMAT_MACROS")
 
 # compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
 # For all builds:
@@ -642,6 +642,7 @@ set(PANDAS_TEST_LINK_LIBS ${PANDAS_MIN_TEST_LIBS})
 ############################################################
 
 add_subdirectory(src/pandas)
+add_subdirectory(src/pandas/meta)
 add_subdirectory(src/pandas/util)
 
 set(PANDAS_SRCS

pandas/native.pxd

@@ -74,7 +74,7 @@ cdef extern from "pandas/api.h" namespace "pandas":
         c_bool Equals(const DataType& other)
         string ToString()
 
-    ctypedef shared_ptr[DataType] TypePtr
+    ctypedef shared_ptr[const DataType] TypePtr
 
     cdef cppclass Int8Type(DataType):
         pass

pandas/native.pyx

@@ -277,7 +277,7 @@ cdef Array wrap_array(const lp.ArrayPtr& arr):
 
 cdef PandasType wrap_type(const lp.TypePtr& sp_type):
     cdef:
-        lp.DataType* type = sp_type.get()
+        const lp.DataType* type = sp_type.get()
         PandasType result
 
     if type.type() == lp.TypeId_CATEGORY:

src/pandas/array-test.cc

@@ -12,6 +12,7 @@
 #include "pandas/array.h"
 #include "pandas/common.h"
 #include "pandas/memory.h"
+#include "pandas/meta/cartesian_product.h"
 #include "pandas/test-util.h"
 #include "pandas/type.h"
 #include "pandas/types/numeric.h"
@@ -45,6 +46,133 @@ TEST_F(TestArray, Attrs) {
   ASSERT_EQ(values_.size(), array_->length());
 }
 
+template <typename LEFT_ARRAY_TYPE, typename RIGHT_ARRAY_TYPE, std::size_t LENGTH = 10>
+class OperatorTestData {
+ public:
+  OperatorTestData()
+      : left_buffer_(std::make_shared<Buffer>(
+            reinterpret_cast<const std::uint8_t*>(Initialize(left_data_)),
+            LENGTH * sizeof(LEFT_C_TYPE))),
+        right_buffer_(std::make_shared<Buffer>(
+            reinterpret_cast<const std::uint8_t*>(Initialize(right_data_)),
+            LENGTH * sizeof(RIGHT_C_TYPE))),
+        left_array_(LENGTH, left_buffer_),
+        right_array_(LENGTH, right_buffer_) {}
+
+  template <typename C_TYPE>
+  static C_TYPE* Initialize(C_TYPE (&value)[LENGTH]) {
+    for (auto ii = 0; ii < LENGTH; ++ii) {
+      // Start at 1 so that we don't get FPE with operator/
+      value[ii] = static_cast<C_TYPE>(ii + 1);
+    }
+    return value;
+  }
+
+  using LEFT_C_TYPE = typename LEFT_ARRAY_TYPE::T;
+
+  using RIGHT_C_TYPE = typename RIGHT_ARRAY_TYPE::T;
+
+  LEFT_C_TYPE left_data_[LENGTH];
+
+  RIGHT_C_TYPE right_data_[LENGTH];
+
+  std::shared_ptr<Buffer> left_buffer_;
+
+  std::shared_ptr<Buffer> right_buffer_;
+
+  LEFT_ARRAY_TYPE left_array_;
+
+  RIGHT_ARRAY_TYPE right_array_;
+};
+
+template <typename OPERATOR, typename INPLACE_OPERATOR, std::size_t LENGTH = 10>
+class TestInplaceOperator {
+ public:
+  TestInplaceOperator(OPERATOR& operation, INPLACE_OPERATOR& inplace_operation)
+      : operation_(operation), inplace_operation_(inplace_operation) {}
+
+  template <typename PAIR>
+  void operator()() {
+    using FIRST = typename std::tuple_element<0, PAIR>::type;
+    using SECOND = typename std::tuple_element<1, PAIR>::type;
+    OperatorTestData<FIRST, SECOND> test_data;
+    auto result = operation_(test_data.left_array_, test_data.right_array_);
+    for (auto ii = 0; ii < test_data.left_array_.length(); ++ii) {
+      ASSERT_EQ(result.data()[ii],
+          operation_(test_data.left_data_[ii], test_data.right_data_[ii]));
+    }
+    inplace_operation_(test_data.left_array_, test_data.right_array_);
+    for (auto ii = 0; ii < test_data.left_array_.length(); ++ii) {
+      ASSERT_EQ(test_data.left_array_.data()[ii],
+          operation_(test_data.left_data_[ii], test_data.right_data_[ii]));
+    }
+    for (auto ii = 0; ii < test_data.left_array_.length(); ++ii) {
+      ASSERT_EQ(test_data.left_array_.data()[ii], result.data()[ii]);
+    }
+  }
+
+ private:
+  OPERATOR& operation_;
+
+  INPLACE_OPERATOR inplace_operation_;
+};
+
+template <typename OPERATOR, std::size_t LENGTH = 10>
+class TestOperator {
+ public:
+  TestOperator(OPERATOR& operation) : operation_(operation) {}
+
+  template <typename PAIR>
+  void operator()() {
+    using FIRST = typename std::tuple_element<0, PAIR>::type;
+    using SECOND = typename std::tuple_element<1, PAIR>::type;
+    OperatorTestData<FIRST, SECOND> test_data;
+    auto result = operation_(test_data.left_array_, test_data.right_array_);
+    for (auto ii = 0; ii < test_data.left_array_.length(); ++ii) {
+      ASSERT_EQ(result.data()[ii],
+          operation_(test_data.left_data_[ii], test_data.right_data_[ii]));
+    }
+  }
+
+ private:
+  OPERATOR& operation_;
+};
+
+using IntegerTypes = TypeList<IntegerArray<UInt8Type>, IntegerArray<UInt16Type>,
+    IntegerArray<UInt32Type>, IntegerArray<UInt64Type>, IntegerArray<Int8Type>,
+    IntegerArray<Int16Type>, IntegerArray<Int32Type>, IntegerArray<Int64Type>>;
+
+using FloatingPointTypes = TypeList<FloatingArray<FloatType>, FloatingArray<DoubleType>>;
+
+using NumericTypes = decltype(IntegerTypes() + FloatingPointTypes());
+
+TEST(TestArrayOperators, Addition) {
+  auto plus = [](auto const& left, auto const& right) { return left + right; };
+  auto plus_inplace = [](auto& left, auto const& right) { left += right; };
+
+  using AdditionTests =
+      decltype(typename CartesianProduct<FloatingPointTypes, NumericTypes>::type() +
+               CartesianProduct<IntegerTypes, IntegerTypes>::type());
+
+  TestInplaceOperator<decltype(plus), decltype(plus_inplace)> tester(plus, plus_inplace);
+  AdditionTests::iterate(tester);
+}
+
+TEST(TestArrayOperators, Division) {
+  auto divide = [](auto const& left, auto const& right) { return left / right; };
+  auto divide_inplace = [](auto& left, auto const& right) { left /= right; };
+
+  using DivisionTests = typename CartesianProduct<IntegerTypes>::type;
+  TestOperator<decltype(divide)> test(divide);
+  DivisionTests::iterate(test);
+
+  using InplaceDivisionTests =
+      typename CartesianProduct<FloatingPointTypes, NumericTypes>::type;
+  TestInplaceOperator<decltype(divide), decltype(divide_inplace)> inplace_test(
+      divide, divide_inplace);
+  InplaceDivisionTests::iterate(inplace_test);
+}
+
 // ----------------------------------------------------------------------
 // Array view object
 

src/pandas/array.cc

@@ -11,11 +11,8 @@ namespace pandas {
 // ----------------------------------------------------------------------
 // Array
 
-Array::Array(const std::shared_ptr<DataType>& type, int64_t length)
-    : type_(type), length_(length) {}
-
 Status Array::Copy(std::shared_ptr<Array>* out) const {
-  return Copy(0, length_, out);
+  return Copy(0, length(), out);
 }
 
 // ----------------------------------------------------------------------

src/pandas/array.h

@@ -20,9 +20,19 @@ class Array {
  public:
   virtual ~Array() {}
 
-  int64_t length() const { return length_; }
-  std::shared_ptr<DataType> type() const { return type_; }
-  DataType::TypeId type_id() const { return type_->type(); }
+  virtual int64_t length() const = 0;
+  // There are two methods to obtain the data type.
+  // The signature without a shared_ptr allows sub-classes
+  // to have a covariant return type, which eliminates the
+  // need/danger of doing a static_cast when dealing with
+  // a concrete sub-class. Ideally, the shared_ptr signature
+  // would suffice, but the compiler cannot treat a shared_ptr
+  // to a base class and a shared_ptr to a subclass as a
+  // covariant return type.
+  virtual TypePtr type() const = 0;
+  virtual const DataType& type_reference() const = 0;
+
+  DataType::TypeId type_id() const { return type()->type(); }
 
   // Copy a section of the array into a new output array
   virtual Status Copy(
@@ -42,13 +52,10 @@ class Array {
   virtual bool owns_data() const = 0;
 
  protected:
-  std::shared_ptr<DataType> type_;
-  int64_t length_;
+  Array() {}
 
-  Array(const std::shared_ptr<DataType>& type, int64_t length);
-
- private:
-  DISALLOW_COPY_AND_ASSIGN(Array);
+  Array(const Array& other) = default;
+  Array(Array&& other) = default;
 };
 
 // An object that is a view on a section of another array (possibly the whole

src/pandas/meta/CMakeLists.txt

@@ -0,0 +1,11 @@
+# This file is a part of pandas. See LICENSE for details about reuse and
+# copyright holders
+
+#######################################
+# Unit tests
+#######################################
+
+set(PANDAS_TEST_LINK_LIBS pandas_test_util ${PANDAS_MIN_TEST_LIBS})
+
+ADD_PANDAS_TEST(typelist-test)
+ADD_PANDAS_TEST(cartesian_product-test)

src/pandas/meta/cartesian_product-test.cc

@@ -0,0 +1,79 @@
+// This file is a part of pandas. See LICENSE for details about reuse and
+// copyright holders
+
+#include <cstdint>
+
+#include "gtest/gtest.h"
+
+#include "pandas/meta/cartesian_product.h"
+
+namespace pandas {
+
+class TestCartesianProduct : public ::testing::Test {
+ public:
+  void SetUp() {
+    saw_i_f_ = saw_i_i_ = saw_f_i_ = saw_f_f_ = false;
+    i_f_index_ = i_i_index_ = f_i_index_ = f_f_index_ = -1;
+    count_ = 0;
+  }
+
+  template <typename T>
+  void operator()() {
+    if (std::is_same<T, std::tuple<int32_t, float>>::value) {
+      saw_i_f_ = true;
+      i_f_index_ = count_++;
+    } else if (std::is_same<T, std::tuple<int32_t, int32_t>>::value) {
+      saw_i_i_ = true;
+      i_i_index_ = count_++;
+    } else if (std::is_same<T, std::tuple<float, int32_t>>::value) {
+      saw_f_i_ = true;
+      f_i_index_ = count_++;
+    } else if (std::is_same<T, std::tuple<float, float>>::value) {
+      saw_f_f_ = true;
+      f_f_index_ = count_++;
+    }
+  }
+
+ protected:
+  bool saw_i_f_;
+  bool saw_i_i_;
+  bool saw_f_i_;
+  bool saw_f_f_;
+
+  int64_t i_f_index_;
+  int64_t i_i_index_;
+  int64_t f_i_index_;
+  int64_t f_f_index_;
+
+  int64_t count_;
+};
+
+TEST_F(TestCartesianProduct, SingleTypeList) {
+  using Product = typename CartesianProduct<TypeList<int32_t, float>>::type;
+  Product::iterate(*this);
+  ASSERT_TRUE(saw_i_i_);
+  ASSERT_TRUE(saw_i_f_);
+  ASSERT_TRUE(saw_f_i_);
+  ASSERT_TRUE(saw_f_f_);
+
+  ASSERT_EQ(i_i_index_, 0);
+  ASSERT_EQ(i_f_index_, 1);
+  ASSERT_EQ(f_i_index_, 2);
+  ASSERT_EQ(f_f_index_, 3);
+}
+
+TEST_F(TestCartesianProduct, MultipleTypeList) {
+  using Product =
+      typename CartesianProduct<TypeList<int32_t, float>, TypeList<float>>::type;
+  Product::iterate(*this);
+  ASSERT_TRUE(saw_i_f_);
+  ASSERT_TRUE(saw_f_f_);
+  ASSERT_FALSE(saw_i_i_);
+  ASSERT_FALSE(saw_f_i_);
+
+  ASSERT_EQ(i_f_index_, 0);
+  ASSERT_EQ(f_f_index_, 1);
+  ASSERT_EQ(i_i_index_, -1);
+  ASSERT_EQ(f_i_index_, -1);
+}
+}