Skip to content

Commit

Permalink
Fixed copy semantic of arrays (#244)
Browse files Browse the repository at this point in the history 
* Fixed copy semantic of non composed layouts

* Fixed copy semantic of list arrays

* Fixed copy semantic of struct layout

* Fixed copy semantic of run_end_encoded_array

* Fixed copy semantic of union layouts

* Fixed build on Exotic architectures

* Changes according to the review
  • Loading branch information
@JohanMabille
JohanMabille authored Oct 22, 2024
1 parent 68335e5 commit e604599
Show file tree
Hide file tree
Showing 12 changed files with 661 additions and 276 deletions.
3 changes: 3 additions & 0 deletions include/sparrow/arrow_interface/arrow_array_schema_utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ namespace sparrow
child->release(child);
}
}
// TODO: We assume Arrow structures allocated inside sparrow always use operator new
//delete child;
//t.children[i] = nullptr;
}
delete[] t.children;
t.children = nullptr;
Expand Down
56 changes: 25 additions & 31 deletions include/sparrow/layout/layout_utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,45 +18,31 @@

namespace sparrow::detail
{

template<class LAYOUT_TYPE>
class layout_functor_base
{
public:
using layout_type = LAYOUT_TYPE;
constexpr layout_functor_base() = default;
constexpr layout_functor_base& operator=(layout_functor_base&&) = default;
constexpr layout_functor_base(const layout_functor_base&) = default;
constexpr layout_functor_base(layout_functor_base&&) = default;
constexpr layout_functor_base& operator=(const layout_functor_base&) = default;

constexpr layout_functor_base(layout_type * layout)
: p_layout(layout)
{
}

protected:
layout_type * p_layout = nullptr;
};


// Functor to get the value of the layout at index i.
//
// This is usefull to create a iterator over the values of a layout.
// This functor will be passed to the functor_index_iterator.
template<class LAYOUT_TYPE, class VALUE_TYPE>
class layout_value_functor : public layout_functor_base<LAYOUT_TYPE>
class layout_value_functor
{
public:
using base_type = layout_functor_base<LAYOUT_TYPE>;
using base_type::base_type;
using base_type::operator=;
public:

using value_type = VALUE_TYPE;
using layout_type = LAYOUT_TYPE;

constexpr explicit layout_value_functor(layout_type* layout = nullptr)
: p_layout(layout)
{
}

value_type operator()(std::size_t i) const
{
return this->p_layout->value(i);
}

private:

layout_type* p_layout;
};


Expand All @@ -65,18 +51,26 @@ namespace sparrow::detail
// This is usefull to create a iterator over the nullable-values of a layout.
// This functor will be passed to the functor_index_iterator.
template<class LAYOUT_TYPE, class VALUE_TYPE>
class layout_bracket_functor : public layout_functor_base<LAYOUT_TYPE>
class layout_bracket_functor
{
public:
using base_type = layout_functor_base<LAYOUT_TYPE>;
using base_type::base_type;
using base_type::operator=;

using value_type = VALUE_TYPE;
using layout_type = LAYOUT_TYPE;

constexpr explicit layout_bracket_functor(layout_type* layout = nullptr)
: p_layout(layout)
{
}

value_type operator()(std::size_t i) const
{
return this->p_layout->operator[](i);
}

private:

layout_type* p_layout;
};

}; // namespace sparrow::detail
152 changes: 128 additions & 24 deletions include/sparrow/layout/list_layout/list_array.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,16 @@ namespace sparrow
using const_reference = nullable<inner_const_reference, bitmap_const_reference>;
using iterator_tag = typename base_type::iterator_tag;

explicit list_array_crtp_base(arrow_proxy proxy);

const array_wrapper* raw_flat_array() const;
array_wrapper* raw_flat_array();

protected:

explicit list_array_crtp_base(arrow_proxy proxy);

list_array_crtp_base(const self_type&);
list_array_crtp_base& operator=(const self_type&);

private:

using list_size_type = inner_types::list_size_type;
Expand All @@ -144,6 +149,8 @@ namespace sparrow
inner_reference value(size_type i);
inner_const_reference value(size_type i) const;

cloning_ptr<array_wrapper> make_flat_array();

// data members
cloning_ptr<array_wrapper> p_flat_array;

Expand All @@ -169,11 +176,16 @@ namespace sparrow

explicit list_array_impl(arrow_proxy proxy);

list_array_impl(const self_type&);
list_array_impl& operator=(const self_type&);

private:

static constexpr std::size_t OFFSET_BUFFER_INDEX = 1;
std::pair<offset_type, offset_type> offset_range(size_type i) const;

offset_type* make_list_offsets();

offset_type* p_list_offsets;

// friend classes
Expand All @@ -195,12 +207,18 @@ namespace sparrow

explicit list_view_array_impl(arrow_proxy proxy);

list_view_array_impl(const self_type&);
list_view_array_impl& operator=(const self_type&);

private:

static constexpr std::size_t OFFSET_BUFFER_INDEX = 1;
static constexpr std::size_t SIZES_BUFFER_INDEX = 2;
std::pair<offset_type, offset_type> offset_range(size_type i) const;

offset_type* make_list_offsets();
offset_type* make_list_sizes();

offset_type* p_list_offsets;
offset_type* p_list_sizes;

Expand All @@ -222,6 +240,9 @@ namespace sparrow

explicit fixed_sized_list_array(arrow_proxy proxy);

fixed_sized_list_array(const self_type&) = default;
fixed_sized_list_array& operator=(const self_type&) = default;

private:

static uint64_t list_size_from_format(const std::string_view format);
Expand All @@ -234,13 +255,32 @@ namespace sparrow
friend class list_array_crtp_base<self_type>;
};

/***************************************
* list_array_crtp_base implementation *
***************************************/

template <class DERIVED>
list_array_crtp_base<DERIVED>::list_array_crtp_base(arrow_proxy proxy)
: base_type(std::move(proxy))
, p_flat_array(array_factory(this->storage().children()[0].view()))
, p_flat_array(make_flat_array())
{
}

template <class DERIVED>
list_array_crtp_base<DERIVED>::list_array_crtp_base(const self_type& rhs)
: base_type(rhs)
, p_flat_array(make_flat_array())
{
}

template <class DERIVED>
auto list_array_crtp_base<DERIVED>::operator=(const self_type& rhs) -> self_type&
{
base_type::operator=(rhs);
p_flat_array = make_flat_array();
return *this;
}

template <class DERIVED>
auto list_array_crtp_base<DERIVED>::raw_flat_array() const -> const array_wrapper*
{
Expand Down Expand Up @@ -302,51 +342,91 @@ namespace sparrow
return list_value{p_flat_array.get(), static_cast<st>(r.first), static_cast<st>(r.second)};
}

template <class DERIVED>
cloning_ptr<array_wrapper> list_array_crtp_base<DERIVED>::make_flat_array()
{
return array_factory(this->storage().children()[0].view());
}

/**********************************
* list_array_impl implementation *
**********************************/

#ifdef __GNUC__
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wcast-align"
#endif


template <bool BIG>
inline list_array_impl<BIG>::list_array_impl(arrow_proxy proxy)
list_array_impl<BIG>::list_array_impl(arrow_proxy proxy)
: base_type(std::move(proxy))
, p_list_offsets(reinterpret_cast<offset_type*>(
this->storage().buffers()[OFFSET_BUFFER_INDEX].data() + this->storage().offset()
))
, p_list_offsets(make_list_offsets())
{
}
#ifdef __GNUC__
# pragma GCC diagnostic pop
#endif

template <bool BIG>
list_array_impl<BIG>::list_array_impl(const self_type& rhs)
: base_type(rhs)
, p_list_offsets(make_list_offsets())
{
}

template <bool BIG>
auto list_array_impl<BIG>::operator=(const self_type& rhs) -> self_type&
{
if (this != &rhs)
{
base_type::operator=(rhs);
p_list_offsets = make_list_offsets();
}
return *this;
}

template <bool BIG>
auto list_array_impl<BIG>::offset_range(size_type i) const -> std::pair<offset_type, offset_type>
{
return std::make_pair(p_list_offsets[i], p_list_offsets[i + 1]);
}

#ifdef __GNUC__
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wcast-align"
#endif
template <bool BIG>
auto list_array_impl<BIG>::make_list_offsets() -> offset_type*
{
return reinterpret_cast<offset_type*>(
this->storage().buffers()[OFFSET_BUFFER_INDEX].data() + this->storage().offset()
);
}

/***************************************
* list_view_array_impl implementation *
***************************************/

template <bool BIG>
inline list_view_array_impl<BIG>::list_view_array_impl(arrow_proxy proxy)
: base_type(std::move(proxy))
, p_list_offsets(reinterpret_cast<offset_type*>(
this->storage().buffers()[OFFSET_BUFFER_INDEX].data() + this->storage().offset()
))
, p_list_sizes(reinterpret_cast<offset_type*>(
this->storage().buffers()[SIZES_BUFFER_INDEX].data() + this->storage().offset()
))
, p_list_offsets(make_list_offsets())
, p_list_sizes(make_list_sizes())
{
}

#ifdef __GNUC__
# pragma GCC diagnostic pop
#endif
template <bool BIG>
list_view_array_impl<BIG>::list_view_array_impl(const self_type& rhs)
: base_type(rhs)
, p_list_offsets(make_list_offsets())
, p_list_sizes(make_list_sizes())
{
}

template <bool BIG>
auto list_view_array_impl<BIG>::operator=(const self_type& rhs) -> self_type&
{
if (this != &rhs)
{
base_type::operator=(rhs);
p_list_offsets = make_list_offsets();
p_list_sizes = make_list_sizes();
}
return *this;
}

template <bool BIG>
inline auto
Expand All @@ -356,6 +436,30 @@ namespace sparrow
return std::make_pair(offset, offset + p_list_sizes[i]);
}

template <bool BIG>
auto list_view_array_impl<BIG>::make_list_offsets() -> offset_type*
{
return reinterpret_cast<offset_type*>(
this->storage().buffers()[OFFSET_BUFFER_INDEX].data() + this->storage().offset()
);
}

template <bool BIG>
auto list_view_array_impl<BIG>::make_list_sizes() -> offset_type*
{
return reinterpret_cast<offset_type*>(
this->storage().buffers()[SIZES_BUFFER_INDEX].data() + this->storage().offset()
);
}

#ifdef __GNUC__
# pragma GCC diagnostic pop
#endif

/*****************************************
* fixed_sized_list_array implementation *
*****************************************/

inline auto fixed_sized_list_array::list_size_from_format(const std::string_view format) -> uint64_t
{
SPARROW_ASSERT(format.size() >= 3, "Invalid format string");
Expand Down
Loading

0 comments on commit e604599

Please sign in to comment.