From 1d7fbd3c577ae9fa4c69de522b1192398295b3cb Mon Sep 17 00:00:00 2001 From: Daniel Poetzl Date: Thu, 5 Apr 2018 12:08:15 +0100 Subject: [PATCH 1/2] Refactor sharing map nodes to reduce memory consumption --- src/util/sharing_map.h | 515 +++++++++++++++++++------------------ src/util/sharing_node.h | 488 +++++++++++++++++++++-------------- unit/Makefile | 1 + unit/util/sharing_map.cpp | 72 +++++- unit/util/sharing_node.cpp | 234 +++++++++++------ 5 files changed, 799 insertions(+), 511 deletions(-) diff --git a/src/util/sharing_map.h b/src/util/sharing_map.h index d33624f5f78..18752eba0f4 100644 --- a/src/util/sharing_map.h +++ b/src/util/sharing_map.h @@ -12,30 +12,39 @@ Author: Daniel Poetzl #ifndef CPROVER_UTIL_SHARING_MAP_H #define CPROVER_UTIL_SHARING_MAP_H -#include -#include -#include -#include +#ifdef SM_DEBUG +#include +#endif + #include +#include #include -#include -#include +#include +#include +#include +#include +#include #include "irep.h" #include "sharing_node.h" #include "threeval.h" -#define _sm_assert(b) assert(b) -//#define _sm_assert(b) +#ifdef SM_INTERNAL_CHECKS +#define SM_ASSERT(b) INVARIANT(b, "Sharing map internal invariant") +#else +#define SM_ASSERT(b) +#endif +// clang-format off #define SHARING_MAPT(R) \ - template \ - R sharing_mapt + template \ + R sharing_mapt #define SHARING_MAPT2(CV, ST) \ - template \ - CV typename sharing_mapt::ST \ - sharing_mapt + template \ + CV typename sharing_mapt::ST \ + sharing_mapt +// clang-format on // Note: Due to a bug in Visual Studio we need to add an additional "const" // qualifier to the return values of insert(), place(), and find(). The type @@ -108,11 +117,10 @@ Author: Daniel Poetzl /// /// The first two symbols denote dynamic properties of a given map, whereas the /// last two symbols are static configuration parameters of the map class. -template < - class keyT, - class valueT, - class hashT=std::hash, - class predT=std::equal_to> +template , + class equalT = std::equal_to> class sharing_mapt { public: @@ -130,10 +138,7 @@ class sharing_mapt typedef std::pair value_type; typedef hashT hash; - typedef predT key_equal; - - typedef sharing_mapt self_type; - typedef sharing_nodet node_type; + typedef equalT key_equal; typedef std::size_t size_type; @@ -151,40 +156,23 @@ class sharing_mapt typedef std::vector keyst; - typedef typename node_type::subt subt; - typedef typename node_type::containert containert; - - // key-value map - node_type map; - - // number of elements in the map - size_type num=0; - - // dummy element returned when no element was found - static mapped_type dummy; - - // compile-time configuration - - static const std::string not_found_msg; - - /// Number of bits in the hash deemed significant - static const std::size_t bits; +protected: + typedef sharing_node_innert innert; + typedef sharing_node_leaft leaft; - /// Size of a chunk of the hash that represents a character - static const std::size_t chunk; + typedef sharing_node_baset baset; - static const std::size_t mask; - static const std::size_t steps; + typedef typename innert::to_mapt to_mapt; + typedef typename innert::leaf_listt leaf_listt; +public: // interface - size_type erase( - const key_type &k, - const tvt &key_exists=tvt::unknown()); + size_type erase(const key_type &k, const tvt &key_exists = tvt::unknown()); size_type erase_all( const keyst &ks, - const tvt &key_exists=tvt::unknown()); // applies to all keys + const tvt &key_exists = tvt::unknown()); // applies to all keys // return true if element was inserted const_find_type insert( @@ -220,11 +208,11 @@ class sharing_mapt /// Swap with other map /// /// Complexity: O(1) - void swap(self_type &other) + void swap(sharing_mapt &other) { map.swap(other.map); - size_t tmp=num; + std::size_t tmp = num; num=other.num; other.num=tmp; } @@ -299,100 +287,136 @@ class sharing_mapt void get_view(viewt &view) const; void get_delta_view( - const self_type &other, + const sharing_mapt &other, delta_viewt &delta_view, - const bool only_common=true) const; + const bool only_common = true) const; protected: // helpers - node_type *get_container_node(const key_type &k); - const node_type *get_container_node(const key_type &k) const; + innert *get_container_node(const key_type &k); + const innert *get_container_node(const key_type &k) const; - const node_type *get_leaf_node(const key_type &k) const; + const leaft *get_leaf_node(const key_type &k) const + { + const innert *cp = get_container_node(k); + if(cp == nullptr) + return nullptr; - void gather_all(const node_type &n, delta_viewt &delta_view) const; -}; + const leaft *lp; + lp = cp->find_leaf(k); -/// Get a view of the elements in the map -/// A view is a list of pairs with the components being const references to the -/// keys and values in the map. -/// -/// Complexity: -/// - Worst case: O(N * H * log(S)) -/// - Best case: O(N + H) -/// -/// \param[out] view: Empty view -SHARING_MAPT(void)::get_view(viewt &view) const -{ - assert(view.empty()); + return lp; + } - std::stack stack; + void iterate( + const baset &n, + const unsigned depth, + std::function f) const; - if(empty()) - return; + void gather_all(const baset &n, const unsigned depth, delta_viewt &delta_view) + const; + + // dummy element returned when no element was found + static mapped_type dummy; + + static const std::string not_found_msg; + + // config + static const std::size_t bits; + static const std::size_t chunk; + + // derived config + static const std::size_t mask; + static const std::size_t steps; + + // key-value map + innert map; + + // number of elements in the map + size_type num = 0; +}; - stack.push(&map); +SHARING_MAPT(void) +::iterate( + const baset &n, + unsigned depth, + std::function f) const +{ + typedef std::pair stack_itemt; + + std::stack stack; + stack.push({depth, &n}); do { - const node_type *n=stack.top(); + const stack_itemt &si = stack.top(); + + const unsigned depth = si.first; + const baset *bp = si.second; + stack.pop(); - if(n->is_container()) + if(depth < steps) // internal { - for(const auto &child : n->get_container()) + const innert *ip = static_cast(bp); + const to_mapt &m = ip->get_to_map(); + SM_ASSERT(!m.empty()); + + for(const auto &item : m) { - view.push_back(view_itemt(child.get_key(), child.get_value())); + const innert *i = &item.second; + stack.push({depth + 1, i}); } } - else + else // container { - assert(n->is_internal()); + SM_ASSERT(depth == steps); - for(const auto &child : n->get_sub()) + const innert *cp = static_cast(bp); + const leaf_listt &ll = cp->get_container(); + SM_ASSERT(!ll.empty()); + + for(const auto &l : ll) { - stack.push(&child.second); + f(l.get_key(), l.get_value()); } } } while(!stack.empty()); } -SHARING_MAPT(void)::gather_all(const node_type &n, delta_viewt &delta_view) - const +/// Get a view of the elements in the map +/// A view is a list of pairs with the components being const references to the +/// keys and values in the map. +/// +/// Complexity: +/// - Worst case: O(N * H * log(S)) +/// - Best case: O(N + H) +/// +/// \param[out] view: Empty view +SHARING_MAPT(void)::get_view(viewt &view) const { - std::stack stack; - stack.push(&n); + SM_ASSERT(view.empty()); - do - { - const node_type *n=stack.top(); - stack.pop(); + if(empty()) + return; - if(n->is_container()) - { - for(const auto &child : n->get_container()) - { - delta_view.push_back( - delta_view_itemt( - false, - child.get_key(), - child.get_value(), - dummy)); - } - } - else - { - assert(n->is_internal()); + auto f = [&view](const key_type &k, const mapped_type &m) { + view.push_back(view_itemt(k, m)); + }; - for(const auto &child : n->get_sub()) - { - stack.push(&child.second); - } - } - } - while(!stack.empty()); + iterate(map, 0, f); +} + +SHARING_MAPT(void) +::gather_all(const baset &n, unsigned depth, delta_viewt &delta_view) const +{ + auto f = [&delta_view](const key_type &k, const mapped_type &m) { + delta_view.push_back(delta_view_itemt(false, k, m, dummy)); + }; + + iterate(n, depth, f); } /// Get a delta view of the elements in the map @@ -428,15 +452,13 @@ SHARING_MAPT(void)::gather_all(const node_type &n, delta_viewt &delta_view) /// \param[out] delta_view: Empty delta view /// \param only_common: Indicates if the returned delta view should only /// contain key-value pairs for keys that exist in both maps -SHARING_MAPT(void)::get_delta_view( - const self_type &other, +SHARING_MAPT(void) +::get_delta_view( + const sharing_mapt &other, delta_viewt &delta_view, const bool only_common) const { - assert(delta_view.empty()); - - typedef std::pair stack_itemt; - std::stack stack; + SM_ASSERT(delta_view.empty()); if(empty()) return; @@ -445,140 +467,129 @@ SHARING_MAPT(void)::get_delta_view( { if(!only_common) { - gather_all(map, delta_view); + gather_all(map, 0, delta_view); } + return; } - stack.push(stack_itemt(&map, &other.map)); + typedef std::tuple stack_itemt; + std::stack stack; + + // We do a DFS "in lockstep" simultaneously on both maps. For + // corresponding nodes we check whether they are shared between the + // maps, and if not, we recurse into the corresponding subtrees. + + // The stack contains the children of already visited nodes that we + // still have to visit during the traversal. + + stack.push(stack_itemt(0, &map, &other.map)); do { - const stack_itemt si=stack.top(); - stack.pop(); + const stack_itemt &si = stack.top(); - const node_type *n1=si.first; - const node_type *n2=si.second; + const unsigned depth = std::get<0>(si); + const baset *bp1 = std::get<1>(si); + const baset *bp2 = std::get<2>(si); - if(n1->is_internal()) + stack.pop(); + + if(depth < steps) // internal { - _sn_assert(n2->is_internal()); + const innert *ip1 = static_cast(bp1); + const innert *ip2 = static_cast(bp2); - for(const auto &child : n1->get_sub()) + const to_mapt &m = ip1->get_to_map(); + + for(const auto &item : m) { - const node_type *p; + const innert *p; - p=n2->find_child(child.first); + p = ip2->find_child(item.first); if(p==nullptr) { if(!only_common) { - gather_all(child.second, delta_view); + gather_all(item.second, depth + 1, delta_view); } } - else if(!child.second.shares_with(*p)) + else if(!item.second.shares_with(*p)) { - stack.push(stack_itemt(&child.second, p)); + stack.push(stack_itemt(depth + 1, &item.second, p)); } } } - else if(n1->is_container()) + else // container { - _sn_assert(n2->is_container()); + SM_ASSERT(depth == steps); + + const innert *cp1 = static_cast(bp1); + const innert *cp2 = static_cast(bp2); - for(const auto &l1 : n1->get_container()) + const leaf_listt &ll1 = cp1->get_container(); + + for(const auto &l1 : ll1) { const key_type &k1=l1.get_key(); - bool found=false; - - for(const auto &l2 : n2->get_container()) - { - const key_type &k2=l2.get_key(); + const leaft *p; - if(l1.shares_with(l2)) - { - found=true; - break; - } + p = cp2->find_leaf(k1); - if(key_equal()(k1, k2)) + if(p != nullptr) + { + if(!l1.shares_with(*p)) { - delta_view.push_back( - delta_view_itemt( - true, - k1, - l1.get_value(), - l2.get_value())); - - found=true; - break; + delta_view.push_back({true, k1, l1.get_value(), p->get_value()}); } } - - if(!only_common && !found) + else if(!only_common) { - delta_view.push_back( - delta_view_itemt( - false, - l1.get_key(), - l1.get_value(), - dummy)); + delta_view.push_back({false, l1.get_key(), l1.get_value(), dummy}); } } } - else - { - UNREACHABLE; - } } while(!stack.empty()); } -SHARING_MAPT2(, node_type *)::get_container_node(const key_type &k) +SHARING_MAPT2(, innert *)::get_container_node(const key_type &k) { - size_t key=hash()(k); - node_type *p=↦ + std::size_t key = hash()(k); + innert *ip = ↦ for(std::size_t i = 0; i < steps; i++) { std::size_t bit = key & mask; - key>>=chunk; - p=p->add_child(bit); + ip = ip->add_child(bit); + + key >>= chunk; } - return p; + return ip; } -SHARING_MAPT2(const, node_type *)::get_container_node(const key_type &k) const +SHARING_MAPT2(const, innert *)::get_container_node(const key_type &k) const { - size_t key=hash()(k); - const node_type *p=↦ + if(empty()) + return nullptr; + + std::size_t key = hash()(k); + const innert *ip = ↦ for(std::size_t i = 0; i < steps; i++) { std::size_t bit = key & mask; - key>>=chunk; - p=p->find_child(bit); - if(p==nullptr) + ip = ip->find_child(bit); + if(ip == nullptr) return nullptr; - } - - assert(p->is_container()); - - return p; -} - -SHARING_MAPT2(const, node_type *)::get_leaf_node(const key_type &k) const -{ - const node_type *p=get_container_node(k); - if(p==nullptr) - return nullptr; - p=p->find_leaf(k); + key >>= chunk; + } - return p; + return ip; } /// Erase element @@ -590,50 +601,62 @@ SHARING_MAPT2(const, node_type *)::get_leaf_node(const key_type &k) const /// \param k: The key of the element to erase /// \param key_exists: Hint to indicate whether the element is known to exist /// (possible values `unknown` or` true`) -SHARING_MAPT2(, size_type)::erase( - const key_type &k, - const tvt &key_exists) +SHARING_MAPT2(, size_type)::erase(const key_type &k, const tvt &key_exists) { - assert(!key_exists.is_false()); + SM_ASSERT(!key_exists.is_false()); + SM_ASSERT(!key_exists.is_true() || has_key(k)); // check if key exists if(key_exists.is_unknown() && !has_key(k)) return 0; - node_type *del=nullptr; + innert *del = nullptr; std::size_t del_bit = 0; + std::size_t del_level = 0; - size_t key=hash()(k); - node_type *p=↦ + std::size_t key = hash()(k); + innert *ip = ↦ for(std::size_t i = 0; i < steps; i++) { std::size_t bit = key & mask; - key>>=chunk; - const subt &sub=as_const(p)->get_sub(); - if(sub.size()>1 || del==nullptr) + const to_mapt &m = as_const(ip)->get_to_map(); + + if(m.size() > 1 || del == nullptr) { - del=p; + del = ip; del_bit=bit; + del_level = i; } - p=p->add_child(bit); - } + ip = ip->add_child(bit); - _sm_assert(p->is_container()); + key >>= chunk; + } - const containert &c = as_const(p)->get_container(); + const leaf_listt &ll = as_const(ip)->get_container(); - if(c.size() == 1) + // forward list has one element + if(!ll.empty() && std::next(ll.begin()) == ll.end()) { - del->remove_child(del_bit); + if(del_level < steps - 1) + { + del->remove_child(del_bit); + } + else + { + SM_ASSERT(del_level == steps - 1); + del->remove_child(del_bit); + } + num--; return 1; } - _sm_assert(c.size()>1); - p->remove_leaf(k); + SM_ASSERT(!ll.empty()); + + ip->remove_leaf(k); num--; return 1; @@ -650,11 +673,10 @@ SHARING_MAPT2(, size_type)::erase( /// (possible values `unknown` or `true`). Applies to all elements (i.e., have /// to use `unknown` if for at least one element it is not known whether it /// exists) -SHARING_MAPT2(, size_type)::erase_all( - const keyst &ks, - const tvt &key_exists) +SHARING_MAPT2(, size_type) +::erase_all(const keyst &ks, const tvt &key_exists) { - size_type cnt=0; + size_type cnt = 0; for(const key_type &k : ks) { @@ -679,22 +701,23 @@ SHARING_MAPT2(, size_type)::erase_all( SHARING_MAPT2(const, const_find_type) ::insert(const key_type &k, const mapped_type &m, const tvt &key_exists) { - _sn_assert(!key_exists.is_true()); + SM_ASSERT(!key_exists.is_true()); + SM_ASSERT(!key_exists.is_false() || !has_key(k)); if(key_exists.is_unknown()) { - const node_type *p=as_const(this)->get_leaf_node(k); - if(p!=nullptr) - return const_find_type(p->get_value(), false); + const leaft *lp = as_const(this)->get_leaf_node(k); + if(lp != nullptr) + return const_find_type(lp->get_value(), false); } - node_type *p=get_container_node(k); - _sn_assert(p!=nullptr); + innert *cp = get_container_node(k); + SM_ASSERT(cp != nullptr); - p=p->place_leaf(k, m); + const leaft *lp = cp->place_leaf(k, m); num++; - return const_find_type(as_const(p)->get_value(), true); + return const_find_type(lp->get_value(), true); } // Insert element, return const reference @@ -716,18 +739,18 @@ ::insert(const value_type &p, const tvt &key_exists) /// indicating if new element was inserted SHARING_MAPT2(const, find_type)::place(const key_type &k, const mapped_type &m) { - node_type *c=get_container_node(k); - _sm_assert(c!=nullptr); + innert *cp = get_container_node(k); + SM_ASSERT(cp != nullptr); - node_type *p=c->find_leaf(k); + leaft *lp = cp->find_leaf(k); - if(p!=nullptr) - return find_type(p->get_value(), false); + if(lp != nullptr) + return find_type(lp->get_value(), false); - p=c->place_leaf(k, m); + lp = cp->place_leaf(k, m); num++; - return find_type(p->get_value(), true); + return find_type(lp->get_value(), true); } /// Insert element, return non-const reference @@ -749,19 +772,19 @@ SHARING_MAPT2(const, find_type)::place(const value_type &p) /// boolean indicating if element was found. SHARING_MAPT2(const, find_type)::find(const key_type &k, const tvt &key_exists) { - _sm_assert(!key_exists.is_false()); + SM_ASSERT(!key_exists.is_false()); + SM_ASSERT(!key_exists.is_true() || has_key(k)); if(key_exists.is_unknown() && !has_key(k)) return find_type(dummy, false); - node_type *p=get_container_node(k); - _sm_assert(p!=nullptr); - - p=p->find_leaf(k); - _sm_assert(p!=nullptr); + innert *cp = get_container_node(k); + SM_ASSERT(cp != nullptr); - return find_type(p->get_value(), true); + leaft *lp = cp->find_leaf(k); + SM_ASSERT(lp != nullptr); + return find_type(lp->get_value(), true); } /// Find element @@ -775,12 +798,12 @@ SHARING_MAPT2(const, find_type)::find(const key_type &k, const tvt &key_exists) /// found), and boolean indicating if element was found. SHARING_MAPT2(const, const_find_type)::find(const key_type &k) const { - const node_type *p=get_leaf_node(k); + const leaft *lp = get_leaf_node(k); - if(p==nullptr) + if(lp == nullptr) return const_find_type(dummy, false); - return const_find_type(p->get_value(), true); + return const_find_type(lp->get_value(), true); } /// Get element at key @@ -841,14 +864,12 @@ SHARING_MAPT2(, mapped_type &)::operator[](const key_type &k) SHARING_MAPT(const std::string)::not_found_msg="key not found"; -// config -SHARING_MAPT(const size_t)::bits=18; -SHARING_MAPT(const size_t)::chunk=3; +SHARING_MAPT2(, mapped_type)::dummy; -// derived config -SHARING_MAPT(const size_t)::mask=0xffff>>(16-chunk); -SHARING_MAPT(const size_t)::steps=bits/chunk; +SHARING_MAPT(const std::size_t)::bits = 18; +SHARING_MAPT(const std::size_t)::chunk = 3; -SHARING_MAPT2(, mapped_type)::dummy; +SHARING_MAPT(const std::size_t)::mask = 0xffff >> (16 - chunk); +SHARING_MAPT(const std::size_t)::steps = bits / chunk; #endif diff --git a/src/util/sharing_node.h b/src/util/sharing_node.h index c81c7072b15..ab74840853a 100644 --- a/src/util/sharing_node.h +++ b/src/util/sharing_node.h @@ -12,14 +12,55 @@ Author: Daniel Poetzl #ifndef CPROVER_UTIL_SHARING_NODE_H #define CPROVER_UTIL_SHARING_NODE_H -#include +#ifdef SN_DEBUG +#include +#endif + +#include +#include + +#ifndef SN_SMALL_MAP +#define SN_SMALL_MAP 1 +#endif + +#ifndef SN_SHARE_KEYS +#define SN_SHARE_KEYS 0 +#endif + +#if SN_SMALL_MAP == 1 +#include "small_map.h" +#else #include -#include +#endif #include "invariant.h" +#include "make_unique.h" +#include "small_shared_ptr.h" +#include "small_shared_two_way_ptr.h" -#define _sn_assert(b) INVARIANT(b, "Sharing-node internal invariant") -//#define _sn_assert(b) +#ifdef SN_INTERNAL_CHECKS +#define SN_ASSERT(b) INVARIANT(b, "Sharing node internal invariant") +#define SN_ASSERT_USE(v, b) SN_ASSERT(b) +#else +#define SN_ASSERT(b) +#define SN_ASSERT_USE(v, b) v = v; +#endif + +// clang-format off +#define SN_TYPE_PAR_DECL \ + template > + +#define SN_TYPE_PAR_DEF \ + template +// clang-format on + +#define SN_TYPE_ARGS keyT, valueT, equalT + +#define SN_PTR_TYPE_ARGS d_internalt, d_containert + +#define SN_PTR_TYPE_ARG d_leaft template const T *as_const(T *t) @@ -27,324 +68,397 @@ const T *as_const(T *t) return t; } -template < - class keyT, - class valueT, - class predT=std::equal_to, - bool no_sharing=false> -class sharing_nodet -{ -public: - friend void sharing_node_test(); +// Inner nodes (internal nodes or container nodes) - typedef keyT key_type; - typedef valueT mapped_type; +typedef small_shared_two_way_pointeet d_baset; - typedef predT key_equal; +SN_TYPE_PAR_DECL class sharing_node_innert; - typedef sharing_nodet self_type; +SN_TYPE_PAR_DECL class d_internalt : public d_baset +{ +public: + typedef sharing_node_innert innert; +#if SN_SMALL_MAP == 1 + typedef small_mapt to_mapt; +#else + typedef std::map to_mapt; +#endif - typedef std::map subt; - typedef std::list containert; + to_mapt m; +}; - typedef const std::pair const_find_type; - typedef const std::pair find_type; +SN_TYPE_PAR_DECL class sharing_node_leaft; - sharing_nodet() : data(empty_data) - { - _sn_assert(data.use_count()>=2); - } +SN_TYPE_PAR_DECL class d_containert : public d_baset +{ +public: + typedef sharing_node_leaft leaft; + typedef std::forward_list leaf_listt; - sharing_nodet(const key_type &k, const mapped_type &m) : data(empty_data) - { - _sn_assert(data.use_count()>=2); + leaf_listt con; +}; - dt &d=write(); +class sharing_node_baset +{ +}; - _sn_assert(d.k==nullptr); - d.k=std::make_shared(k); +SN_TYPE_PAR_DEF class sharing_node_innert : public sharing_node_baset +{ +public: + typedef d_internalt d_it; + typedef d_containert d_ct; - _sn_assert(d.m==nullptr); - d.m=std::make_shared(m); - } + typedef typename d_it::to_mapt to_mapt; - sharing_nodet(const self_type &other) + typedef typename d_ct::leaft leaft; + typedef typename d_ct::leaf_listt leaf_listt; + + sharing_node_innert() : data(empty_data) { -#ifdef SM_NO_SHARING - data=std::make_shared
(*other.data); -#else - if(no_sharing) - { - data=std::make_shared
(*other.data); - } - else - { - data=other.data; - } -#endif } - // check type of node - - bool is_empty() const + bool empty() const { - _sn_assert(is_well_formed()); - return data==empty_data; + return data == empty_data; } - bool is_internal() const + void clear() { - _sn_assert(is_well_formed()); - _sn_assert(!is_empty()); - return !get_sub().empty(); + data = empty_data; } - bool is_container() const + bool shares_with(const sharing_node_innert &other) const { - _sn_assert(is_well_formed()); - _sn_assert(!is_empty()); - return !get_container().empty(); + return data == other.data; } - bool is_leaf() const + void swap(sharing_node_innert &other) { - _sn_assert(is_well_formed()); - _sn_assert(!is_empty()); - return read().is_leaf(); + data.swap(other.data); } - // accessors + // Types - const key_type &get_key() const + bool is_internal() const { - _sn_assert(is_leaf()); - return *read().k; + return data.is_derived_u(); } - const mapped_type &get_value() const + bool is_container() const { - _sn_assert(is_leaf()); - return *read().m; + return data.is_derived_v(); } - mapped_type &get_value() + d_it &write_internal() { - _sn_assert(is_leaf()); - return *write().m; - } + SN_ASSERT(data.use_count() > 0); - subt &get_sub() - { - return write().sub; + if(data == empty_data) + { + data = make_shared_derived_u(); + } + else if(data.use_count() > 1) + { + data = make_shared_derived_u(*data.get_derived_u()); + } + + SN_ASSERT(data.use_count() == 1); + + return *data.get_derived_u(); } - const subt &get_sub() const + const d_it &read_internal() const { - return read().sub; + SN_ASSERT(!empty()); + + return *data.get_derived_u(); } - containert &get_container() + d_ct &write_container() { - return write().con; + SN_ASSERT(data.use_count() > 0); + + if(data == empty_data) + { + data = make_shared_derived_v(); + } + else if(data.use_count() > 1) + { + data = make_shared_derived_v(*data.get_derived_v()); + } + + SN_ASSERT(data.use_count() == 1); + + return *data.get_derived_v(); } - const containert &get_container() const + const d_ct &read_container() const { - return read().con; + SN_ASSERT(!empty()); + + return *data.get_derived_v(); } - // internal nodes + // Accessors - const self_type *find_child(const std::size_t n) const + const to_mapt &get_to_map() const { - const subt &s=get_sub(); - typename subt::const_iterator it=s.find(n); - - if(it!=s.end()) - return &it->second; - - return nullptr; + return read_internal().m; } - self_type *add_child(const std::size_t n) + to_mapt &get_to_map() { - subt &s=get_sub(); - return &s[n]; + return write_internal().m; } - void remove_child(const std::size_t n) + const leaf_listt &get_container() const { - subt &s=get_sub(); - size_t r=s.erase(n); + return read_container().con; + } - _sn_assert(r==1); + leaf_listt &get_container() + { + return write_container().con; } - // container nodes + // Containers - const self_type *find_leaf(const key_type &k) const + const leaft *find_leaf(const keyT &k) const { - const containert &c=get_container(); + SN_ASSERT(is_container()); + + const leaf_listt &c = get_container(); for(const auto &n : c) { - if(key_equal()(n.get_key(), k)) + if(equalT()(n.get_key(), k)) return &n; } return nullptr; } - self_type *find_leaf(const key_type &k) + leaft *find_leaf(const keyT &k) { - containert &c=get_container(); + SN_ASSERT(is_container()); + + leaf_listt &c = get_container(); for(auto &n : c) { - if(key_equal()(n.get_key(), k)) + if(equalT()(n.get_key(), k)) return &n; } + // If we return nullptr the call must be followed by a call to + // place_leaf(k, ...) return nullptr; } // add leaf, key must not exist yet - self_type *place_leaf(const key_type &k, const mapped_type &m) + leaft *place_leaf(const keyT &k, const valueT &v) { - _sn_assert(as_const(this)->find_leaf(k)==nullptr); + SN_ASSERT(is_container()); + + SN_ASSERT(as_const(this)->find_leaf(k) == nullptr); - containert &c=get_container(); - c.push_back(self_type(k, m)); + leaf_listt &c = get_container(); + c.push_front(leaft(k, v)); - return &c.back(); + return &c.front(); } // remove leaf, key must exist - void remove_leaf(const key_type &k) + void remove_leaf(const keyT &k) { - containert &c=get_container(); + SN_ASSERT(is_container()); + + leaf_listt &c = get_container(); + + SN_ASSERT(!c.empty()); + + const leaft &first = c.front(); + + if(equalT()(first.get_key(), k)) + { + c.pop_front(); + return; + } + + typename leaf_listt::const_iterator last_it = c.begin(); - for(typename containert::const_iterator it=c.begin(); - it!=c.end(); it++) + typename leaf_listt::const_iterator it = c.begin(); + it++; + + for(; it != c.end(); it++) { - const self_type &n=*it; + const leaft &leaf = *it; - if(key_equal()(n.get_key(), k)) + if(equalT()(leaf.get_key(), k)) { - c.erase(it); + c.erase_after(last_it); return; } + + last_it = it; } UNREACHABLE; } - // misc + // Handle children + + const typename d_it::innert *find_child(const std::size_t n) const + { + SN_ASSERT(is_internal()); + + const to_mapt &m = get_to_map(); + typename to_mapt::const_iterator it = m.find(n); + + if(it != m.end()) + return &it->second; + + return nullptr; + } + + typename d_it::innert *add_child(const std::size_t n) + { + SN_ASSERT(is_internal()); + + to_mapt &m = get_to_map(); + return &m[n]; + } + + void remove_child(const std::size_t n) + { + SN_ASSERT(is_internal()); + + to_mapt &m = get_to_map(); + std::size_t r = m.erase(n); + + SN_ASSERT_USE(r, r == 1); + } + + small_shared_two_way_ptrt data; + static small_shared_two_way_ptrt empty_data; +}; + +SN_TYPE_PAR_DEF small_shared_two_way_ptrt + sharing_node_innert::empty_data = + small_shared_two_way_ptrt(); + +// Leafs + +SN_TYPE_PAR_DECL class d_leaft : public small_shared_pointeet +{ +public: +#if SN_SHARE_KEYS == 1 + std::shared_ptr k; +#else + keyT k; +#endif + valueT v; +}; + +SN_TYPE_PAR_DEF class sharing_node_leaft : public sharing_node_baset +{ +public: + typedef d_leaft d_lt; + + sharing_node_leaft(const keyT &k, const valueT &v) : data(empty_data) + { + SN_ASSERT(empty()); + + auto &d = write(); + +// Copy key +#if SN_SHARE_KEYS == 1 + SN_ASSERT(d.k == nullptr); + d.k = std::make_shared(k); +#else + d.k = k; +#endif + + // Copy value + d.v = v; + } + + bool empty() const + { + return data == empty_data; + } void clear() { - *this=self_type(); + data = empty_data; } - bool shares_with(const self_type &other) const + bool shares_with(const sharing_node_leaft &other) const { - return data==other.data; + return data == other.data; } - void swap(self_type &other) + void swap(sharing_node_leaft &other) { data.swap(other.data); } -protected: - class dt + d_lt &write() { - public: - dt() {} + SN_ASSERT(data.use_count() > 0); - dt(const dt &d) : k(d.k), sub(d.sub), con(d.con) + if(data == empty_data) { - if(d.is_leaf()) - { - _sn_assert(m==nullptr); - m=std::make_shared(*d.m); - } + data = make_small_shared_ptr(); } - - bool is_leaf() const + else if(data.use_count() > 1) { - _sn_assert(k==nullptr || m!=nullptr); - return k!=nullptr; + data = make_small_shared_ptr(*data); } - std::shared_ptr k; - std::shared_ptr m; + SN_ASSERT(data.use_count() == 1); - subt sub; - containert con; - }; - - const dt &read() const - { return *data; } - dt &write() + const d_lt &read() const { - detach(); return *data; } - void detach() - { - _sn_assert(data.use_count()>0); + // Accessors - if(data==empty_data) - data=std::make_shared
(); - else if(data.use_count()>1) - data=std::make_shared
(*data); + const keyT &get_key() const + { + SN_ASSERT(!empty()); - _sn_assert(data.use_count()==1); +#if SN_SHARE_KEYS == 1 + return *read().k; +#else + return read().k; +#endif } - bool is_well_formed() const + const valueT &get_value() const { - if(data==nullptr) - return false; - - const dt &d=*data; + SN_ASSERT(!empty()); - bool b; + return read().v; + } - // empty node - b=data==empty_data; - // internal node - b|=d.k==nullptr && d.m==nullptr && get_container().empty() && - !get_sub().empty(); - // container node - b|=d.k==nullptr && d.m==nullptr && !get_container().empty() && - get_sub().empty(); - // leaf node - b|=d.k!=nullptr && d.m!=nullptr && get_container().empty() && - get_sub().empty(); + valueT &get_value() + { + SN_ASSERT(!empty()); - return b; + return write().v; } - std::shared_ptr
data; - static std::shared_ptr
empty_data; - - // dummy node returned when node was not found - static sharing_nodet dummy; + small_shared_ptrt data; + static small_shared_ptrt empty_data; }; -template -std::shared_ptr::dt> - sharing_nodet::empty_data= - std::make_shared::dt>(); - -template -sharing_nodet - sharing_nodet::dummy; +SN_TYPE_PAR_DEF small_shared_ptrt + sharing_node_leaft::empty_data = + make_small_shared_ptr(); #endif diff --git a/unit/Makefile b/unit/Makefile index 407f29f812a..c095a23b01c 100644 --- a/unit/Makefile +++ b/unit/Makefile @@ -27,6 +27,7 @@ SRC += unit_tests.cpp \ util/message.cpp \ util/optional.cpp \ util/sharing_node.cpp \ + util/sharing_map.cpp \ util/small_map.cpp \ util/small_shared_two_way_ptr.cpp \ util/string_utils/split_string.cpp \ diff --git a/unit/util/sharing_map.cpp b/unit/util/sharing_map.cpp index e08598fced4..b6629d3ece9 100644 --- a/unit/util/sharing_map.cpp +++ b/unit/util/sharing_map.cpp @@ -6,6 +6,12 @@ Author: Daniel Poetzl \*******************************************************************/ +#define SHARING_MAP_INTERNAL_CHECKS +#define SHARING_NODE_INTERNAL_CHECKS + +#include +#include + #include #include @@ -191,6 +197,10 @@ void sharing_map_copy_test() class some_keyt { public: + some_keyt() : s(0) + { + } + some_keyt(size_t s) : s(s) { } @@ -240,14 +250,43 @@ void sharing_map_view_test() { SECTION("View") { + typedef std::pair pt; + smt sm; + smt::viewt view; + std::vector pairs; + + auto sort_view = [&pairs, &view]() { + pairs.clear(); + for(auto &p : view) + { + pairs.push_back({p.first, p.second}); + } + std::sort(pairs.begin(), pairs.end()); + }; fill(sm); - smt::viewt view; sm.get_view(view); REQUIRE(view.size() == 3); + + sort_view(); + + REQUIRE((pairs[0] == pt("i", "0"))); + REQUIRE((pairs[1] == pt("j", "1"))); + REQUIRE((pairs[2] == pt("k", "2"))); + + sm.insert("l", "3"); + + view.clear(); + sm.get_view(view); + + REQUIRE(view.size() == 4); + + sort_view(); + + REQUIRE((pairs[3] == pt("l", "3"))); } SECTION("Delta view (no sharing, same keys)") @@ -326,6 +365,32 @@ void sharing_map_view_test() } } +void sharing_map_memory_test() +{ + int n; + + n = 1000; + //n = 5000000; + + //sharing_mapt m; + std::map m; + //std::unordered_map m; + + std::random_device r; + std::default_random_engine e(r()); + std::uniform_int_distribution uniform_dist(0, INT_MAX); + + for(int i = 0; i < n; i++) + { + int v; + + //v = i; + v = uniform_dist(e); + + m.insert(std::make_pair(v, v)); + } +} + TEST_CASE("Sharing map interface") { sharing_map_interface_test(); @@ -345,3 +410,8 @@ TEST_CASE("Sharing map views") { sharing_map_view_test(); } + +TEST_CASE("Sharing map memory") +{ + sharing_map_memory_test(); +} diff --git a/unit/util/sharing_node.cpp b/unit/util/sharing_node.cpp index 9d72919cfed..470cf7beafd 100644 --- a/unit/util/sharing_node.cpp +++ b/unit/util/sharing_node.cpp @@ -2,121 +2,203 @@ /// \file Tests for sharing-node utility +#define SHARING_NODE_INTERNAL_CHECKS + #include #include void sharing_node_test() { - typedef sharing_nodet snt; + SECTION("Leaf test") + { + typedef sharing_node_leaft leaft; + + // Basic leaf + { + const leaft leaf(1, 2); + + REQUIRE(!leaf.empty()); + REQUIRE(leaf.shares_with(leaf)); + REQUIRE(leaf.get_key() == 1); + REQUIRE(leaf.get_value() == 2); + } + + // Shared leaf + { + const leaft leaf1(1, 2); + const leaft leaf2(leaf1); + + REQUIRE(leaf1.shares_with(leaf2)); + REQUIRE(leaf2.get_key() == 1); + REQUIRE(leaf2.get_value() == 2); + } + + // Modify shared leaf + { + const leaft leaf1(1, 2); + leaft leaf2(leaf1); + + REQUIRE(leaf2.shares_with(leaf1)); + + auto &v = leaf2.get_value(); + v = 3; + REQUIRE(leaf2.get_value() == 3); + REQUIRE(!leaf2.shares_with(leaf1)); + } + + // Detaching + { + leaft leaf(1, 2); + + auto p = leaf.data.get(); + leaf.write(); + + REQUIRE(leaf.data.get() == p); + } + } - SECTION("Internal nodes") + SECTION("Inner node test") { - snt sn1; - const snt &sn2 = sn1; + typedef sharing_node_innert innert; - const snt *p2; + // Empty container + { + const innert c; - REQUIRE(sn1.is_empty()); + REQUIRE(c.empty()); + } - sn1.add_child(0); - sn1.add_child(1); - sn1.add_child(2); + // Single container + { + innert c; + const innert &cc = c; - REQUIRE(!sn2.is_empty()); - REQUIRE(sn2.is_internal()); - REQUIRE(!sn2.is_container()); - REQUIRE(!sn2.is_leaf()); + c.place_leaf(1, 2); + c.place_leaf(3, 4); - REQUIRE(sn2.get_sub().size() == 3); + REQUIRE(!cc.get_container().empty()); + REQUIRE(cc.find_leaf(1) != nullptr); + REQUIRE(cc.find_leaf(3) != nullptr); - p2 = sn2.find_child(0); - REQUIRE(p2 != nullptr); + auto leaf = c.find_leaf(1); + REQUIRE(leaf->get_key() == 1); + REQUIRE(leaf->get_value() == 2); - p2 = sn1.find_child(0); - REQUIRE(p2 != nullptr); + c.remove_leaf(1); + c.remove_leaf(3); - p2 = sn2.find_child(3); - REQUIRE(p2 == nullptr); + REQUIRE(cc.get_container().empty()); + } - p2 = sn1.find_child(3); - REQUIRE(p2 == nullptr); + // Shared container + { + innert c1; + c1.place_leaf(1, 2); + c1.place_leaf(3, 4); - sn1.remove_child(0); - REQUIRE(sn2.get_sub().size() == 2); + innert c2(c1); + auto leaf = c2.find_leaf(1); + auto &v = leaf->get_value(); + v = 7; - sn1.clear(); - REQUIRE(sn2.is_empty()); - } + REQUIRE(!c1.shares_with(c2)); - SECTION("Container nodes") - { - snt sn1; + { + auto leaf1 = c1.find_leaf(1); + auto leaf2 = c2.find_leaf(1); - snt *p1; - const snt *p2; + REQUIRE(!leaf1->shares_with(*leaf2)); + } - sn1.add_child(0); - sn1.add_child(1); + { + auto leaf1 = c1.find_leaf(3); + auto leaf2 = c2.find_leaf(3); - p1 = sn1.add_child(2); - p2 = p1; + REQUIRE(leaf1->shares_with(*leaf2)); + } + } - REQUIRE(p1->find_leaf("a") == nullptr); - REQUIRE(p2->find_leaf("a") == nullptr); + // Internal node mapping to other internal nodes + { + innert i; + const innert &ci = i; - p1->place_leaf("a", "b"); - REQUIRE(p2->get_container().size() == 1); - p1->place_leaf("c", "d"); - REQUIRE(p2->get_container().size() == 2); + i.add_child(0); - REQUIRE(p2->is_container()); + REQUIRE(!i.empty()); + REQUIRE(!i.get_to_map().empty()); + REQUIRE(ci.find_child(0) != nullptr); - p1->remove_leaf("a"); - REQUIRE(p2->get_container().size() == 1); - } + i.remove_child(0); - SECTION("Copy test 1") - { - snt sn1; - snt sn2; + REQUIRE(i.get_to_map().empty()); + REQUIRE(ci.find_child(0) == nullptr); - sn2 = sn1; - REQUIRE(sn1.data.use_count() == 3); - REQUIRE(sn2.data.use_count() == 3); + innert *p; - sn1.add_child(0); - REQUIRE(sn1.data.use_count() == 1); - // the newly created node is empty as well - REQUIRE(sn2.data.use_count() == 3); + p = i.add_child(1); - sn2 = sn1; - REQUIRE(sn2.data.use_count() == 2); + REQUIRE(p != nullptr); + } } - SECTION("Copy test 2") + SECTION("Combined") { - snt sn1; - const snt &sn1c = sn1; - snt *p; + typedef sharing_node_leaft leaft; + typedef sharing_node_innert innert; + + innert map; + + REQUIRE(map.empty()); + + innert *ip; + innert *cp; + leaft *lp; + + // Add 0 -> 0 -> (0, 1) + + ip = ↦ + + ip = ip->add_child(0); + REQUIRE(ip != nullptr); + + cp = ip->add_child(0); + REQUIRE(cp != nullptr); + + lp = cp->place_leaf(0, 1); + REQUIRE(lp != nullptr); + + // Add 1 -> 2 -> (3, 4) + + ip = ↦ + + ip = ip->add_child(1); + REQUIRE(ip != nullptr); + + cp = ip->add_child(2); + REQUIRE(cp != nullptr); + + lp = cp->place_leaf(3, 4); + REQUIRE(lp != nullptr); + + // Add 1 -> 3 -> (4, 5) + + ip = ↦ - p = sn1.add_child(0); - p->place_leaf("x", "y"); + ip = ip->add_child(1); + REQUIRE(ip != nullptr); - p = sn1.add_child(1); - p->place_leaf("a", "b"); - p->place_leaf("c", "d"); + cp = ip->add_child(3); + REQUIRE(cp != nullptr); - snt sn2; - const snt &sn2c = sn2; - sn2 = sn1; + lp = cp->place_leaf(4, 5); + REQUIRE(lp != nullptr); - REQUIRE(sn1.is_internal()); - REQUIRE(sn2.is_internal()); + // Copy - sn1.remove_child(0); - REQUIRE(sn1c.get_sub().size() == 1); + innert map2(map); - REQUIRE(sn2c.get_sub().size() == 2); + REQUIRE(map2.shares_with(map)); } } From af2defdd8a2379c1974b78140a873e81b4a4e1b3 Mon Sep 17 00:00:00 2001 From: Daniel Poetzl Date: Tue, 22 May 2018 15:51:49 +0100 Subject: [PATCH 2/2] Removed obsolete sharing map unit test --- unit/util/sharing_map.cpp | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/unit/util/sharing_map.cpp b/unit/util/sharing_map.cpp index b6629d3ece9..3116520ec80 100644 --- a/unit/util/sharing_map.cpp +++ b/unit/util/sharing_map.cpp @@ -365,32 +365,6 @@ void sharing_map_view_test() } } -void sharing_map_memory_test() -{ - int n; - - n = 1000; - //n = 5000000; - - //sharing_mapt m; - std::map m; - //std::unordered_map m; - - std::random_device r; - std::default_random_engine e(r()); - std::uniform_int_distribution uniform_dist(0, INT_MAX); - - for(int i = 0; i < n; i++) - { - int v; - - //v = i; - v = uniform_dist(e); - - m.insert(std::make_pair(v, v)); - } -} - TEST_CASE("Sharing map interface") { sharing_map_interface_test(); @@ -410,8 +384,3 @@ TEST_CASE("Sharing map views") { sharing_map_view_test(); } - -TEST_CASE("Sharing map memory") -{ - sharing_map_memory_test(); -}