From 33fae1cb6606751a5a7b6aead0e63d9d7e8080e8 Mon Sep 17 00:00:00 2001 From: kaldoran <kaldoran@live.fr> Date: Wed, 13 Apr 2016 10:31:25 -0400 Subject: [PATCH] pht: add prefix cache --- include/opendht/indexation/pht.h | 139 +++++++++++++++++++++++++++++-- src/indexation/pht.cpp | 30 ++++--- 2 files changed, 154 insertions(+), 15 deletions(-) diff --git a/include/opendht/indexation/pht.h b/include/opendht/indexation/pht.h index 2e687cba..70a03eee 100644 --- a/include/opendht/indexation/pht.h +++ b/include/opendht/indexation/pht.h @@ -47,6 +47,16 @@ struct Prefix { return Prefix(*this, len); } + /** + * Method for getting the state of the bit at the position pos. + * @param pos : Pos of the needed bit + * @return : true if the bit is at 1 + * false otherwise + */ + bool isActivBit(size_t pos) const { + return ((this->content_[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1; + } + Prefix getFullSize() { return Prefix(*this, content_.size()*8); } /** @@ -136,7 +146,7 @@ class Pht { public: /* This is the maximum number of entries per node. This parameter is - * critical and influences the traffic alot during a lookup operation. + * critical and influences the traffic a lot during a lookup operation. */ static constexpr const size_t MAX_NODE_ENTRY_COUNT {16}; @@ -166,6 +176,125 @@ public: void insert(Key k, Value v, Dht::DoneCallbackSimple cb = {}); private: + class Cache { + public: + /** + * Insert all needed node into the tree according to a prefix + * @param p : Prefix that we need to insert + */ + void insert(const Prefix& p) { + size_t i = 0; + auto now = clock::now(); + + std::shared_ptr<Node> curr_node; + + while ( ( leaves_.size() > 0 && leaves_.begin()->first + NODE_EXPIRE_TIME < now ) + || leaves_.size() > MAX_ELEMENT ) { + + leaves_.erase(leaves_.begin()); + } + + if ( !(curr_node = root_.lock()) ) { + + /* Root does not exist, need to create one*/ + curr_node = std::make_shared<Node>(); + root_ = curr_node; + } + + curr_node->last_reply = now; + + /* Iterate through all bit of the Blob */ + for ( i = 0; i < p.size_; i++ ) { + + /* According to the bit define which node is the next one */ + auto& next = ( p.isActivBit(i) ) ? curr_node->right_child : curr_node->left_child; + + /** + * If lock, node exists + * else create it + */ + if (auto n = next.lock()) { + curr_node = std::move(n); + } else { + /* Create the next node if doesn't exist*/ + auto tmp_curr_node = std::make_shared<Node>(); + tmp_curr_node->parent = curr_node; + next = tmp_curr_node; + curr_node = std::move(tmp_curr_node); + } + + curr_node->last_reply = now; + } + + /* Insert the leaf (curr_node) into the multimap */ + leaves_.emplace(std::move(now), std::move(curr_node) ); + } + + /** + * Lookup into the tree to return the maximum prefix length in the cache tree + * + * @param p : Prefix that we are looking for + * @return : The size of the longest prefix known in the cache between 0 and p.size_ + */ + int lookup(const Prefix& p) { + int pos = 0; + auto now = clock::now(), last_node_time = now; + + /* Before lookup remove the useless one [i.e. too old] */ + while ( leaves_.size() > 0 && leaves_.begin()->first + NODE_EXPIRE_TIME < now ) { + leaves_.erase(leaves_.begin()); + } + + auto next = root_; + std::shared_ptr<Node> curr_node; + + while ( auto n = next.lock() ) { + /* Safe since pos is equal to 0 until here */ + if ( (unsigned) pos >= p.size_ ) break; + + curr_node = n; + last_node_time = curr_node->last_reply; + curr_node->last_reply = now; + + /* Get the Prefix bit by bit, starting from left */ + next = ( p.isActivBit(pos) ) ? curr_node->right_child : curr_node->left_child; + + ++pos; + } + + if ( pos > 0 ) { + auto to_erase = leaves_.find(last_node_time); + if ( to_erase != leaves_.end() ) + leaves_.erase( to_erase ); + + leaves_.emplace( std::move(now), std::move(curr_node) ); + } + + return --pos; + } + + + private: + static constexpr const size_t MAX_ELEMENT {1024}; + static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {5}; + + struct Node { + time_point last_reply; /* Made the assocation between leaves and leaves multimap */ + std::shared_ptr<Node> parent; /* Share_ptr to the parent, it allow the self destruction of tree */ + std::weak_ptr<Node> left_child; /* Left child, for bit equal to 1 */ + std::weak_ptr<Node> right_child; /* Right child, for bit equal to 0 */ + }; + + std::weak_ptr<Node> root_; /* Root of the tree */ + + /** + * This mutlimap contains all prefix insert in the tree in time order + * We could then delete the last one if there is too much node + * The tree will self destroy is branch ( thanks to share_ptr ) + */ + std::multimap<time_point, std::shared_ptr<Node>> leaves_; + }; + /** * Linearizes the key into a unidimensional key. A pht only takes * unidimensional key. @@ -184,9 +313,9 @@ private: * asynchronously. */ void lookupStep(Prefix k, std::shared_ptr<int> lo, std::shared_ptr<int> hi, - std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals, - LookupCallback cb, Dht::DoneCallbackSimple done_cb, - std::shared_ptr<unsigned> max_common_prefix_len, bool all_values = false); + std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals, LookupCallback cb, + Dht::DoneCallbackSimple done_cb, std::shared_ptr<unsigned> max_common_prefix_len, + int start = -1, bool all_values = false); /** * Updates the canary token on the node responsible for the specified @@ -196,7 +325,7 @@ private: const std::string name_; const std::string canary_; - + Cache cache_; std::shared_ptr<DhtRunner> dht_; }; diff --git a/src/indexation/pht.cpp b/src/indexation/pht.cpp index 80aad4b8..2e0b4673 100644 --- a/src/indexation/pht.cpp +++ b/src/indexation/pht.cpp @@ -5,18 +5,20 @@ namespace dht { namespace indexation { const ValueType IndexEntry::TYPE = ValueType::USER_DATA; +constexpr std::chrono::minutes Pht::Cache::NODE_EXPIRE_TIME; void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals, LookupCallback cb, Dht::DoneCallbackSimple done_cb, - std::shared_ptr<unsigned> max_common_prefix_len, bool all_values) + std::shared_ptr<unsigned> max_common_prefix_len, int start, bool all_values) { struct node_lookup_result { bool done {false}; bool is_pht {false}; }; - auto mid = (*lo + *hi)/2; + /* start could be under 0 but after the compare it to 0 it always will be unsigned, so we can cast it*/ + auto mid = (start >= 0) ? (unsigned) start : (*lo + *hi)/2; auto first_res = std::make_shared<node_lookup_result>(); auto second_res = std::make_shared<node_lookup_result>(); auto on_done = [=](bool ok) { @@ -27,21 +29,26 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, } else if (is_leaf or *lo > *hi) { // leaf node + auto to_insert = p.getPrefix(mid); + cache_.insert(to_insert); + if (cb) { if (vals->size() == 0 and max_common_prefix_len and mid > 0) { auto p_ = (p.getPrefix(mid)).getSibling().getFullSize(); *lo = mid; *hi = p_.size_; - lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values); + lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values); } - cb(*vals, p.getPrefix(mid)); + + cb(*vals, to_insert); } + if (done_cb) done_cb(true); } else if (first_res->is_pht) { // internal node *lo = mid+1; - lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values); + lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values); } else { // first get failed before second. if (done_cb) @@ -83,6 +90,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, } return true; }; + dht_->get(p.getPrefix(mid).hash(), std::bind(on_get, std::placeholders::_1, first_res), [=](bool ok) { @@ -96,7 +104,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, if (not first_res->is_pht) { // Not a PHT node. *hi = mid-1; - lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values); + lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values); } else { first_res->done = true; if (second_res->done) @@ -104,6 +112,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, } } }, pht_filter); + if (mid < p.size_) dht_->get(p.getPrefix(mid+1).hash(), std::bind(on_get, std::placeholders::_1, second_res), @@ -132,7 +141,8 @@ void Pht::lookup(Key k, Pht::LookupCallback cb, Dht::DoneCallbackSimple done_cb, auto lo = std::make_shared<int>(0); auto hi = std::make_shared<int>(prefix.size_); std::shared_ptr<unsigned> max_common_prefix_len = not exact_match ? std::make_shared<unsigned>(0) : nullptr; - lookupStep(prefix, lo, hi, values, cb, done_cb, max_common_prefix_len); + + lookupStep(prefix, lo, hi, values, cb, done_cb, max_common_prefix_len, cache_.lookup(prefix)); } void Pht::updateCanary(Prefix p) { @@ -140,7 +150,7 @@ void Pht::updateCanary(Prefix p) { dht::Value canary_value; canary_value.user_type = canary_; dht_->put(p.hash(), std::move(canary_value), - [=](bool ok){ + [=](bool){ static std::bernoulli_distribution d(0.5); crypto::random_device rd; if (p.size_ && d(rd)) @@ -164,7 +174,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) { auto final_prefix = std::make_shared<Prefix>(); lookupStep(kp, lo, hi, vals, - [=](std::vector<std::shared_ptr<Value>>& values, Prefix p) { + [=](std::vector<std::shared_ptr<Value>>&, Prefix p) { *final_prefix = Prefix(p); }, [=](bool ok){ @@ -183,7 +193,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) { updateCanary(*final_prefix); dht_->put(final_prefix->hash(), std::move(entry), done_cb); } - }, nullptr, true + }, nullptr, cache_.lookup(kp), true ); } -- GitLab