From 33fae1cb6606751a5a7b6aead0e63d9d7e8080e8 Mon Sep 17 00:00:00 2001
From: kaldoran <kaldoran@live.fr>
Date: Wed, 13 Apr 2016 10:31:25 -0400
Subject: [PATCH] pht: add prefix cache
---
include/opendht/indexation/pht.h | 139 +++++++++++++++++++++++++++++--
src/indexation/pht.cpp | 30 ++++---
2 files changed, 154 insertions(+), 15 deletions(-)
diff --git a/include/opendht/indexation/pht.h b/include/opendht/indexation/pht.h
index 2e687cba..70a03eee 100644
--- a/include/opendht/indexation/pht.h
+++ b/include/opendht/indexation/pht.h
@@ -47,6 +47,16 @@ struct Prefix {
return Prefix(*this, len);
}
+ /**
+ * Method for getting the state of the bit at the position pos.
+ * @param pos : Pos of the needed bit
+ * @return : true if the bit is at 1
+ * false otherwise
+ */
+ bool isActivBit(size_t pos) const {
+ return ((this->content_[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1;
+ }
+
Prefix getFullSize() { return Prefix(*this, content_.size()*8); }
/**
@@ -136,7 +146,7 @@ class Pht {
public:
/* This is the maximum number of entries per node. This parameter is
- * critical and influences the traffic alot during a lookup operation.
+ * critical and influences the traffic a lot during a lookup operation.
*/
static constexpr const size_t MAX_NODE_ENTRY_COUNT {16};
@@ -166,6 +176,125 @@ public:
void insert(Key k, Value v, Dht::DoneCallbackSimple cb = {});
private:
+ class Cache {
+ public:
+ /**
+ * Insert all needed node into the tree according to a prefix
+ * @param p : Prefix that we need to insert
+ */
+ void insert(const Prefix& p) {
+ size_t i = 0;
+ auto now = clock::now();
+
+ std::shared_ptr<Node> curr_node;
+
+ while ( ( leaves_.size() > 0 && leaves_.begin()->first + NODE_EXPIRE_TIME < now )
+ || leaves_.size() > MAX_ELEMENT ) {
+
+ leaves_.erase(leaves_.begin());
+ }
+
+ if ( !(curr_node = root_.lock()) ) {
+
+ /* Root does not exist, need to create one*/
+ curr_node = std::make_shared<Node>();
+ root_ = curr_node;
+ }
+
+ curr_node->last_reply = now;
+
+ /* Iterate through all bit of the Blob */
+ for ( i = 0; i < p.size_; i++ ) {
+
+ /* According to the bit define which node is the next one */
+ auto& next = ( p.isActivBit(i) ) ? curr_node->right_child : curr_node->left_child;
+
+ /**
+ * If lock, node exists
+ * else create it
+ */
+ if (auto n = next.lock()) {
+ curr_node = std::move(n);
+ } else {
+ /* Create the next node if doesn't exist*/
+ auto tmp_curr_node = std::make_shared<Node>();
+ tmp_curr_node->parent = curr_node;
+ next = tmp_curr_node;
+ curr_node = std::move(tmp_curr_node);
+ }
+
+ curr_node->last_reply = now;
+ }
+
+ /* Insert the leaf (curr_node) into the multimap */
+ leaves_.emplace(std::move(now), std::move(curr_node) );
+ }
+
+ /**
+ * Lookup into the tree to return the maximum prefix length in the cache tree
+ *
+ * @param p : Prefix that we are looking for
+ * @return : The size of the longest prefix known in the cache between 0 and p.size_
+ */
+ int lookup(const Prefix& p) {
+ int pos = 0;
+ auto now = clock::now(), last_node_time = now;
+
+ /* Before lookup remove the useless one [i.e. too old] */
+ while ( leaves_.size() > 0 && leaves_.begin()->first + NODE_EXPIRE_TIME < now ) {
+ leaves_.erase(leaves_.begin());
+ }
+
+ auto next = root_;
+ std::shared_ptr<Node> curr_node;
+
+ while ( auto n = next.lock() ) {
+ /* Safe since pos is equal to 0 until here */
+ if ( (unsigned) pos >= p.size_ ) break;
+
+ curr_node = n;
+ last_node_time = curr_node->last_reply;
+ curr_node->last_reply = now;
+
+ /* Get the Prefix bit by bit, starting from left */
+ next = ( p.isActivBit(pos) ) ? curr_node->right_child : curr_node->left_child;
+
+ ++pos;
+ }
+
+ if ( pos > 0 ) {
+ auto to_erase = leaves_.find(last_node_time);
+ if ( to_erase != leaves_.end() )
+ leaves_.erase( to_erase );
+
+ leaves_.emplace( std::move(now), std::move(curr_node) );
+ }
+
+ return --pos;
+ }
+
+
+ private:
+ static constexpr const size_t MAX_ELEMENT {1024};
+ static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {5};
+
+ struct Node {
+ time_point last_reply; /* Made the assocation between leaves and leaves multimap */
+ std::shared_ptr<Node> parent; /* Share_ptr to the parent, it allow the self destruction of tree */
+ std::weak_ptr<Node> left_child; /* Left child, for bit equal to 1 */
+ std::weak_ptr<Node> right_child; /* Right child, for bit equal to 0 */
+ };
+
+ std::weak_ptr<Node> root_; /* Root of the tree */
+
+ /**
+ * This mutlimap contains all prefix insert in the tree in time order
+ * We could then delete the last one if there is too much node
+ * The tree will self destroy is branch ( thanks to share_ptr )
+ */
+ std::multimap<time_point, std::shared_ptr<Node>> leaves_;
+ };
+
/**
* Linearizes the key into a unidimensional key. A pht only takes
* unidimensional key.
@@ -184,9 +313,9 @@ private:
* asynchronously.
*/
void lookupStep(Prefix k, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
- std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals,
- LookupCallback cb, Dht::DoneCallbackSimple done_cb,
- std::shared_ptr<unsigned> max_common_prefix_len, bool all_values = false);
+ std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals, LookupCallback cb,
+ Dht::DoneCallbackSimple done_cb, std::shared_ptr<unsigned> max_common_prefix_len,
+ int start = -1, bool all_values = false);
/**
* Updates the canary token on the node responsible for the specified
@@ -196,7 +325,7 @@ private:
const std::string name_;
const std::string canary_;
-
+ Cache cache_;
std::shared_ptr<DhtRunner> dht_;
};
diff --git a/src/indexation/pht.cpp b/src/indexation/pht.cpp
index 80aad4b8..2e0b4673 100644
--- a/src/indexation/pht.cpp
+++ b/src/indexation/pht.cpp
@@ -5,18 +5,20 @@ namespace dht {
namespace indexation {
const ValueType IndexEntry::TYPE = ValueType::USER_DATA;
+constexpr std::chrono::minutes Pht::Cache::NODE_EXPIRE_TIME;
void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals,
LookupCallback cb, Dht::DoneCallbackSimple done_cb,
- std::shared_ptr<unsigned> max_common_prefix_len, bool all_values)
+ std::shared_ptr<unsigned> max_common_prefix_len, int start, bool all_values)
{
struct node_lookup_result {
bool done {false};
bool is_pht {false};
};
- auto mid = (*lo + *hi)/2;
+ /* start could be under 0 but after the compare it to 0 it always will be unsigned, so we can cast it*/
+ auto mid = (start >= 0) ? (unsigned) start : (*lo + *hi)/2;
auto first_res = std::make_shared<node_lookup_result>();
auto second_res = std::make_shared<node_lookup_result>();
auto on_done = [=](bool ok) {
@@ -27,21 +29,26 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
}
else if (is_leaf or *lo > *hi) {
// leaf node
+ auto to_insert = p.getPrefix(mid);
+ cache_.insert(to_insert);
+
if (cb) {
if (vals->size() == 0 and max_common_prefix_len and mid > 0) {
auto p_ = (p.getPrefix(mid)).getSibling().getFullSize();
*lo = mid;
*hi = p_.size_;
- lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values);
+ lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
}
- cb(*vals, p.getPrefix(mid));
+
+ cb(*vals, to_insert);
}
+
if (done_cb)
done_cb(true);
} else if (first_res->is_pht) {
// internal node
*lo = mid+1;
- lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values);
+ lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
} else {
// first get failed before second.
if (done_cb)
@@ -83,6 +90,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
}
return true;
};
+
dht_->get(p.getPrefix(mid).hash(),
std::bind(on_get, std::placeholders::_1, first_res),
[=](bool ok) {
@@ -96,7 +104,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
if (not first_res->is_pht) {
// Not a PHT node.
*hi = mid-1;
- lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values);
+ lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
} else {
first_res->done = true;
if (second_res->done)
@@ -104,6 +112,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
}
}
}, pht_filter);
+
if (mid < p.size_)
dht_->get(p.getPrefix(mid+1).hash(),
std::bind(on_get, std::placeholders::_1, second_res),
@@ -132,7 +141,8 @@ void Pht::lookup(Key k, Pht::LookupCallback cb, Dht::DoneCallbackSimple done_cb,
auto lo = std::make_shared<int>(0);
auto hi = std::make_shared<int>(prefix.size_);
std::shared_ptr<unsigned> max_common_prefix_len = not exact_match ? std::make_shared<unsigned>(0) : nullptr;
- lookupStep(prefix, lo, hi, values, cb, done_cb, max_common_prefix_len);
+
+ lookupStep(prefix, lo, hi, values, cb, done_cb, max_common_prefix_len, cache_.lookup(prefix));
}
void Pht::updateCanary(Prefix p) {
@@ -140,7 +150,7 @@ void Pht::updateCanary(Prefix p) {
dht::Value canary_value;
canary_value.user_type = canary_;
dht_->put(p.hash(), std::move(canary_value),
- [=](bool ok){
+ [=](bool){
static std::bernoulli_distribution d(0.5);
crypto::random_device rd;
if (p.size_ && d(rd))
@@ -164,7 +174,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) {
auto final_prefix = std::make_shared<Prefix>();
lookupStep(kp, lo, hi, vals,
- [=](std::vector<std::shared_ptr<Value>>& values, Prefix p) {
+ [=](std::vector<std::shared_ptr<Value>>&, Prefix p) {
*final_prefix = Prefix(p);
},
[=](bool ok){
@@ -183,7 +193,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) {
updateCanary(*final_prefix);
dht_->put(final_prefix->hash(), std::move(entry), done_cb);
}
- }, nullptr, true
+ }, nullptr, cache_.lookup(kp), true
);
}
--
GitLab