pht: add prefix cache

include/opendht/indexation/pht.h

@@ -47,6 +47,16 @@ struct Prefix {
         return Prefix(*this, len);
     }
 
+    /**
+     * Method for getting the state of the bit at the position pos.
+     * @param pos : Pos of the needed bit
+     * @return : true if the bit is at 1
+     *           false otherwise
+     */
+    bool isActivBit(size_t pos) const {
+        return ((this->content_[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1;
+    }
+
     Prefix getFullSize() { return Prefix(*this, content_.size()*8); }
 
     /**
@@ -166,6 +176,125 @@ public:
     void insert(Key k, Value v, Dht::DoneCallbackSimple cb = {});
 
 private:
+    class Cache {
+    public:
+        /**
+         * Insert all needed node into the tree according to a prefix
+         * @param p : Prefix that we need to insert
+         */
+        void insert(const Prefix& p) {
+            size_t i = 0;
+            auto now = clock::now();
+
+            std::shared_ptr<Node> curr_node;
+
+            while ( ( leaves_.size() > 0 && leaves_.begin()->first + NODE_EXPIRE_TIME < now )
+                    || leaves_.size() > MAX_ELEMENT ) {
+
+                leaves_.erase(leaves_.begin());
+            }
+
+            if ( !(curr_node = root_.lock()) ) {
+
+                /* Root does not exist, need to create one*/
+                curr_node = std::make_shared<Node>();
+                root_ = curr_node;
+            }
+            
+            curr_node->last_reply = now;
+
+            /* Iterate through all bit of the Blob */
+            for ( i = 0; i < p.size_; i++ ) {
+
+                /* According to the bit define which node is the next one */
+                auto& next = ( p.isActivBit(i) ) ? curr_node->right_child : curr_node->left_child;
+
+                /** 
+                 * If lock, node exists
+                 * else create it
+                 */
+                if (auto n = next.lock()) {
+                    curr_node = std::move(n);
+                } else {
+                    /* Create the next node if doesn't exist*/
+                    auto tmp_curr_node = std::make_shared<Node>();
+                    tmp_curr_node->parent = curr_node;
+                    next = tmp_curr_node;
+                    curr_node = std::move(tmp_curr_node);
+                }
+
+                curr_node->last_reply = now;
+            }
+
+            /* Insert the leaf (curr_node) into the multimap */
+            leaves_.emplace(std::move(now), std::move(curr_node) );
+        }
+
+        /**
+         * Lookup into the tree to return the maximum prefix length in the cache tree
+         *
+         * @param p : Prefix that we are looking for
+         * @return  : The size of the longest prefix known in the cache between 0 and p.size_
+         */
+        int lookup(const Prefix& p) {
+            int pos = 0;
+            auto now = clock::now(), last_node_time = now;
+            
+            /* Before lookup remove the useless one [i.e. too old] */
+            while ( leaves_.size() > 0 &&  leaves_.begin()->first + NODE_EXPIRE_TIME < now ) {
+                leaves_.erase(leaves_.begin());
+            }
+
+            auto next = root_;
+            std::shared_ptr<Node> curr_node;
+
+            while ( auto n = next.lock() ) {
+                /* Safe since pos is equal to 0 until here */
+                if ( (unsigned) pos >= p.size_ ) break;
+
+                curr_node = n;
+                last_node_time = curr_node->last_reply;
+                curr_node->last_reply = now;
+
+                /* Get the Prefix bit by bit, starting from left */
+                next = ( p.isActivBit(pos) ) ? curr_node->right_child : curr_node->left_child;
+
+                ++pos;
+            }
+
+            if ( pos > 0 ) {
+                auto to_erase = leaves_.find(last_node_time);
+                if ( to_erase != leaves_.end() ) 
+                    leaves_.erase( to_erase );
+
+                leaves_.emplace( std::move(now), std::move(curr_node) );
+            } 
+
+            return --pos;
+        }
+
+
+    private:
+        static constexpr const size_t MAX_ELEMENT {1024};
+        static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {5};
+
+        struct Node {
+            time_point last_reply;           /* Made the assocation between leaves and leaves multimap */
+            std::shared_ptr<Node> parent;    /* Share_ptr to the parent, it allow the self destruction of tree */
+            std::weak_ptr<Node> left_child;  /* Left child, for bit equal to 1 */
+            std::weak_ptr<Node> right_child; /* Right child, for bit equal to 0 */
+        };
+
+        std::weak_ptr<Node> root_;                         /* Root of the tree */
+
+        /**
+         * This mutlimap contains all prefix insert in the tree in time order
+         * We could then delete the last one if there is too much node
+         * The tree will self destroy is branch ( thanks to share_ptr )
+         */
+        std::multimap<time_point, std::shared_ptr<Node>> leaves_;
+    };
+
     /**
      * Linearizes the key into a unidimensional key. A pht only takes
      * unidimensional key.
@@ -184,9 +313,9 @@ private:
      * asynchronously.
      */
     void lookupStep(Prefix k, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
-            std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals,
-            LookupCallback cb, Dht::DoneCallbackSimple done_cb,
-            std::shared_ptr<unsigned> max_common_prefix_len, bool all_values = false);
+            std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals, LookupCallback cb,
+            Dht::DoneCallbackSimple done_cb, std::shared_ptr<unsigned> max_common_prefix_len,
+            int start = -1, bool all_values = false);
 
     /**
      * Updates the canary token on the node responsible for the specified
@@ -196,7 +325,7 @@ private:
 
     const std::string name_;
     const std::string canary_;
-
+    Cache cache_;
     std::shared_ptr<DhtRunner> dht_;
 };
 

src/indexation/pht.cpp

@@ -5,18 +5,20 @@ namespace dht {
 namespace indexation {
 
 const ValueType IndexEntry::TYPE = ValueType::USER_DATA;
+constexpr std::chrono::minutes Pht::Cache::NODE_EXPIRE_TIME;
 
 void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
         std::shared_ptr<std::vector<std::shared_ptr<Value>>> vals,
         LookupCallback cb, Dht::DoneCallbackSimple done_cb,
-        std::shared_ptr<unsigned> max_common_prefix_len, bool all_values)
+        std::shared_ptr<unsigned> max_common_prefix_len, int start, bool all_values)
 {
     struct node_lookup_result {
         bool done {false};
         bool is_pht {false};
     };
 
-    auto mid = (*lo + *hi)/2;
+    /* start could be under 0 but after the compare it to 0 it always will be unsigned, so we can cast it*/
+    auto mid = (start >= 0) ? (unsigned) start : (*lo + *hi)/2;
     auto first_res = std::make_shared<node_lookup_result>();
     auto second_res = std::make_shared<node_lookup_result>();
     auto on_done = [=](bool ok) {
@@ -27,21 +29,26 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
         }
         else if (is_leaf or *lo > *hi) {
             // leaf node
+            auto to_insert = p.getPrefix(mid);
+            cache_.insert(to_insert);
+
             if (cb) {
                 if (vals->size() == 0 and max_common_prefix_len and mid > 0) {
                     auto p_ = (p.getPrefix(mid)).getSibling().getFullSize();
                     *lo = mid;
                     *hi = p_.size_;
-                    lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values);
+                    lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
                 }
-                cb(*vals, p.getPrefix(mid));
+
+                cb(*vals, to_insert);
             }
+
             if (done_cb)
                 done_cb(true);
         } else if (first_res->is_pht) {
             // internal node
             *lo = mid+1;
-            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values);
+            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
         } else {
             // first get failed before second.
             if (done_cb)
@@ -83,6 +90,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
             }
             return true;
         };
+
         dht_->get(p.getPrefix(mid).hash(),
                 std::bind(on_get, std::placeholders::_1, first_res),
                 [=](bool ok) {
@@ -96,7 +104,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
                         if (not first_res->is_pht) {
                             // Not a PHT node.
                             *hi = mid-1;
-                            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, all_values);
+                            lookupStep(p, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
                         } else {
                             first_res->done = true;
                             if (second_res->done)
@@ -104,6 +112,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
                         }
                     }
                 }, pht_filter);
+
         if (mid < p.size_)
            dht_->get(p.getPrefix(mid+1).hash(),
                     std::bind(on_get, std::placeholders::_1, second_res),
@@ -132,7 +141,8 @@ void Pht::lookup(Key k, Pht::LookupCallback cb, Dht::DoneCallbackSimple done_cb,
     auto lo = std::make_shared<int>(0);
     auto hi = std::make_shared<int>(prefix.size_);
     std::shared_ptr<unsigned> max_common_prefix_len = not exact_match ? std::make_shared<unsigned>(0) : nullptr;
-    lookupStep(prefix, lo, hi, values, cb, done_cb, max_common_prefix_len);
+
+    lookupStep(prefix, lo, hi, values, cb, done_cb, max_common_prefix_len, cache_.lookup(prefix));
 }
 
 void Pht::updateCanary(Prefix p) {
@@ -140,7 +150,7 @@ void Pht::updateCanary(Prefix p) {
     dht::Value canary_value;
     canary_value.user_type = canary_;
     dht_->put(p.hash(), std::move(canary_value),
-        [=](bool ok){
+        [=](bool){
             static std::bernoulli_distribution d(0.5);
             crypto::random_device rd;
             if (p.size_ && d(rd))
@@ -164,7 +174,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) {
     auto final_prefix = std::make_shared<Prefix>();
 
     lookupStep(kp, lo, hi, vals,
-        [=](std::vector<std::shared_ptr<Value>>& values, Prefix p) {
+        [=](std::vector<std::shared_ptr<Value>>&, Prefix p) {
             *final_prefix = Prefix(p);
         },
         [=](bool ok){
@@ -183,7 +193,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) {
                 updateCanary(*final_prefix);
                 dht_->put(final_prefix->hash(), std::move(entry), done_cb);
             }
-        }, nullptr, true
+        }, nullptr, cache_.lookup(kp), true
     );
 }