Skip to content
Snippets Groups Projects
Unverified Commit ae387981 authored by kaldoran's avatar kaldoran
Browse files

Rework prefix [add flags] and prepare prefix

parent 5778af28
No related branches found
No related tags found
No related merge requests found
...@@ -29,12 +29,19 @@ namespace indexation { ...@@ -29,12 +29,19 @@ namespace indexation {
struct Prefix { struct Prefix {
Prefix() {} Prefix() {}
Prefix(InfoHash h) : size_(h.size() * 8), content_(h.begin(), h.end()) { } Prefix(InfoHash h) : size_(h.size() * 8), content_(h.begin(), h.end()) { }
Prefix(const Blob& d) : size_(d.size()*8), content_(d) {} Prefix(const Blob& d, const Blob& f={}) : size_(d.size()*8), flags_(f), content_(d) { }
Prefix(const Prefix& p, size_t first) : Prefix(const Prefix& p, size_t first) :
size_(std::min(first, p.content_.size()*8)), size_(std::min(first, p.content_.size()*8)),
content_(Blob(p.content_.begin(), p.content_.begin()+size_/8)) content_(Blob(p.content_.begin(), p.content_.begin()+size_/8))
{ {
auto rem = size_ % 8; auto rem = size_ % 8;
if ( not flags_.empty() ) {
flags_ = Blob(p.flags_.begin(), p.flags_.begin()+size_/8);
if (rem)
flags_.push_back(p.flags_[size_/8] & (0xFF << (8 - rem)));
}
if (rem) if (rem)
content_.push_back(p.content_[size_/8] & (0xFF << (8 - rem))); content_.push_back(p.content_[size_/8] & (0xFF << (8 - rem)));
} }
...@@ -54,11 +61,12 @@ struct Prefix { ...@@ -54,11 +61,12 @@ struct Prefix {
* false otherwise * false otherwise
* @throw out_of_range Throw out of range if the bit at 'pos' does not exist * @throw out_of_range Throw out of range if the bit at 'pos' does not exist
*/ */
bool isActiveBit(size_t pos) const { bool isFlagActive(size_t pos) const {
if ( pos >= content_.size() * 8 ) return flags_.empty() or isActiveBit(flags_, pos);
throw std::out_of_range("Can't detect active bit at pos, pos larger than prefix size or empty prefix"); }
return ((this->content_[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1; bool isContentBitActive(size_t pos) const {
return isActiveBit(content_, pos);
} }
Prefix getFullSize() { return Prefix(*this, content_.size()*8); } Prefix getFullSize() { return Prefix(*this, content_.size()*8); }
...@@ -82,13 +90,15 @@ struct Prefix { ...@@ -82,13 +90,15 @@ struct Prefix {
std::string toString() const { std::string toString() const {
std::stringstream ss; std::stringstream ss;
auto bn = size_ % 8;
auto n = size_ / 8; ss << "Prefix : " << std::endl << "\tContent_ : ";
for (size_t i = 0; i<n; i++) ss << blobToString(content_);
ss << std::bitset<8>(content_[i]); ss << std::endl;
if (bn)
for (unsigned b=0; b<bn; b++) ss << "\tFlags_ : ";
ss << (char)((content_[n] & (1 << (7 - b))) ? '1':'0'); ss << blobToString(flags_);
ss << std::endl;
return ss.str(); return ss.str();
} }
...@@ -105,9 +115,13 @@ struct Prefix { ...@@ -105,9 +115,13 @@ struct Prefix {
auto longest_prefix_size = std::min(p1.size_, p2.size_); auto longest_prefix_size = std::min(p1.size_, p2.size_);
for (i = 0; i < longest_prefix_size; i++) { for (i = 0; i < longest_prefix_size; i++) {
if (p1.content_.data()[i] != p2.content_.data()[i]) if (p1.content_.data()[i] != p2.content_.data()[i]
or not p1.isFlagActive(i)
or not p2.isFlagActive(i) ) {
break; break;
} }
}
if (i == longest_prefix_size) if (i == longest_prefix_size)
return 8*longest_prefix_size; return 8*longest_prefix_size;
...@@ -124,30 +138,87 @@ struct Prefix { ...@@ -124,30 +138,87 @@ struct Prefix {
} }
/** /**
* This method swap the bit a the position 'bit' and return the new prefix * This method swap the bit a the position 'bit'
* *
* @param bit Position of the bit to swap * @param bit Position of the bit to swap
* @return The prefix with the bit at position 'bit' swapped * @return The prefix with the bit at position 'bit' swapped
* @throw out_of_range Throw out of range if bit does not exist * @throw out_of_range Throw out of range if bit does not exist
*/ */
Prefix swapBit(size_t bit) const { Prefix swapBit(size_t bit) const {
if ( bit > content_.size() * 8 ) if ( bit >= content_.size() * 8 )
throw std::out_of_range("bit larger than prefix size."); throw std::out_of_range("bit larger than prefix size.");
Prefix copy = *this; void swapFlagBit(size_t bit) {
size_t offset_bit = (8 - bit) % 8; swapBit(flags_, bit);
copy.content_[bit / 8] ^= (1 << offset_bit); }
return copy; void addPaddingContent(size_t size) {
content_ = addPadding(content_, size);
}
void updateFlags() {
// Fill first known bit
auto csize = size_ - flags_.size() * 8;
while(csize >= 8) {
flags_.push_back(0xFF);
csize -= 8;
}
// if needed fill remaining bit
if ( csize )
flags_.push_back(0xFF << (8 - csize));
// Complet vector space missing
for ( auto i = flags_.size(); i < content_.size(); i++ )
flags_.push_back(0xFF);
} }
size_t size_ {0}; size_t size_ {0};
Blob flags_ {};
Blob content_ {}; Blob content_ {};
private:
std::string blobToString(const Blob &bl) const {
std::stringstream ss;
auto bn = size_ % 8;
auto n = size_ / 8;
for (size_t i = 0; i < bl.size(); i++)
ss << std::bitset<8>(bl[i]) << " ";
if (bn)
for (unsigned b=0; b < bn; b++)
ss << (char)((bl[n] & (1 << (7 - b))) ? '1':'0');
return ss.str();
}
Blob addPadding(Blob toP, size_t size) {
Blob copy = toP;
for ( auto i = copy.size(); i < size; i++ )
copy.push_back(0);
swapBit(copy, size_ + 1);
return copy;
}
bool isActiveBit(const Blob &b, size_t pos) const {
if ( pos >= size_ )
throw std::out_of_range("Can't detect active bit at pos, pos larger than prefix size or empty prefix");
return ((b[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1;
}
void swapBit(Blob &b, size_t bit) {
if ( bit >= b.size() * 8 )
throw std::out_of_range("bit larger than prefix size.");
size_t offset_bit = (8 - bit) % 8;
b[bit / 8] ^= (1 << offset_bit);
}
}; };
using Value = std::pair<InfoHash, dht::Value::Id>; using Value = std::pair<InfoHash, dht::Value::Id>;
struct IndexEntry : public dht::Value::Serializable<IndexEntry> { struct IndexEntry : public dht::Value::Serializable<IndexEntry> {
static const ValueType TYPE; static const ValueType TYPE;
...@@ -168,7 +239,6 @@ struct IndexEntry : public dht::Value::Serializable<IndexEntry> { ...@@ -168,7 +239,6 @@ struct IndexEntry : public dht::Value::Serializable<IndexEntry> {
MSGPACK_DEFINE_MAP(prefix, value); MSGPACK_DEFINE_MAP(prefix, value);
}; };
class Pht { class Pht {
static constexpr const char* INVALID_KEY = "Key does not match the PHT key spec."; static constexpr const char* INVALID_KEY = "Key does not match the PHT key spec.";
...@@ -176,6 +246,7 @@ class Pht { ...@@ -176,6 +246,7 @@ class Pht {
static constexpr const char* INDEX_PREFIX = "index.pht."; static constexpr const char* INDEX_PREFIX = "index.pht.";
public: public:
/* This is the maximum number of entries per node. This parameter is /* This is the maximum number of entries per node. This parameter is
* critical and influences the traffic a lot during a lookup operation. * critical and influences the traffic a lot during a lookup operation.
*/ */
...@@ -207,11 +278,8 @@ public: ...@@ -207,11 +278,8 @@ public:
} }
Pht(std::string name, KeySpec k_spec, std::shared_ptr<DhtRunner> dht) Pht(std::string name, KeySpec k_spec, std::shared_ptr<DhtRunner> dht)
: name_(INDEX_PREFIX + name), canary_(name_ + ".canary"), keySpec_(k_spec), dht_(dht) : name_(INDEX_PREFIX + name), canary_(name_ + ".canary"), keySpec_(k_spec), dht_(dht) {}
{
if (k_spec.size() != 1)
throw std::invalid_argument("PHT only supports unidimensional data.");
}
virtual ~Pht () { } virtual ~Pht () { }
/** /**
...@@ -281,7 +349,7 @@ private: ...@@ -281,7 +349,7 @@ private:
private: private:
static constexpr const size_t MAX_ELEMENT {1024}; static constexpr const size_t MAX_ELEMENT {1024};
static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {10}; static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {5};
struct Node { struct Node {
time_point last_reply; /* Made the assocation between leaves and leaves multimap */ time_point last_reply; /* Made the assocation between leaves and leaves multimap */
...@@ -324,6 +392,33 @@ private: ...@@ -324,6 +392,33 @@ private:
std::shared_ptr<unsigned> max_common_prefix_len, std::shared_ptr<unsigned> max_common_prefix_len,
int start = -1, bool all_values = false); int start = -1, bool all_values = false);
Prefix zcurve(const std::vector<Prefix>& all_prefix) const {
Prefix p;
if ( all_prefix.size() == 1 ) return all_prefix[0];
for ( size_t j = 0, bit = 0; j < all_prefix[0].content_.size(); j++) {
uint8_t mask = 0x80;
for ( int i = 0; i < 8; ) {
uint8_t content = 0;
uint8_t flags = 0;
for ( int k = 0 ; k < 8; k++, bit++ ) {
auto diff = k - i;
auto x = all_prefix[bit].content_[j] & mask;
auto y = all_prefix[bit].flags_[j] & mask;
content |= ( diff >= 0 ) ? x >> diff : x << std::abs(diff);
flags |= ( diff >= 0 ) ? y >> diff : y << std::abs(diff);
if ( bit == all_prefix.size() - 1 ) { bit = -1; ++i; mask >>= 1; }
}
p.content_.push_back(content);
p.flags_.push_back(flags);
p.size_ += 8;
}
}
return p;
}
/** /**
* Linearizes the key into a unidimensional key. A pht only takes * Linearizes the key into a unidimensional key. A pht only takes
* unidimensional key. * unidimensional key.
......
...@@ -4,7 +4,7 @@ lib_LTLIBRARIES = libopendht.la ...@@ -4,7 +4,7 @@ lib_LTLIBRARIES = libopendht.la
AM_CPPFLAGS = -I../include/opendht AM_CPPFLAGS = -I../include/opendht
libopendht_la_CXXFLAGS = @CXXFLAGS@ libopendht_la_CXXFLAGS = @CXXFLAGS@
libopendht_la_LDFLAGS = @LDFLAGS@ @GNUTLS_LIBS@ @nettle_LIBS@ libopendht_la_LDFLAGS = @LDFLAGS@ @GNUTLS_LIBS@ @nettle_LIBS@ -version-info 6:0:2
libopendht_la_LIBADD = ./argon2/libargon2.la libopendht_la_LIBADD = ./argon2/libargon2.la
libopendht_la_SOURCES = \ libopendht_la_SOURCES = \
......
...@@ -10,8 +10,12 @@ void Pht::Cache::insert(const Prefix& p) { ...@@ -10,8 +10,12 @@ void Pht::Cache::insert(const Prefix& p) {
std::shared_ptr<Node> curr_node; std::shared_ptr<Node> curr_node;
while ((leaves_.size() > 0 and leaves_.begin()->first + NODE_EXPIRE_TIME < now) or leaves_.size() > MAX_ELEMENT) while ((leaves_.size() > 0
and leaves_.begin()->first + NODE_EXPIRE_TIME < now)
or leaves_.size() > MAX_ELEMENT) {
leaves_.erase(leaves_.begin()); leaves_.erase(leaves_.begin());
}
if (not (curr_node = root_.lock()) ) { if (not (curr_node = root_.lock()) ) {
/* Root does not exist, need to create one*/ /* Root does not exist, need to create one*/
...@@ -25,7 +29,8 @@ void Pht::Cache::insert(const Prefix& p) { ...@@ -25,7 +29,8 @@ void Pht::Cache::insert(const Prefix& p) {
for ( i = 0; i < p.size_; i++ ) { for ( i = 0; i < p.size_; i++ ) {
/* According to the bit define which node is the next one */ /* According to the bit define which node is the next one */
auto& next = ( p.isActiveBit(i) ) ? curr_node->right_child : curr_node->left_child; auto& next = ( p.isContentBitActive(i) ) ? curr_node->right_child : curr_node->left_child;
/** /**
* If lock, node exists * If lock, node exists
* else create it * else create it
...@@ -52,7 +57,9 @@ int Pht::Cache::lookup(const Prefix& p) { ...@@ -52,7 +57,9 @@ int Pht::Cache::lookup(const Prefix& p) {
auto now = clock::now(), last_node_time = now; auto now = clock::now(), last_node_time = now;
/* Before lookup remove the useless one [i.e. too old] */ /* Before lookup remove the useless one [i.e. too old] */
while ( leaves_.size() > 0 and leaves_.begin()->first + NODE_EXPIRE_TIME < now ) { while ( leaves_.size() > 0
and leaves_.begin()->first + NODE_EXPIRE_TIME < now ) {
leaves_.erase(leaves_.begin()); leaves_.erase(leaves_.begin());
} }
...@@ -69,7 +76,7 @@ int Pht::Cache::lookup(const Prefix& p) { ...@@ -69,7 +76,7 @@ int Pht::Cache::lookup(const Prefix& p) {
curr_node->last_reply = now; curr_node->last_reply = now;
/* Get the Prefix bit by bit, starting from left */ /* Get the Prefix bit by bit, starting from left */
next = ( p.isActiveBit(pos) ) ? curr_node->right_child : curr_node->left_child; next = ( p.isContentBitActive(pos) ) ? curr_node->right_child : curr_node->left_child;
} }
if ( pos >= 0 ) { if ( pos >= 0 ) {
...@@ -110,7 +117,6 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, ...@@ -110,7 +117,6 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
} }
else if (is_leaf or *lo > *hi) { else if (is_leaf or *lo > *hi) {
// leaf node // leaf node
Prefix to_insert = p.getPrefix(mid);
if (cb) { if (cb) {
if (vals->size() == 0 and max_common_prefix_len and mid > 0) { if (vals->size() == 0 and max_common_prefix_len and mid > 0) {
...@@ -120,7 +126,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, ...@@ -120,7 +126,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values); lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
} }
cb(*vals, to_insert); cb(*vals, p.getPrefix(mid));
} }
if (done_cb) if (done_cb)
...@@ -151,6 +157,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, ...@@ -151,6 +157,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
if (max_common_prefix_len) { /* inexact match case */ if (max_common_prefix_len) { /* inexact match case */
auto common_bits = Prefix::commonBits(p, entry.prefix); auto common_bits = Prefix::commonBits(p, entry.prefix);
if (vals->empty()) { if (vals->empty()) {
vals->emplace_back(std::make_shared<IndexEntry>(entry)); vals->emplace_back(std::make_shared<IndexEntry>(entry));
*max_common_prefix_len = common_bits; *max_common_prefix_len = common_bits;
...@@ -167,6 +174,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi, ...@@ -167,6 +174,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
} else if (all_values or entry.prefix == p.content_) /* exact match case */ } else if (all_values or entry.prefix == p.content_) /* exact match case */
vals->emplace_back(std::make_shared<IndexEntry>(entry)); vals->emplace_back(std::make_shared<IndexEntry>(entry));
} }
return true; return true;
}; };
...@@ -278,7 +286,7 @@ void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shar ...@@ -278,7 +286,7 @@ void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shar
updateCanary(*p); updateCanary(*p);
checkPhtUpdate(*p, entry, time_p); checkPhtUpdate(*p, entry, time_p);
cache_.insert(*p); cache_.insert(*p);
dht_->put(p->hash(), std::move(entry), done_cb, time_p); dht_->put(p->hash(), std::move(entry), done_cb /*, time_p */);
}; };
if ( not check_split or final_prefix->size_ == kp.size_ ) { if ( not check_split or final_prefix->size_ == kp.size_ ) {
...@@ -294,19 +302,49 @@ void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shar ...@@ -294,19 +302,49 @@ void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shar
); );
} }
Prefix Pht::linearize(Key k) const { Prefix linearize(Key k) const {
if (not validKey(k)) { throw std::invalid_argument(INVALID_KEY); } if (not validKey(k)) { throw std::invalid_argument(INVALID_KEY); }
std::vector<Prefix> all_prefix;
auto max = std::max_element(keySpec_.begin(), keySpec_.end(),
[](const std::pair<string, size_t>& a, const std::pair<string, size_t>& b) {
return a.second < b.second;
});
for ( auto i = 0; i < k.size; i++ ) {
Prefix p = Blob {k.begin()->second.begin(), k.begin()->second.end()}; Prefix p = Blob {k.begin()->second.begin(), k.begin()->second.end()};
p.addPaddingContent(max);
p.updateFlags();
all_prefix.push_back(p);
}
return zcurve(all_prefix);
/*
virtual Prefix linearize(Key k) const {
if (not validKey(k)) { throw std::invalid_argument(INVALID_KEY); }
std::vector<Prefix> all_prefix;
auto max = std::max_element(keySpec_.begin(), keySpec_.end(),
[&](const std::pair<std::string, size_t>& a, const std::pair<std::string, size_t>& b) {
return a.second < b.second;
})->second + 1;
for ( auto const& it : k ) {
Prefix p = Blob {it.second.begin(), it.second.end()};
p.addPaddingContent(max);
p.updateFlags();
all_prefix.push_back(p);
auto bit_loc = p.size_ + 1; auto bit_loc = p.size_ + 1;
for ( auto i = p.content_.size(); i < keySpec_.begin()->second + 1; i++ ) for ( auto i = p.content_.size(); i < keySpec_.begin()->second + 1; i++ )
p.content_.push_back(0); p.content_.push_back(0);
return p.swapBit(bit_loc); return zcurve(all_prefix);
};*/
}; };
void Pht::getRealPrefix(std::shared_ptr<Prefix> p, IndexEntry entry, RealInsertCallback end_cb ) { void Pht::getRealPrefix(std::shared_ptr<Prefix> p, IndexEntry entry, RealInsertCallback end_cb ) {
if ( p->size_ == 0 ) { if ( p->size_ == 0 ) {
end_cb(p, std::move(entry)); end_cb(p, std::move(entry));
return; return;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment