Skip to content
Snippets Groups Projects
Commit cdca40cb authored by kaldoran's avatar kaldoran Committed by Adrien Béraud
Browse files

Rework prefix [add flags] and prepare prefix

parent 2a3c3013
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
include/opendht/indexation/pht.h
+ 124
− 29
View file @ cdca40cb
......@@ -28,13 +28,20 @@ namespace indexation {
*/
struct Prefix {
Prefix() {}
Prefix(InfoHash h) : size_(h.size() * 8), content_(h.begin(), h.end()) {}
Prefix(const Blob& d) : size_(d.size()*8), content_(d) {}
Prefix(InfoHash h) : size_(h.size() * 8), content_(h.begin(), h.end()) { }
Prefix(const Blob& d, const Blob& f={}) : size_(d.size()*8), flags_(f), content_(d) { }
Prefix(const Prefix& p, size_t first) :
size_(std::min(first, p.content_.size()*8)),
content_(Blob(p.content_.begin(), p.content_.begin()+size_/8))
{
auto rem = size_ % 8;
if ( not flags_.empty() ) {
flags_ = Blob(p.flags_.begin(), p.flags_.begin()+size_/8);
if (rem)
flags_.push_back(p.flags_[size_/8] & (0xFF << (8 - rem)));
}
if (rem)
content_.push_back(p.content_[size_/8] & (0xFF << (8 - rem)));
}
......@@ -54,11 +61,12 @@ struct Prefix {
* false otherwise
* @throw out_of_range Throw out of range if the bit at 'pos' does not exist
*/
bool isActiveBit(size_t pos) const {
if ( pos >= content_.size() * 8 )
throw std::out_of_range("Can't detect active bit at pos, pos larger than prefix size or empty prefix");
bool isFlagActive(size_t pos) const {
return flags_.empty() or isActiveBit(flags_, pos);
}
return ((this->content_[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1;
bool isContentBitActive(size_t pos) const {
return isActiveBit(content_, pos);
}
Prefix getFullSize() { return Prefix(*this, content_.size()*8); }
......@@ -82,13 +90,15 @@ struct Prefix {
std::string toString() const {
std::stringstream ss;
auto bn = size_ % 8;
auto n = size_ / 8;
for (size_t i = 0; i<n; i++)
ss << std::bitset<8>(content_[i]);
if (bn)
for (unsigned b=0; b<bn; b++)
ss << (char)((content_[n] & (1 << (7 - b))) ? '1':'0');
ss << "Prefix : " << std::endl << "\tContent_ : ";
ss << blobToString(content_);
ss << std::endl;
ss << "\tFlags_ : ";
ss << blobToString(flags_);
ss << std::endl;
return ss.str();
}
......@@ -105,8 +115,12 @@ struct Prefix {
auto longest_prefix_size = std::min(p1.size_, p2.size_);
for (i = 0; i < longest_prefix_size; i++) {
if (p1.content_.data()[i] != p2.content_.data()[i])
break;
if (p1.content_.data()[i] != p2.content_.data()[i]
or not p1.isFlagActive(i)
or not p2.isFlagActive(i) ) {
break;
}
}
if (i == longest_prefix_size)
......@@ -124,30 +138,87 @@ struct Prefix {
}
/**
* This method swap the bit a the position 'bit' and return the new prefix
* This method swap the bit a the position 'bit'
*
* @param bit Position of the bit to swap
* @return The prefix with the bit at position 'bit' swapped
* @throw out_of_range Throw out of range if bit does not exist
*/
Prefix swapBit(size_t bit) const {
if ( bit > content_.size() * 8 )
if ( bit >= content_.size() * 8 )
throw std::out_of_range("bit larger than prefix size.");
Prefix copy = *this;
size_t offset_bit = (8 - bit) % 8;
copy.content_[bit / 8] ^= (1 << offset_bit);
void swapFlagBit(size_t bit) {
swapBit(flags_, bit);
}
return copy;
void addPaddingContent(size_t size) {
content_ = addPadding(content_, size);
}
void updateFlags() {
// Fill first known bit
auto csize = size_ - flags_.size() * 8;
while(csize >= 8) {
flags_.push_back(0xFF);
csize -= 8;
}
// if needed fill remaining bit
if ( csize )
flags_.push_back(0xFF << (8 - csize));
// Complet vector space missing
for ( auto i = flags_.size(); i < content_.size(); i++ )
flags_.push_back(0xFF);
}
size_t size_ {0};
Blob flags_ {};
Blob content_ {};
private:
std::string blobToString(const Blob &bl) const {
std::stringstream ss;
auto bn = size_ % 8;
auto n = size_ / 8;
for (size_t i = 0; i < bl.size(); i++)
ss << std::bitset<8>(bl[i]) << " ";
if (bn)
for (unsigned b=0; b < bn; b++)
ss << (char)((bl[n] & (1 << (7 - b))) ? '1':'0');
return ss.str();
}
Blob addPadding(Blob toP, size_t size) {
Blob copy = toP;
for ( auto i = copy.size(); i < size; i++ )
copy.push_back(0);
swapBit(copy, size_ + 1);
return copy;
}
bool isActiveBit(const Blob &b, size_t pos) const {
if ( pos >= size_ )
throw std::out_of_range("Can't detect active bit at pos, pos larger than prefix size or empty prefix");
return ((b[pos / 8] >> (7 - (pos % 8)) ) & 1) == 1;
}
void swapBit(Blob &b, size_t bit) {
if ( bit >= b.size() * 8 )
throw std::out_of_range("bit larger than prefix size.");
size_t offset_bit = (8 - bit) % 8;
b[bit / 8] ^= (1 << offset_bit);
}
};
using Value = std::pair<InfoHash, dht::Value::Id>;
struct IndexEntry : public dht::Value::Serializable<IndexEntry> {
static const ValueType TYPE;
......@@ -168,7 +239,6 @@ struct IndexEntry : public dht::Value::Serializable<IndexEntry> {
MSGPACK_DEFINE_MAP(prefix, value);
};
class Pht {
static constexpr const char* INVALID_KEY = "Key does not match the PHT key spec.";
......@@ -176,6 +246,7 @@ class Pht {
static constexpr const char* INDEX_PREFIX = "index.pht.";
public:
/* This is the maximum number of entries per node. This parameter is
* critical and influences the traffic a lot during a lookup operation.
*/
......@@ -207,11 +278,8 @@ public:
}
Pht(std::string name, KeySpec k_spec, std::shared_ptr<DhtRunner> dht)
: name_(INDEX_PREFIX + name), canary_(name_ + ".canary"), keySpec_(k_spec), dht_(dht)
{
if (k_spec.size() != 1)
throw std::invalid_argument("PHT only supports unidimensional data.");
}
: name_(INDEX_PREFIX + name), canary_(name_ + ".canary"), keySpec_(k_spec), dht_(dht) {}
virtual ~Pht () { }
/**
......@@ -281,7 +349,7 @@ private:
private:
static constexpr const size_t MAX_ELEMENT {1024};
static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {10};
static constexpr const std::chrono::minutes NODE_EXPIRE_TIME {5};
struct Node {
time_point last_reply; /* Made the assocation between leaves and leaves multimap */
......@@ -324,6 +392,33 @@ private:
std::shared_ptr<unsigned> max_common_prefix_len,
int start = -1, bool all_values = false);
Prefix zcurve(const std::vector<Prefix>& all_prefix) const {
Prefix p;
if ( all_prefix.size() == 1 ) return all_prefix[0];
for ( size_t j = 0, bit = 0; j < all_prefix[0].content_.size(); j++) {
uint8_t mask = 0x80;
for ( int i = 0; i < 8; ) {
uint8_t content = 0;
uint8_t flags = 0;
for ( int k = 0 ; k < 8; k++, bit++ ) {
auto diff = k - i;
auto x = all_prefix[bit].content_[j] & mask;
auto y = all_prefix[bit].flags_[j] & mask;
content |= ( diff >= 0 ) ? x >> diff : x << std::abs(diff);
flags |= ( diff >= 0 ) ? y >> diff : y << std::abs(diff);
if ( bit == all_prefix.size() - 1 ) { bit = -1; ++i; mask >>= 1; }
}
p.content_.push_back(content);
p.flags_.push_back(flags);
p.size_ += 8;
}
}
return p;
}
/**
* Linearizes the key into a unidimensional key. A pht only takes
* unidimensional key.
......
src/Makefile.am
+ 2
− 1
View file @ cdca40cb
......@@ -3,7 +3,8 @@ SUBDIRS = argon2
lib_LTLIBRARIES = libopendht.la
AM_CPPFLAGS = -I../include/opendht
libopendht_la_LDFLAGS = @LDFLAGS@ @GnuTLS_LIBS@ @Nettle_LIBS@ -version-number @OPENDHT_MAJOR_VERSION@:@OPENDHT_MINOR_VERSION@:@OPENDHT_PATCH_VERSION@
libopendht_la_CXXFLAGS = @CXXFLAGS@
libopendht_la_LDFLAGS = @LDFLAGS@ @GNUTLS_LIBS@ @nettle_LIBS@ -version-info 6:0:2
libopendht_la_LIBADD = ./argon2/libargon2.la
libopendht_la_SOURCES = \
......
src/indexation/pht.cpp
+ 53
− 15
View file @ cdca40cb
......@@ -10,8 +10,12 @@ void Pht::Cache::insert(const Prefix& p) {
std::shared_ptr<Node> curr_node;
while ((leaves_.size() > 0 and leaves_.begin()->first + NODE_EXPIRE_TIME < now) or leaves_.size() > MAX_ELEMENT)
while ((leaves_.size() > 0
and leaves_.begin()->first + NODE_EXPIRE_TIME < now)
or leaves_.size() > MAX_ELEMENT) {
leaves_.erase(leaves_.begin());
}
if (not (curr_node = root_.lock()) ) {
/* Root does not exist, need to create one*/
......@@ -25,7 +29,8 @@ void Pht::Cache::insert(const Prefix& p) {
for ( i = 0; i < p.size_; i++ ) {
/* According to the bit define which node is the next one */
auto& next = ( p.isActiveBit(i) ) ? curr_node->right_child : curr_node->left_child;
auto& next = ( p.isContentBitActive(i) ) ? curr_node->right_child : curr_node->left_child;
/**
* If lock, node exists
* else create it
......@@ -52,7 +57,9 @@ int Pht::Cache::lookup(const Prefix& p) {
auto now = clock::now(), last_node_time = now;
/* Before lookup remove the useless one [i.e. too old] */
while ( leaves_.size() > 0 and leaves_.begin()->first + NODE_EXPIRE_TIME < now ) {
while ( leaves_.size() > 0
and leaves_.begin()->first + NODE_EXPIRE_TIME < now ) {
leaves_.erase(leaves_.begin());
}
......@@ -69,7 +76,7 @@ int Pht::Cache::lookup(const Prefix& p) {
curr_node->last_reply = now;
/* Get the Prefix bit by bit, starting from left */
next = ( p.isActiveBit(pos) ) ? curr_node->right_child : curr_node->left_child;
next = ( p.isContentBitActive(pos) ) ? curr_node->right_child : curr_node->left_child;
}
if ( pos >= 0 ) {
......@@ -110,7 +117,6 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
}
else if (is_leaf or *lo > *hi) {
// leaf node
Prefix to_insert = p.getPrefix(mid);
if (cb) {
if (vals->size() == 0 and max_common_prefix_len and mid > 0) {
......@@ -120,7 +126,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
lookupStep(p_, lo, hi, vals, cb, done_cb, max_common_prefix_len, -1, all_values);
}
cb(*vals, to_insert);
cb(*vals, p.getPrefix(mid));
}
if (done_cb)
......@@ -151,6 +157,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
if (max_common_prefix_len) { /* inexact match case */
auto common_bits = Prefix::commonBits(p, entry.prefix);
if (vals->empty()) {
vals->emplace_back(std::make_shared<IndexEntry>(entry));
*max_common_prefix_len = common_bits;
......@@ -167,6 +174,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
} else if (all_values or entry.prefix == p.content_) /* exact match case */
vals->emplace_back(std::make_shared<IndexEntry>(entry));
}
return true;
};
......@@ -278,7 +286,7 @@ void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shar
updateCanary(*p);
checkPhtUpdate(*p, entry, time_p);
cache_.insert(*p);
dht_->put(p->hash(), std::move(entry), done_cb, time_p);
dht_->put(p->hash(), std::move(entry), done_cb /*, time_p */);
};
if ( not check_split or final_prefix->size_ == kp.size_ ) {
......@@ -294,19 +302,49 @@ void Pht::insert(Prefix kp, IndexEntry entry, std::shared_ptr<int> lo, std::shar
);
}
Prefix Pht::linearize(Key k) const {
Prefix linearize(Key k) const {
if (not validKey(k)) { throw std::invalid_argument(INVALID_KEY); }
std::vector<Prefix> all_prefix;
auto max = std::max_element(keySpec_.begin(), keySpec_.end(),
[](const std::pair<string, size_t>& a, const std::pair<string, size_t>& b) {
return a.second < b.second;
});
for ( auto i = 0; i < k.size; i++ ) {
Prefix p = Blob {k.begin()->second.begin(), k.begin()->second.end()};
p.addPaddingContent(max);
p.updateFlags();
all_prefix.push_back(p);
}
Prefix p = Blob {k.begin()->second.begin(), k.begin()->second.end()};
auto bit_loc = p.size_ + 1;
for ( auto i = p.content_.size(); i < keySpec_.begin()->second + 1; i++ )
p.content_.push_back(0);
return p.swapBit(bit_loc);
return zcurve(all_prefix);
/*
virtual Prefix linearize(Key k) const {
if (not validKey(k)) { throw std::invalid_argument(INVALID_KEY); }
std::vector<Prefix> all_prefix;
auto max = std::max_element(keySpec_.begin(), keySpec_.end(),
[&](const std::pair<std::string, size_t>& a, const std::pair<std::string, size_t>& b) {
return a.second < b.second;
})->second + 1;
for ( auto const& it : k ) {
Prefix p = Blob {it.second.begin(), it.second.end()};
p.addPaddingContent(max);
p.updateFlags();
all_prefix.push_back(p);
auto bit_loc = p.size_ + 1;
for ( auto i = p.content_.size(); i < keySpec_.begin()->second + 1; i++ )
p.content_.push_back(0);
return zcurve(all_prefix);
};*/
};
void Pht::getRealPrefix(std::shared_ptr<Prefix> p, IndexEntry entry, RealInsertCallback end_cb ) {
if ( p->size_ == 0 ) {
end_cb(p, std::move(entry));
return;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment