Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
opendht
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Code
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Locked files
Deploy
Model registry
Analyze
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
savoirfairelinux
opendht
Commits
33fae1cb
Commit
33fae1cb
authored
9 years ago
by
kaldoran
Committed by
Adrien Béraud
8 years ago
Browse files
Options
Downloads
Patches
Plain Diff
pht: add prefix cache
parent
4e8408e2
No related branches found
No related tags found
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
include/opendht/indexation/pht.h
+134
-5
134 additions, 5 deletions
include/opendht/indexation/pht.h
src/indexation/pht.cpp
+20
-10
20 additions, 10 deletions
src/indexation/pht.cpp
with
154 additions
and
15 deletions
include/opendht/indexation/pht.h
+
134
−
5
View file @
33fae1cb
...
...
@@ -47,6 +47,16 @@ struct Prefix {
return
Prefix
(
*
this
,
len
);
}
/**
* Method for getting the state of the bit at the position pos.
* @param pos : Pos of the needed bit
* @return : true if the bit is at 1
* false otherwise
*/
bool
isActivBit
(
size_t
pos
)
const
{
return
((
this
->
content_
[
pos
/
8
]
>>
(
7
-
(
pos
%
8
))
)
&
1
)
==
1
;
}
Prefix
getFullSize
()
{
return
Prefix
(
*
this
,
content_
.
size
()
*
8
);
}
/**
...
...
@@ -166,6 +176,125 @@ public:
void
insert
(
Key
k
,
Value
v
,
Dht
::
DoneCallbackSimple
cb
=
{});
private
:
class
Cache
{
public:
/**
* Insert all needed node into the tree according to a prefix
* @param p : Prefix that we need to insert
*/
void
insert
(
const
Prefix
&
p
)
{
size_t
i
=
0
;
auto
now
=
clock
::
now
();
std
::
shared_ptr
<
Node
>
curr_node
;
while
(
(
leaves_
.
size
()
>
0
&&
leaves_
.
begin
()
->
first
+
NODE_EXPIRE_TIME
<
now
)
||
leaves_
.
size
()
>
MAX_ELEMENT
)
{
leaves_
.
erase
(
leaves_
.
begin
());
}
if
(
!
(
curr_node
=
root_
.
lock
())
)
{
/* Root does not exist, need to create one*/
curr_node
=
std
::
make_shared
<
Node
>
();
root_
=
curr_node
;
}
curr_node
->
last_reply
=
now
;
/* Iterate through all bit of the Blob */
for
(
i
=
0
;
i
<
p
.
size_
;
i
++
)
{
/* According to the bit define which node is the next one */
auto
&
next
=
(
p
.
isActivBit
(
i
)
)
?
curr_node
->
right_child
:
curr_node
->
left_child
;
/**
* If lock, node exists
* else create it
*/
if
(
auto
n
=
next
.
lock
())
{
curr_node
=
std
::
move
(
n
);
}
else
{
/* Create the next node if doesn't exist*/
auto
tmp_curr_node
=
std
::
make_shared
<
Node
>
();
tmp_curr_node
->
parent
=
curr_node
;
next
=
tmp_curr_node
;
curr_node
=
std
::
move
(
tmp_curr_node
);
}
curr_node
->
last_reply
=
now
;
}
/* Insert the leaf (curr_node) into the multimap */
leaves_
.
emplace
(
std
::
move
(
now
),
std
::
move
(
curr_node
)
);
}
/**
* Lookup into the tree to return the maximum prefix length in the cache tree
*
* @param p : Prefix that we are looking for
* @return : The size of the longest prefix known in the cache between 0 and p.size_
*/
int
lookup
(
const
Prefix
&
p
)
{
int
pos
=
0
;
auto
now
=
clock
::
now
(),
last_node_time
=
now
;
/* Before lookup remove the useless one [i.e. too old] */
while
(
leaves_
.
size
()
>
0
&&
leaves_
.
begin
()
->
first
+
NODE_EXPIRE_TIME
<
now
)
{
leaves_
.
erase
(
leaves_
.
begin
());
}
auto
next
=
root_
;
std
::
shared_ptr
<
Node
>
curr_node
;
while
(
auto
n
=
next
.
lock
()
)
{
/* Safe since pos is equal to 0 until here */
if
(
(
unsigned
)
pos
>=
p
.
size_
)
break
;
curr_node
=
n
;
last_node_time
=
curr_node
->
last_reply
;
curr_node
->
last_reply
=
now
;
/* Get the Prefix bit by bit, starting from left */
next
=
(
p
.
isActivBit
(
pos
)
)
?
curr_node
->
right_child
:
curr_node
->
left_child
;
++
pos
;
}
if
(
pos
>
0
)
{
auto
to_erase
=
leaves_
.
find
(
last_node_time
);
if
(
to_erase
!=
leaves_
.
end
()
)
leaves_
.
erase
(
to_erase
);
leaves_
.
emplace
(
std
::
move
(
now
),
std
::
move
(
curr_node
)
);
}
return
--
pos
;
}
private
:
static
constexpr
const
size_t
MAX_ELEMENT
{
1024
};
static
constexpr
const
std
::
chrono
::
minutes
NODE_EXPIRE_TIME
{
5
};
struct
Node
{
time_point
last_reply
;
/* Made the assocation between leaves and leaves multimap */
std
::
shared_ptr
<
Node
>
parent
;
/* Share_ptr to the parent, it allow the self destruction of tree */
std
::
weak_ptr
<
Node
>
left_child
;
/* Left child, for bit equal to 1 */
std
::
weak_ptr
<
Node
>
right_child
;
/* Right child, for bit equal to 0 */
};
std
::
weak_ptr
<
Node
>
root_
;
/* Root of the tree */
/**
* This mutlimap contains all prefix insert in the tree in time order
* We could then delete the last one if there is too much node
* The tree will self destroy is branch ( thanks to share_ptr )
*/
std
::
multimap
<
time_point
,
std
::
shared_ptr
<
Node
>>
leaves_
;
};
/**
* Linearizes the key into a unidimensional key. A pht only takes
* unidimensional key.
...
...
@@ -184,9 +313,9 @@ private:
* asynchronously.
*/
void
lookupStep
(
Prefix
k
,
std
::
shared_ptr
<
int
>
lo
,
std
::
shared_ptr
<
int
>
hi
,
std
::
shared_ptr
<
std
::
vector
<
std
::
shared_ptr
<
Value
>>>
vals
,
LookupCallback
cb
,
Dht
::
DoneCallbackSimple
done_cb
,
std
::
shared_ptr
<
unsigned
>
max_common_prefix_len
,
bool
all_values
=
false
);
std
::
shared_ptr
<
std
::
vector
<
std
::
shared_ptr
<
Value
>>>
vals
,
LookupCallback
cb
,
Dht
::
DoneCallbackSimple
done_cb
,
std
::
shared_ptr
<
unsigned
>
max_common_prefix_len
,
int
start
=
-
1
,
bool
all_values
=
false
);
/**
* Updates the canary token on the node responsible for the specified
...
...
@@ -196,7 +325,7 @@ private:
const
std
::
string
name_
;
const
std
::
string
canary_
;
Cache
cache_
;
std
::
shared_ptr
<
DhtRunner
>
dht_
;
};
...
...
This diff is collapsed.
Click to expand it.
src/indexation/pht.cpp
+
20
−
10
View file @
33fae1cb
...
...
@@ -5,18 +5,20 @@ namespace dht {
namespace
indexation
{
const
ValueType
IndexEntry
::
TYPE
=
ValueType
::
USER_DATA
;
constexpr
std
::
chrono
::
minutes
Pht
::
Cache
::
NODE_EXPIRE_TIME
;
void
Pht
::
lookupStep
(
Prefix
p
,
std
::
shared_ptr
<
int
>
lo
,
std
::
shared_ptr
<
int
>
hi
,
std
::
shared_ptr
<
std
::
vector
<
std
::
shared_ptr
<
Value
>>>
vals
,
LookupCallback
cb
,
Dht
::
DoneCallbackSimple
done_cb
,
std
::
shared_ptr
<
unsigned
>
max_common_prefix_len
,
bool
all_values
)
std
::
shared_ptr
<
unsigned
>
max_common_prefix_len
,
int
start
,
bool
all_values
)
{
struct
node_lookup_result
{
bool
done
{
false
};
bool
is_pht
{
false
};
};
auto
mid
=
(
*
lo
+
*
hi
)
/
2
;
/* start could be under 0 but after the compare it to 0 it always will be unsigned, so we can cast it*/
auto
mid
=
(
start
>=
0
)
?
(
unsigned
)
start
:
(
*
lo
+
*
hi
)
/
2
;
auto
first_res
=
std
::
make_shared
<
node_lookup_result
>
();
auto
second_res
=
std
::
make_shared
<
node_lookup_result
>
();
auto
on_done
=
[
=
](
bool
ok
)
{
...
...
@@ -27,21 +29,26 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
}
else
if
(
is_leaf
or
*
lo
>
*
hi
)
{
// leaf node
auto
to_insert
=
p
.
getPrefix
(
mid
);
cache_
.
insert
(
to_insert
);
if
(
cb
)
{
if
(
vals
->
size
()
==
0
and
max_common_prefix_len
and
mid
>
0
)
{
auto
p_
=
(
p
.
getPrefix
(
mid
)).
getSibling
().
getFullSize
();
*
lo
=
mid
;
*
hi
=
p_
.
size_
;
lookupStep
(
p_
,
lo
,
hi
,
vals
,
cb
,
done_cb
,
max_common_prefix_len
,
all_values
);
lookupStep
(
p_
,
lo
,
hi
,
vals
,
cb
,
done_cb
,
max_common_prefix_len
,
-
1
,
all_values
);
}
cb
(
*
vals
,
p
.
getPrefix
(
mid
));
cb
(
*
vals
,
to_insert
);
}
if
(
done_cb
)
done_cb
(
true
);
}
else
if
(
first_res
->
is_pht
)
{
// internal node
*
lo
=
mid
+
1
;
lookupStep
(
p
,
lo
,
hi
,
vals
,
cb
,
done_cb
,
max_common_prefix_len
,
all_values
);
lookupStep
(
p
,
lo
,
hi
,
vals
,
cb
,
done_cb
,
max_common_prefix_len
,
-
1
,
all_values
);
}
else
{
// first get failed before second.
if
(
done_cb
)
...
...
@@ -83,6 +90,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
}
return
true
;
};
dht_
->
get
(
p
.
getPrefix
(
mid
).
hash
(),
std
::
bind
(
on_get
,
std
::
placeholders
::
_1
,
first_res
),
[
=
](
bool
ok
)
{
...
...
@@ -96,7 +104,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
if
(
not
first_res
->
is_pht
)
{
// Not a PHT node.
*
hi
=
mid
-
1
;
lookupStep
(
p
,
lo
,
hi
,
vals
,
cb
,
done_cb
,
max_common_prefix_len
,
all_values
);
lookupStep
(
p
,
lo
,
hi
,
vals
,
cb
,
done_cb
,
max_common_prefix_len
,
-
1
,
all_values
);
}
else
{
first_res
->
done
=
true
;
if
(
second_res
->
done
)
...
...
@@ -104,6 +112,7 @@ void Pht::lookupStep(Prefix p, std::shared_ptr<int> lo, std::shared_ptr<int> hi,
}
}
},
pht_filter
);
if
(
mid
<
p
.
size_
)
dht_
->
get
(
p
.
getPrefix
(
mid
+
1
).
hash
(),
std
::
bind
(
on_get
,
std
::
placeholders
::
_1
,
second_res
),
...
...
@@ -132,7 +141,8 @@ void Pht::lookup(Key k, Pht::LookupCallback cb, Dht::DoneCallbackSimple done_cb,
auto
lo
=
std
::
make_shared
<
int
>
(
0
);
auto
hi
=
std
::
make_shared
<
int
>
(
prefix
.
size_
);
std
::
shared_ptr
<
unsigned
>
max_common_prefix_len
=
not
exact_match
?
std
::
make_shared
<
unsigned
>
(
0
)
:
nullptr
;
lookupStep
(
prefix
,
lo
,
hi
,
values
,
cb
,
done_cb
,
max_common_prefix_len
);
lookupStep
(
prefix
,
lo
,
hi
,
values
,
cb
,
done_cb
,
max_common_prefix_len
,
cache_
.
lookup
(
prefix
));
}
void
Pht
::
updateCanary
(
Prefix
p
)
{
...
...
@@ -140,7 +150,7 @@ void Pht::updateCanary(Prefix p) {
dht
::
Value
canary_value
;
canary_value
.
user_type
=
canary_
;
dht_
->
put
(
p
.
hash
(),
std
::
move
(
canary_value
),
[
=
](
bool
ok
){
[
=
](
bool
){
static
std
::
bernoulli_distribution
d
(
0.5
);
crypto
::
random_device
rd
;
if
(
p
.
size_
&&
d
(
rd
))
...
...
@@ -164,7 +174,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) {
auto
final_prefix
=
std
::
make_shared
<
Prefix
>
();
lookupStep
(
kp
,
lo
,
hi
,
vals
,
[
=
](
std
::
vector
<
std
::
shared_ptr
<
Value
>>&
values
,
Prefix
p
)
{
[
=
](
std
::
vector
<
std
::
shared_ptr
<
Value
>>&
,
Prefix
p
)
{
*
final_prefix
=
Prefix
(
p
);
},
[
=
](
bool
ok
){
...
...
@@ -183,7 +193,7 @@ void Pht::insert(Key k, Value v, Dht::DoneCallbackSimple done_cb) {
updateCanary
(
*
final_prefix
);
dht_
->
put
(
final_prefix
->
hash
(),
std
::
move
(
entry
),
done_cb
);
}
},
nullptr
,
true
},
nullptr
,
cache_
.
lookup
(
kp
),
true
);
}
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment