diff --git a/include/opendht/default_types.h b/include/opendht/default_types.h index ea0f6a155c641218f0167612beab7dd5b3242ff6..67578f77e97c1a3f08dd92a51911a16bfde69794 100644 --- a/include/opendht/default_types.h +++ b/include/opendht/default_types.h @@ -21,6 +21,8 @@ #include "value.h" +MSGPACK_ADD_ENUM(dht::Value::Field); + namespace dht { enum class ImStatus : uint8_t { NONE = 0, @@ -175,7 +177,7 @@ public: pk.pack_bin_body((const char*)ice_data.data(), ice_data.size()); #else // hack for backward compatibility with old opendht compiled with msgpack 1.0 - // remove when enough people have moved to new versions + // remove when enough people have moved to new versions pk.pack_array(ice_data.size()); for (uint8_t b : ice_data) pk.pack(b); diff --git a/include/opendht/utils.h b/include/opendht/utils.h index 0b892fd1539e2895310573f119b61406ca4fb00a..9dcb1660be57ba763beedfa52b879935ec8267d6 100644 --- a/include/opendht/utils.h +++ b/include/opendht/utils.h @@ -168,4 +168,6 @@ unpackMsg(Blob b) { msgpack::unpacked unpackMsg(Blob b); +msgpack::object* findMapValue(msgpack::object& map, const std::string& key); + } // namespace dht diff --git a/include/opendht/value.h b/include/opendht/value.h index 58351e70ced099808cdd4b55ba5906fe11e4ebf7..5c7c2ce90ed5beceb61840d84b402a7bc70aead9 100644 --- a/include/opendht/value.h +++ b/include/opendht/value.h @@ -35,10 +35,12 @@ #include <functional> #include <memory> #include <chrono> +#include <set> namespace dht { struct Value; +struct Query; /** * A storage policy is applied once to every incoming value storage requests. @@ -114,6 +116,14 @@ struct ValueType { */ struct Value { + enum class Field { + None = 0, + Id, + ValueType, + OwnerPk, + UserType, + }; + typedef uint64_t Id; static const Id INVALID_ID {0}; @@ -135,14 +145,18 @@ struct Value return f1(v) and f2(v); }; } - static Filter chain(std::initializer_list<Filter> l) { - const std::vector<Filter> list(l.begin(), l.end()); - return [list](const Value& v){ - for (const auto& f : list) + template <typename T> + static Filter chainAll(T&& set) { + using namespace std::placeholders; + return std::bind([](const Value& v, T& s) { + for (const auto& f : s) if (f and not f(v)) return false; return true; - }; + }, _1, std::move(set)); + } + static Filter chain(std::initializer_list<Filter> l) { + return chainAll(std::move(l)); } static Filter chainOr(Filter&& f1, Filter&& f2) { if (not f1 or not f2) return AllFilter(); @@ -162,6 +176,11 @@ struct Value return v.type == tid; }; } + static Filter TypeFilter(const ValueType::Id& tid) { + return [tid](const Value& v) { + return v.type == tid; + }; + } static Filter IdFilter(const Id id) { return [id](const Value& v) { @@ -175,6 +194,23 @@ struct Value }; } + static Filter ownerFilter(const crypto::PublicKey& pk) { + return ownerFilter(pk.getId()); + } + + static Filter ownerFilter(const InfoHash& pkh) { + return [pkh](const Value& v) { + return v.owner and v.owner->getId() == pkh; + }; + } + + static Filter userTypeFilter(const std::string& ut) + { + return [ut](const Value& v) { + return v.user_type == ut; + }; + } + class SerializableBase { public: @@ -415,6 +451,31 @@ struct Value pk.pack(std::string("dat")); msgpack_pack_to_encrypt(pk); } + template <typename Packer> + void msgpack_pack_fields(const std::set<Value::Field>& fields, Packer& pk) const + { + for (const auto& field : fields) + switch (field) { + case Value::Field::Id: + pk.pack(id); + break; + case Value::Field::ValueType: + pk.pack(type); + break; + case Value::Field::OwnerPk: + if (owner) + owner->msgpack_pack(pk); + else + InfoHash().msgpack_pack(pk); + break; + case Value::Field::UserType: + pk.pack(user_type); + break; + default: + break; + } + } + void msgpack_unpack(msgpack::object o); void msgpack_unpack_body(const msgpack::object& o); Blob getPacked() const { @@ -424,6 +485,8 @@ struct Value return {buffer.data(), buffer.data()+buffer.size()}; } + void msgpack_unpack_fields(const std::set<Value::Field>& fields, const msgpack::object& o, unsigned offset); + Id id {INVALID_ID}; /** @@ -467,6 +530,338 @@ struct Value using ValuesExport = std::pair<InfoHash, Blob>; +/** + * @class FieldValue + * @brief Describes a value filter. + * @details + * This structure holds the value for a specified field. It's type can either be + * uint64_t, InfoHash or Blob. + */ +struct FieldValue +{ + FieldValue() {} + FieldValue(Value::Field f, uint64_t int_value) : field(f), intValue(int_value) {} + FieldValue(Value::Field f, InfoHash hash_value) : field(f), hashValue(hash_value) {} + FieldValue(Value::Field f, Blob blob_value) : field(f), blobValue(blob_value) {} + + bool operator==(const FieldValue& fd) const; + + // accessors + Value::Field getField() const { return field; } + uint64_t getInt() const { return intValue; } + InfoHash getHash() const { return hashValue; } + Blob getBlob() const { return blobValue; } + + template <typename Packer> + void msgpack_pack(Packer& p) const { + p.pack_map(2); + p.pack(std::string("f")); p.pack(static_cast<uint8_t>(field)); + + p.pack(std::string("v")); + switch (field) { + case Value::Field::Id: + case Value::Field::ValueType: + p.pack(intValue); + break; + case Value::Field::OwnerPk: + p.pack(hashValue); + break; + case Value::Field::UserType: + p.pack_bin(blobValue.size()); + p.pack_bin_body((const char*)blobValue.data(), blobValue.size()); + break; + default: + throw msgpack::type_error(); + } + } + + void msgpack_unpack(msgpack::object msg) { + hashValue = {}; + blobValue.clear(); + + if (auto f = findMapValue(msg, "f")) + field = (Value::Field)f->as<unsigned>(); + else + throw msgpack::type_error(); + + auto v = findMapValue(msg, "v"); + if (not v) + throw msgpack::type_error(); + else + switch (field) { + case Value::Field::Id: + case Value::Field::ValueType: + intValue = v->as<decltype(intValue)>(); + break; + case Value::Field::OwnerPk: + hashValue = v->as<decltype(hashValue)>(); + break; + case Value::Field::UserType: + blobValue = unpackBlob(*v); + break; + default: + throw msgpack::type_error(); + } + } + + Value::Filter getLocalFilter() const; + +private: + Value::Field field {Value::Field::None}; + // three possible value types + uint64_t intValue {}; + InfoHash hashValue {}; + Blob blobValue {}; +}; + + +/** + * @struct FieldSelectorDescription + * @brief Describes a selection. + * @details + * This is meant to narrow data to a set of specified fields. This structure is + * used to construct a Select structure. + */ +struct FieldSelectorDescription +{ + FieldSelectorDescription() {} + FieldSelectorDescription(Value::Field f) : field(f) {} + + Value::Field getField() const { return field; } + + bool operator==(const FieldSelectorDescription& fd) const { return field == fd.field; } + + template <typename Packer> + void msgpack_pack(Packer& p) const { p.pack(static_cast<uint8_t>(field)); } + void msgpack_unpack(msgpack::object msg) { field = static_cast<Value::Field>(msg.as<int>()); } +private: + Value::Field field {Value::Field::None}; +}; + +/** + * @class Select + * @brief Serializable Value field selection. + * @details + * This is a container for a list of FieldSelectorDescription instances. It + * describes a complete SELECT query for dht::Value. + */ +struct Select +{ + Select() { } + Select(const std::string& q_str); + + bool isSatisfiedBy(const Select& os) const; + + /** + * Selects a field of type Value::Field. + * + * @param field the field to require. + * + * @return the resulting Select instance. + */ + Select& field(Value::Field field) { + fieldSelection_.emplace_back(field); + return *this; + } + + /** + * Computes the set of selected fields based on previous require* calls. + * + * @return the set of fields. + */ + std::set<Value::Field> getSelection() const { + std::set<Value::Field> fields {}; + for (const auto& f : fieldSelection_) { + fields.insert(f.getField()); + } + return fields; + } + + template <typename Packer> + void msgpack_pack(Packer& pk) const { pk.pack(fieldSelection_); } + void msgpack_unpack(const msgpack::object& o) { + fieldSelection_.clear(); + fieldSelection_ = o.as<decltype(fieldSelection_)>(); + } + + friend std::ostream& operator<<(std::ostream& s, const dht::Select& q); +private: + std::vector<FieldSelectorDescription> fieldSelection_ {}; +}; + +/** + * @class Where + * @brief Serializable dht::Value filter. + * @details + * This is container for a list of FieldValue instances. It describes a + * complete WHERE query for dht::Value. + */ +struct Where +{ + Where() { } + Where(const std::string& q_str); + + bool isSatisfiedBy(const Where& where) const; + + /** + * Adds restriction on Value::Id based on the id argument. + * + * @param id the id. + * + * @return the resulting Where instance. + */ + Where& id(Value::Id id) { + filters_.emplace_back(Value::Field::Id, id); + return *this; + } + + /** + * Adds restriction on Value::ValueType based on the type argument. + * + * @param type the value type. + * + * @return the resulting Where instance. + */ + Where& valueType(ValueType::Id type) { + filters_.emplace_back(Value::Field::ValueType, type); + return *this; + } + + /** + * Adds restriction on Value::OwnerPk based on the owner_pk_hash argument. + * + * @param owner_pk_hash the owner public key fingerprint. + * + * @return the resulting Where instance. + */ + Where& owner(InfoHash owner_pk_hash) { + filters_.emplace_back(Value::Field::OwnerPk, owner_pk_hash); + return *this; + } + + /** + * Adds restriction on Value::UserType based on the user_type argument. + * + * @param user_type the user type. + * + * @return the resulting Where instance. + */ + Where& userType(std::string user_type) { + filters_.emplace_back(Value::Field::UserType, Blob {user_type.begin(), user_type.end()}); + return *this; + } + + /** + * Computes the Value::Filter based on the list of field value set. + * + * @return the resulting Value::Filter. + */ + Value::Filter getFilter() const { + std::vector<Value::Filter> fset(filters_.size()); + std::transform(filters_.begin(), filters_.end(), fset.begin(), [](const FieldValue& f) { + return f.getLocalFilter(); + }); + return Value::Filter::chainAll(std::move(fset)); + } + + template <typename Packer> + void msgpack_pack(Packer& pk) const { pk.pack(filters_); } + void msgpack_unpack(const msgpack::object& o) { + filters_.clear(); + filters_ = o.as<decltype(filters_)>(); + } + + friend std::ostream& operator<<(std::ostream& s, const dht::Where& q); + +private: + std::vector<FieldValue> filters_; +}; + +/** + * @class Query + * @brief Describes a query destined to another peer. + * @details + * This class describes the list of filters on field values and the field + * itselves to include in the peer response to a GET operation. See + * FieldValue. + */ +struct Query +{ + static const std::string QUERY_PARSE_ERROR; + + Query(Select s = {}, Where w = {}) : select(s), where(w) { }; + + /** + * Initializes a query based on a SQL-ish formatted string. The abstract + * form of such a string is the following: + * + * [SELECT <$field$> [WHERE <$field$=$value$>]] + * + * where + * + * - $field$ = *|id|value_type|owner_pk|user_type + * - $value$ = $string$|$integer$ + * - $string$: a simple string WITHOUT SPACES. + * - $integer$: a simple integer. + */ + Query(std::string q_str) { + auto pos_W = q_str.find("WHERE"); + auto pos_w = q_str.find("where"); + auto pos = std::min(pos_W != std::string::npos ? pos_W : q_str.size(), + pos_w != std::string::npos ? pos_w : q_str.size()); + select = q_str.substr(0, pos); + where = q_str.substr(pos, q_str.size()-pos); + } + + /** + * Tell if the query is satisfied by another query. + */ + bool isSatisfiedBy(const Query& q) const; + + template <typename Packer> + void msgpack_pack(Packer& pk) const { + pk.pack_map(2); + pk.pack(std::string("s")); pk.pack(select); /* packing field selectors */ + pk.pack(std::string("w")); pk.pack(where); /* packing filters */ + } + + void msgpack_unpack(const msgpack::object& o); + + friend std::ostream& operator<<(std::ostream& s, const dht::Query& q) { + s << "Query[" << q.select << " " << q.where << "]"; + } + + Select select {}; + Where where {}; +}; + +/*! + * @class FieldValueIndex + * @brief An index for field values. + * @details + * This structures is meant to manipulate a subset of fields normally contained + * in Value. + */ +struct FieldValueIndex { + FieldValueIndex() {} + FieldValueIndex(const Value& v, Select s = {}); + /** + * Tells if all the fields of this are contained in the other + * FieldValueIndex with the same value. + * + * @param other The other FieldValueIndex instance. + */ + bool containedIn(const FieldValueIndex& other) const; + + friend std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi); + + void msgpack_unpack_fields(const std::set<Value::Field>& fields, + const msgpack::object& o, + unsigned offset); + + std::map<Value::Field, FieldValue> index {}; +}; + template <typename T, typename std::enable_if<std::is_base_of<Value::SerializableBase, T>::value, T>::type* = nullptr> Value::Filter diff --git a/src/network_engine.cpp b/src/network_engine.cpp index 15f2431c958d7dfbd6c2a95dc0f27aca89adef5d..159dc5c5cd51b5c940e68f542c7d504aff354201 100644 --- a/src/network_engine.cpp +++ b/src/network_engine.cpp @@ -957,20 +957,6 @@ NetworkEngine::sendError(const sockaddr* sa, send(buffer.data(), buffer.size(), 0, sa, salen); } -msgpack::object* -findMapValue(msgpack::object& map, const std::string& key) { - if (map.type != msgpack::type::MAP) throw msgpack::type_error(); - for (unsigned i = 0; i < map.via.map.size; i++) { - auto& o = map.via.map.ptr[i]; - if(o.key.type != msgpack::type::STR) - continue; - if (o.key.as<std::string>() == key) { - return &o.val; - } - } - return nullptr; -} - void ParsedMessage::msgpack_unpack(msgpack::object msg) { diff --git a/src/utils.cpp b/src/utils.cpp index 33ee40635805cf9f376719ae50ba7666847d6c43..1abf0fe0471048c55662c68ecbe714379d2ff503 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -85,4 +85,15 @@ unpackMsg(Blob b) { return msgpack::unpack((const char*)b.data(), b.size()); } +msgpack::object* +findMapValue(msgpack::object& map, const std::string& key) { + if (map.type != msgpack::type::MAP) throw msgpack::type_error(); + for (unsigned i = 0; i < map.via.map.size; i++) { + auto& o = map.via.map.ptr[i]; + if (o.key.type == msgpack::type::STR && o.key.as<std::string>() == key) + return &o.val; + } + return nullptr; +} + } diff --git a/src/value.cpp b/src/value.cpp index 58a86556d2908ebc9334210c6c17e623f1f2ae2a..d11a7ba3fcafa1e678d0ae943a15337f42c72425 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -25,6 +25,8 @@ namespace dht { +const std::string Query::QUERY_PARSE_ERROR {"Error parsing query."}; + std::ostream& operator<< (std::ostream& s, const Value& v) { s << "Value[id:" << std::hex << v.id << std::dec << " "; @@ -157,4 +159,306 @@ Value::msgpack_unpack_body(const msgpack::object& o) } } +bool +FieldValue::operator==(const FieldValue& vfd) const +{ + if (field != vfd.field) + return false; + switch (field) { + case Value::Field::Id: + case Value::Field::ValueType: + return intValue == vfd.intValue; + case Value::Field::OwnerPk: + return hashValue == vfd.hashValue; + case Value::Field::UserType: + return blobValue == vfd.blobValue; + case Value::Field::None: + return true; + default: + return false; + } +} + +Value::Filter +FieldValue::getLocalFilter() const +{ + switch (field) { + case Value::Field::Id: + return Value::IdFilter(intValue); + case Value::Field::ValueType: + return Value::TypeFilter(intValue); + case Value::Field::OwnerPk: + return Value::ownerFilter(hashValue); + case Value::Field::UserType: + return Value::userTypeFilter(std::string {blobValue.begin(), blobValue.end()}); + default: + return Value::AllFilter(); + } +} + +FieldValueIndex::FieldValueIndex(const Value& v, Select s) +{ + auto selection = s.getSelection(); + if (not selection.empty()) { + std::transform(selection.begin(), selection.end(), std::inserter(index, index.end()), + [](const std::set<Value::Field>::value_type& f) { + return std::make_pair(f, FieldValue {}); + }); + } else { + index.clear(); + for (size_t f = 1 ; f < 5 ; ++f) + index[static_cast<Value::Field>(f)] = {}; + } + for (const auto& fvp : index) { + const auto& f = fvp.first; + switch (f) { + case Value::Field::Id: + index[f] = {f, v.id}; + break; + case Value::Field::ValueType: + index[f] = {f, v.type}; + break; + case Value::Field::OwnerPk: + index[f] = {f, v.owner ? v.owner->getId() : InfoHash() }; + break; + case Value::Field::UserType: + index[f] = {f, Blob {v.user_type.begin(), v.user_type.end()}}; + break; + default: + break; + } + } +} + +bool FieldValueIndex::containedIn(const FieldValueIndex& other) const { + if (index.size() > other.index.size()) + return false; + for (const auto& field : index) { + auto other_field = other.index.find(field.first); + if (other_field == other.index.end()) + return false; + } + return true; +} + +std::ostream& operator<<(std::ostream& os, const FieldValueIndex& fvi) { + os << "Index["; + for (auto v = fvi.index.begin(); v != fvi.index.end(); ++v) { + switch (v->first) { + case Value::Field::Id: + os << "Id:" << std::hex << v->second.getInt(); + break; + case Value::Field::ValueType: + os << "ValueType:" << v->second.getInt(); + break; + case Value::Field::OwnerPk: + os << "Owner:" << v->second.getHash().toString(); + break; + case Value::Field::UserType: { + auto ut = v->second.getBlob(); + os << "UserType:" << std::string(ut.begin(), ut.end()); + break; + } + default: + break; + } + os << (std::next(v) != fvi.index.end() ? "," : ""); + } + return os << "]"; +} + +void +FieldValueIndex::msgpack_unpack_fields(const std::set<Value::Field>& fields, const msgpack::object& o, unsigned offset) +{ + index.clear(); + + unsigned j = 0; + for (const auto& field : fields) { + auto& field_value = o.via.array.ptr[offset+(j++)]; + switch (field) { + case Value::Field::Id: + case Value::Field::ValueType: + index[field] = FieldValue(field, field_value.as<uint64_t>()); + break; + case Value::Field::OwnerPk: + index[field] = FieldValue(field, field_value.as<InfoHash>()); + break; + case Value::Field::UserType: + index[field] = FieldValue(field, field_value.as<Blob>()); + break; + default: + throw msgpack::type_error(); + } + } +} + +void trim_str(std::string& str) { + auto first = std::min(str.size(), str.find_first_not_of(" ")); + auto last = std::min(str.size(), str.find_last_not_of(" ")); + str = str.substr(first, last - first + 1); } + +Select::Select(const std::string& q_str) { + std::istringstream q_iss {q_str}; + std::string token {}; + q_iss >> token; + + if (token == "SELECT" or token == "select") { + q_iss >> token; + std::istringstream fields {token}; + + while (std::getline(fields, token, ',')) { + trim_str(token); + if (token == "id") + field(Value::Field::Id); + else if (token == "value_type") + field(Value::Field::ValueType); + else if (token == "owner_pk") + field(Value::Field::OwnerPk); + else if (token == "user_type") + field(Value::Field::UserType); + } + } +} + +Where::Where(const std::string& q_str) { + std::istringstream q_iss {q_str}; + std::string token {}; + q_iss >> token; + if (token == "WHERE" or token == "where") { + std::getline(q_iss, token); + std::istringstream restrictions {token}; + while (std::getline(restrictions, token, ',')) { + trim_str(token); + std::istringstream eq_ss {token}; + std::string field_str, value_str; + std::getline(eq_ss, field_str, '='); + trim_str(field_str); + std::getline(eq_ss, value_str, '='); + trim_str(value_str); + + if (not value_str.empty()) { + uint64_t v = 0; + std::string s {}; + std::istringstream convert {value_str}; + convert >> v; + if (convert.failbit and value_str.size() > 1 and value_str[0] == '\"' and value_str[value_str.size()-1] == '\"') + s = value_str.substr(1, value_str.size()-2); + else + s = value_str; + if (field_str == "id") + id(v); + else if (field_str == "value_type") + valueType(v); + else if (field_str == "owner_pk") + owner(InfoHash(s)); + else if (field_str == "user_type") + userType(s); + else + throw std::invalid_argument(Query::QUERY_PARSE_ERROR + " (WHERE) wrong token near: " + field_str); + } + } + } +} + +void +Query::msgpack_unpack(const msgpack::object& o) +{ + if (o.type != msgpack::type::MAP) + throw msgpack::type_error(); + + auto rfilters = findMapValue(o, "w"); /* unpacking filters */ + if (rfilters) + where.msgpack_unpack(*rfilters); + else + throw msgpack::type_error(); + + auto rfield_selector = findMapValue(o, "s"); /* unpacking field selectors */ + if (rfield_selector) + select.msgpack_unpack(*rfield_selector); + else + throw msgpack::type_error(); +} + +template <typename T> +bool subset(std::vector<T> fds, std::vector<T> qfds) +{ + for (auto& fd : fds) { + auto correspondance = std::find_if(qfds.begin(), qfds.end(), [&fd](T& _vfd) { return fd == _vfd; }); + if (correspondance == qfds.end()) + return false; + } + return true; +}; + +bool Select::isSatisfiedBy(const Select& os) const { + /* empty, means all values are selected. */ + if (fieldSelection_.empty() and not os.fieldSelection_.empty()) + return false; + else + return subset(fieldSelection_, os.fieldSelection_); +} + +bool Where::isSatisfiedBy(const Where& ow) const { + return subset(ow.filters_, filters_); +} + +bool Query::isSatisfiedBy(const Query& q) const { + return where.isSatisfiedBy(q.where) and select.isSatisfiedBy(q.select); +} + +std::ostream& operator<<(std::ostream& s, const dht::Select& select) { + s << "SELECT " << (select.fieldSelection_.empty() ? "*" : ""); + for (auto fs = select.fieldSelection_.begin() ; fs != select.fieldSelection_.end() ; ++fs) { + switch (fs->getField()) { + case Value::Field::Id: + s << "id"; + break; + case Value::Field::ValueType: + s << "value_type"; + break; + case Value::Field::UserType: + s << "user_type"; + break; + case Value::Field::OwnerPk: + s << "owner_public_key"; + break; + default: + break; + } + s << (std::next(fs) != select.fieldSelection_.end() ? "," : ""); + } + return s; +} + +std::ostream& operator<<(std::ostream& s, const dht::Where& where) { + if (not where.filters_.empty()) { + s << "WHERE "; + for (auto f = where.filters_.begin() ; f != where.filters_.end() ; ++f) { + switch (f->getField()) { + case Value::Field::Id: + s << "id=" << f->getInt(); + break; + case Value::Field::ValueType: + s << "value_type=" << f->getInt(); + break; + case Value::Field::OwnerPk: + s << "owner_pk_hash=" << f->getHash().toString(); + break; + case Value::Field::UserType: { + auto b = f->getBlob(); + s << "user_type=" << std::string {b.begin(), b.end()}; + break; + } + default: + break; + } + s << (std::next(f) != where.filters_.end() ? "," : ""); + } + } + return s; +} + + +} +