You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
390 lines
11 KiB
390 lines
11 KiB
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <ctime>
|
|
#include <sstream>
|
|
|
|
#include <marisa.h>
|
|
|
|
#include "marisa-assert.h"
|
|
|
|
namespace {
|
|
|
|
void TestEmptyTrie() {
|
|
TEST_START();
|
|
|
|
marisa::Trie trie;
|
|
|
|
EXCEPT(trie.save("marisa-test.dat"), MARISA_STATE_ERROR);
|
|
#ifdef _MSC_VER
|
|
EXCEPT(trie.write(::_fileno(stdout)), MARISA_STATE_ERROR);
|
|
#else // _MSC_VER
|
|
EXCEPT(trie.write(::fileno(stdout)), MARISA_STATE_ERROR);
|
|
#endif // _MSC_VER
|
|
EXCEPT(std::cout << trie, MARISA_STATE_ERROR);
|
|
EXCEPT(marisa::fwrite(stdout, trie), MARISA_STATE_ERROR);
|
|
|
|
marisa::Agent agent;
|
|
|
|
EXCEPT(trie.lookup(agent), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.reverse_lookup(agent), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.common_prefix_search(agent), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.predictive_search(agent), MARISA_STATE_ERROR);
|
|
|
|
EXCEPT(trie.num_tries(), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.num_keys(), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.num_nodes(), MARISA_STATE_ERROR);
|
|
|
|
EXCEPT(trie.tail_mode(), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.node_order(), MARISA_STATE_ERROR);
|
|
|
|
EXCEPT(trie.empty(), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.size(), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.total_size(), MARISA_STATE_ERROR);
|
|
EXCEPT(trie.io_size(), MARISA_STATE_ERROR);
|
|
|
|
marisa::Keyset keyset;
|
|
trie.build(keyset);
|
|
|
|
ASSERT(!trie.lookup(agent));
|
|
EXCEPT(trie.reverse_lookup(agent), MARISA_BOUND_ERROR);
|
|
ASSERT(!trie.common_prefix_search(agent));
|
|
ASSERT(!trie.predictive_search(agent));
|
|
|
|
ASSERT(trie.num_tries() == 1);
|
|
ASSERT(trie.num_keys() == 0);
|
|
ASSERT(trie.num_nodes() == 1);
|
|
|
|
ASSERT(trie.tail_mode() == MARISA_DEFAULT_TAIL);
|
|
ASSERT(trie.node_order() == MARISA_DEFAULT_ORDER);
|
|
|
|
ASSERT(trie.empty());
|
|
ASSERT(trie.size() == 0);
|
|
ASSERT(trie.total_size() != 0);
|
|
ASSERT(trie.io_size() != 0);
|
|
|
|
keyset.push_back("");
|
|
trie.build(keyset);
|
|
|
|
ASSERT(trie.lookup(agent));
|
|
trie.reverse_lookup(agent);
|
|
ASSERT(trie.common_prefix_search(agent));
|
|
ASSERT(!trie.common_prefix_search(agent));
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(!trie.predictive_search(agent));
|
|
|
|
ASSERT(trie.num_keys() == 1);
|
|
ASSERT(trie.num_nodes() == 1);
|
|
|
|
ASSERT(!trie.empty());
|
|
ASSERT(trie.size() == 1);
|
|
ASSERT(trie.total_size() != 0);
|
|
ASSERT(trie.io_size() != 0);
|
|
|
|
TEST_END();
|
|
}
|
|
|
|
void TestTinyTrie() {
|
|
TEST_START();
|
|
|
|
marisa::Keyset keyset;
|
|
keyset.push_back("bach");
|
|
keyset.push_back("bet");
|
|
keyset.push_back("chat");
|
|
keyset.push_back("check");
|
|
keyset.push_back("check");
|
|
|
|
marisa::Trie trie;
|
|
trie.build(keyset, 1);
|
|
|
|
ASSERT(trie.num_tries() == 1);
|
|
ASSERT(trie.num_keys() == 4);
|
|
ASSERT(trie.num_nodes() == 7);
|
|
|
|
ASSERT(trie.tail_mode() == MARISA_DEFAULT_TAIL);
|
|
ASSERT(trie.node_order() == MARISA_DEFAULT_ORDER);
|
|
|
|
ASSERT(keyset[0].id() == 2);
|
|
ASSERT(keyset[1].id() == 3);
|
|
ASSERT(keyset[2].id() == 1);
|
|
ASSERT(keyset[3].id() == 0);
|
|
ASSERT(keyset[4].id() == 0);
|
|
|
|
marisa::Agent agent;
|
|
for (std::size_t i = 0; i < keyset.size(); ++i) {
|
|
agent.set_query(keyset[i].ptr(), keyset[i].length());
|
|
ASSERT(trie.lookup(agent));
|
|
ASSERT(agent.key().id() == keyset[i].id());
|
|
|
|
agent.set_query(keyset[i].id());
|
|
trie.reverse_lookup(agent);
|
|
ASSERT(agent.key().length() == keyset[i].length());
|
|
ASSERT(std::memcmp(agent.key().ptr(), keyset[i].ptr(),
|
|
agent.key().length()) == 0);
|
|
}
|
|
|
|
agent.set_query("be");
|
|
ASSERT(!trie.common_prefix_search(agent));
|
|
agent.set_query("beX");
|
|
ASSERT(!trie.common_prefix_search(agent));
|
|
agent.set_query("bet");
|
|
ASSERT(trie.common_prefix_search(agent));
|
|
ASSERT(!trie.common_prefix_search(agent));
|
|
agent.set_query("betX");
|
|
ASSERT(trie.common_prefix_search(agent));
|
|
ASSERT(!trie.common_prefix_search(agent));
|
|
|
|
agent.set_query("chatX");
|
|
ASSERT(!trie.predictive_search(agent));
|
|
agent.set_query("chat");
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().length() == 4);
|
|
ASSERT(!trie.predictive_search(agent));
|
|
|
|
agent.set_query("cha");
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().length() == 4);
|
|
ASSERT(!trie.predictive_search(agent));
|
|
|
|
agent.set_query("c");
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().length() == 5);
|
|
ASSERT(std::memcmp(agent.key().ptr(), "check", 5) == 0);
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().length() == 4);
|
|
ASSERT(std::memcmp(agent.key().ptr(), "chat", 4) == 0);
|
|
ASSERT(!trie.predictive_search(agent));
|
|
|
|
agent.set_query("ch");
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().length() == 5);
|
|
ASSERT(std::memcmp(agent.key().ptr(), "check", 5) == 0);
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().length() == 4);
|
|
ASSERT(std::memcmp(agent.key().ptr(), "chat", 4) == 0);
|
|
ASSERT(!trie.predictive_search(agent));
|
|
|
|
trie.build(keyset, 1 | MARISA_LABEL_ORDER);
|
|
|
|
ASSERT(trie.num_tries() == 1);
|
|
ASSERT(trie.num_keys() == 4);
|
|
ASSERT(trie.num_nodes() == 7);
|
|
|
|
ASSERT(trie.tail_mode() == MARISA_DEFAULT_TAIL);
|
|
ASSERT(trie.node_order() == MARISA_LABEL_ORDER);
|
|
|
|
ASSERT(keyset[0].id() == 0);
|
|
ASSERT(keyset[1].id() == 1);
|
|
ASSERT(keyset[2].id() == 2);
|
|
ASSERT(keyset[3].id() == 3);
|
|
ASSERT(keyset[4].id() == 3);
|
|
|
|
for (std::size_t i = 0; i < keyset.size(); ++i) {
|
|
agent.set_query(keyset[i].ptr(), keyset[i].length());
|
|
ASSERT(trie.lookup(agent));
|
|
ASSERT(agent.key().id() == keyset[i].id());
|
|
|
|
agent.set_query(keyset[i].id());
|
|
trie.reverse_lookup(agent);
|
|
ASSERT(agent.key().length() == keyset[i].length());
|
|
ASSERT(std::memcmp(agent.key().ptr(), keyset[i].ptr(),
|
|
agent.key().length()) == 0);
|
|
}
|
|
|
|
agent.set_query("");
|
|
for (std::size_t i = 0; i < trie.size(); ++i) {
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().id() == i);
|
|
}
|
|
ASSERT(!trie.predictive_search(agent));
|
|
|
|
TEST_END();
|
|
}
|
|
|
|
void MakeKeyset(std::size_t num_keys, marisa::TailMode tail_mode,
|
|
marisa::Keyset *keyset) {
|
|
char key_buf[16];
|
|
for (std::size_t i = 0; i < num_keys; ++i) {
|
|
const std::size_t length =
|
|
static_cast<std::size_t>(std::rand()) % sizeof(key_buf);
|
|
for (std::size_t j = 0; j < length; ++j) {
|
|
key_buf[j] = (char)(std::rand() % 10);
|
|
if (tail_mode == MARISA_TEXT_TAIL) {
|
|
key_buf[j] = static_cast<char>(key_buf[j] + '0');
|
|
}
|
|
}
|
|
keyset->push_back(key_buf, length);
|
|
}
|
|
}
|
|
|
|
void TestLookup(const marisa::Trie &trie, const marisa::Keyset &keyset) {
|
|
marisa::Agent agent;
|
|
for (std::size_t i = 0; i < keyset.size(); ++i) {
|
|
agent.set_query(keyset[i].ptr(), keyset[i].length());
|
|
ASSERT(trie.lookup(agent));
|
|
ASSERT(agent.key().id() == keyset[i].id());
|
|
|
|
agent.set_query(keyset[i].id());
|
|
trie.reverse_lookup(agent);
|
|
ASSERT(agent.key().length() == keyset[i].length());
|
|
ASSERT(std::memcmp(agent.key().ptr(), keyset[i].ptr(),
|
|
agent.key().length()) == 0);
|
|
}
|
|
}
|
|
|
|
void TestCommonPrefixSearch(const marisa::Trie &trie,
|
|
const marisa::Keyset &keyset) {
|
|
marisa::Agent agent;
|
|
for (std::size_t i = 0; i < keyset.size(); ++i) {
|
|
agent.set_query(keyset[i].ptr(), keyset[i].length());
|
|
ASSERT(trie.common_prefix_search(agent));
|
|
ASSERT(agent.key().id() <= keyset[i].id());
|
|
while (trie.common_prefix_search(agent)) {
|
|
ASSERT(agent.key().id() <= keyset[i].id());
|
|
}
|
|
ASSERT(agent.key().id() == keyset[i].id());
|
|
}
|
|
}
|
|
|
|
void TestPredictiveSearch(const marisa::Trie &trie,
|
|
const marisa::Keyset &keyset) {
|
|
marisa::Agent agent;
|
|
for (std::size_t i = 0; i < keyset.size(); ++i) {
|
|
agent.set_query(keyset[i].ptr(), keyset[i].length());
|
|
ASSERT(trie.predictive_search(agent));
|
|
ASSERT(agent.key().id() == keyset[i].id());
|
|
while (trie.predictive_search(agent)) {
|
|
ASSERT(agent.key().id() > keyset[i].id());
|
|
}
|
|
}
|
|
}
|
|
|
|
void TestTrie(int num_tries, marisa::TailMode tail_mode,
|
|
marisa::NodeOrder node_order, marisa::Keyset &keyset) {
|
|
for (std::size_t i = 0; i < keyset.size(); ++i) {
|
|
keyset[i].set_weight(1.0F);
|
|
}
|
|
|
|
marisa::Trie trie;
|
|
trie.build(keyset, num_tries | tail_mode | node_order);
|
|
|
|
ASSERT(trie.num_tries() == (std::size_t)num_tries);
|
|
ASSERT(trie.num_keys() <= keyset.size());
|
|
|
|
ASSERT(trie.tail_mode() == tail_mode);
|
|
ASSERT(trie.node_order() == node_order);
|
|
|
|
TestLookup(trie, keyset);
|
|
TestCommonPrefixSearch(trie, keyset);
|
|
TestPredictiveSearch(trie, keyset);
|
|
|
|
trie.save("marisa-test.dat");
|
|
|
|
trie.clear();
|
|
trie.load("marisa-test.dat");
|
|
|
|
ASSERT(trie.num_tries() == (std::size_t)num_tries);
|
|
ASSERT(trie.num_keys() <= keyset.size());
|
|
|
|
ASSERT(trie.tail_mode() == tail_mode);
|
|
ASSERT(trie.node_order() == node_order);
|
|
|
|
TestLookup(trie, keyset);
|
|
|
|
{
|
|
std::FILE *file;
|
|
#ifdef _MSC_VER
|
|
ASSERT(::fopen_s(&file, "marisa-test.dat", "wb") == 0);
|
|
#else // _MSC_VER
|
|
file = std::fopen("marisa-test.dat", "wb");
|
|
ASSERT(file != NULL);
|
|
#endif // _MSC_VER
|
|
marisa::fwrite(file, trie);
|
|
std::fclose(file);
|
|
trie.clear();
|
|
#ifdef _MSC_VER
|
|
ASSERT(::fopen_s(&file, "marisa-test.dat", "rb") == 0);
|
|
#else // _MSC_VER
|
|
file = std::fopen("marisa-test.dat", "rb");
|
|
ASSERT(file != NULL);
|
|
#endif // _MSC_VER
|
|
marisa::fread(file, &trie);
|
|
std::fclose(file);
|
|
}
|
|
|
|
ASSERT(trie.num_tries() == (std::size_t)num_tries);
|
|
ASSERT(trie.num_keys() <= keyset.size());
|
|
|
|
ASSERT(trie.tail_mode() == tail_mode);
|
|
ASSERT(trie.node_order() == node_order);
|
|
|
|
TestLookup(trie, keyset);
|
|
|
|
trie.clear();
|
|
trie.mmap("marisa-test.dat");
|
|
|
|
ASSERT(trie.num_tries() == (std::size_t)num_tries);
|
|
ASSERT(trie.num_keys() <= keyset.size());
|
|
|
|
ASSERT(trie.tail_mode() == tail_mode);
|
|
ASSERT(trie.node_order() == node_order);
|
|
|
|
TestLookup(trie, keyset);
|
|
|
|
{
|
|
std::stringstream stream;
|
|
stream << trie;
|
|
trie.clear();
|
|
stream >> trie;
|
|
}
|
|
|
|
ASSERT(trie.num_tries() == (std::size_t)num_tries);
|
|
ASSERT(trie.num_keys() <= keyset.size());
|
|
|
|
ASSERT(trie.tail_mode() == tail_mode);
|
|
ASSERT(trie.node_order() == node_order);
|
|
|
|
TestLookup(trie, keyset);
|
|
}
|
|
|
|
void TestTrie(marisa::TailMode tail_mode, marisa::NodeOrder node_order,
|
|
marisa::Keyset &keyset) {
|
|
TEST_START();
|
|
std::cout << ((tail_mode == MARISA_TEXT_TAIL) ? "TEXT" : "BINARY") << ", ";
|
|
std::cout << ((node_order == MARISA_WEIGHT_ORDER) ?
|
|
"WEIGHT" : "LABEL") << ": ";
|
|
|
|
for (int i = 1; i < 5; ++i) {
|
|
TestTrie(i, tail_mode, node_order, keyset);
|
|
}
|
|
|
|
TEST_END();
|
|
}
|
|
|
|
void TestTrie(marisa::TailMode tail_mode) {
|
|
marisa::Keyset keyset;
|
|
MakeKeyset(1000, tail_mode, &keyset);
|
|
|
|
TestTrie(tail_mode, MARISA_WEIGHT_ORDER, keyset);
|
|
TestTrie(tail_mode, MARISA_LABEL_ORDER, keyset);
|
|
}
|
|
|
|
void TestTrie() {
|
|
TestTrie(MARISA_TEXT_TAIL);
|
|
TestTrie(MARISA_BINARY_TAIL);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() try {
|
|
std::srand((unsigned int)std::time(NULL));
|
|
|
|
TestEmptyTrie();
|
|
TestTinyTrie();
|
|
TestTrie();
|
|
|
|
return 0;
|
|
} catch (const marisa::Exception &ex) {
|
|
std::cerr << ex.what() << std::endl;
|
|
throw;
|
|
}
|