/src/llama.cpp/common/trie.cpp
Line | Count | Source |
1 | | #include "trie.h" |
2 | | |
3 | | #include "unicode.h" |
4 | | |
5 | | #include <deque> |
6 | | |
7 | 0 | common_trie::match_result common_trie::check_at(std::string_view sv, size_t start_pos) const { |
8 | 0 | size_t current = 0; // Start at root |
9 | 0 | size_t pos = start_pos; |
10 | | |
11 | | // LOG_DBG("%s: checking at pos %zu, sv='%s'\n", __func__, start_pos, std::string(sv).c_str()); |
12 | |
|
13 | 0 | while (pos < sv.size()) { |
14 | 0 | auto result = common_parse_utf8_codepoint(sv, pos); |
15 | 0 | if (result.status != utf8_parse_result::SUCCESS) { |
16 | 0 | break; |
17 | 0 | } |
18 | | |
19 | 0 | auto it = nodes[current].children.find(result.codepoint); |
20 | 0 | if (it == nodes[current].children.end()) { |
21 | | // Can't continue matching |
22 | 0 | return match_result{match_result::NO_MATCH}; |
23 | 0 | } |
24 | | |
25 | 0 | current = it->second; |
26 | 0 | pos += result.bytes_consumed; |
27 | | |
28 | | // Check if we've matched a complete word |
29 | 0 | if (nodes[current].pattern >= 0) { |
30 | 0 | return match_result{match_result::COMPLETE_MATCH}; |
31 | 0 | } |
32 | 0 | } |
33 | | |
34 | | // Reached end of input while still in the trie (not at root) |
35 | 0 | if (current != 0) { |
36 | | // We're in the middle of a potential match |
37 | 0 | return match_result{match_result::PARTIAL_MATCH}; |
38 | 0 | } |
39 | | |
40 | | // Reached end at root (no match) |
41 | 0 | return match_result{match_result::NO_MATCH}; |
42 | 0 | } |
43 | | |
44 | 0 | int32_t common_trie::insert(const std::string & word) { |
45 | 0 | std::vector<uint32_t> symbols; |
46 | 0 | size_t pos = 0; |
47 | 0 | while (pos < word.length()) { |
48 | 0 | auto result = common_parse_utf8_codepoint(word, pos); |
49 | 0 | if (result.status != utf8_parse_result::SUCCESS) { |
50 | 0 | break; |
51 | 0 | } |
52 | | |
53 | 0 | symbols.push_back(result.codepoint); |
54 | 0 | pos += result.bytes_consumed; |
55 | 0 | } |
56 | 0 | return insert(symbols); |
57 | 0 | } |
58 | | |
59 | 0 | int32_t common_trie::insert(const std::vector<uint32_t> & symbols) { |
60 | 0 | size_t current = 0; |
61 | 0 | for (uint32_t ch : symbols) { |
62 | 0 | auto it = nodes[current].children.find(ch); |
63 | 0 | if (it == nodes[current].children.end()) { |
64 | 0 | size_t child = create_node(); |
65 | 0 | nodes[current].children[ch] = child; |
66 | 0 | current = child; |
67 | 0 | } else { |
68 | 0 | current = it->second; |
69 | 0 | } |
70 | 0 | } |
71 | 0 | if (nodes[current].pattern < 0) { |
72 | 0 | nodes[current].pattern = n_patterns++; |
73 | 0 | } |
74 | 0 | return nodes[current].pattern; |
75 | 0 | } |
76 | | |
77 | 0 | common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie)) { |
78 | 0 | const auto & nodes = t.nodes; |
79 | 0 | const size_t n = nodes.size(); |
80 | |
|
81 | 0 | fail.assign(n, 0); |
82 | 0 | order.reserve(n); |
83 | |
|
84 | 0 | std::deque<size_t> queue{ 0 }; |
85 | 0 | while (!queue.empty()) { |
86 | 0 | size_t u = queue.front(); |
87 | 0 | queue.pop_front(); |
88 | 0 | order.push_back(u); |
89 | 0 | for (const auto & [ch, v] : nodes[u].children) { |
90 | 0 | if (u != 0) { |
91 | 0 | size_t f = fail[u]; |
92 | 0 | while (f && nodes[f].children.find(ch) == nodes[f].children.end()) { |
93 | 0 | f = fail[f]; |
94 | 0 | } |
95 | 0 | auto it = nodes[f].children.find(ch); |
96 | 0 | fail[v] = (it != nodes[f].children.end() && it->second != v) ? it->second : 0; |
97 | 0 | } |
98 | 0 | queue.push_back(v); |
99 | 0 | } |
100 | 0 | } |
101 | | |
102 | | // fail[u] points to a strictly shorter suffix, so the first pattern found on |
103 | | // the fail chain (including u itself) is the longest pattern ending at u |
104 | 0 | match.assign(n, -1); |
105 | 0 | for (size_t u : order) { |
106 | 0 | match[u] = nodes[u].pattern >= 0 ? nodes[u].pattern : (u != 0 ? match[fail[u]] : -1); |
107 | 0 | } |
108 | |
|
109 | 0 | for (const auto & node : nodes) { |
110 | 0 | for (const auto & [ch, v] : node.children) { |
111 | 0 | alphabet.insert(ch); |
112 | 0 | } |
113 | 0 | } |
114 | 0 | } |
115 | | |
116 | 0 | size_t common_aho_corasick::next(size_t state, uint32_t ch) const { |
117 | 0 | const auto & nodes = t.nodes; |
118 | 0 | while (state && nodes[state].children.find(ch) == nodes[state].children.end()) { |
119 | 0 | state = fail[state]; |
120 | 0 | } |
121 | 0 | auto it = nodes[state].children.find(ch); |
122 | 0 | return it != nodes[state].children.end() ? it->second : 0; |
123 | 0 | } |