/src/parallel-hashmap/parallel_hashmap/phmap_dump.h
Line  | Count  | Source  | 
1  |  | #if !defined(phmap_dump_h_guard_)  | 
2  |  | #define phmap_dump_h_guard_  | 
3  |  |  | 
4  |  | // ---------------------------------------------------------------------------  | 
5  |  | // Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com  | 
6  |  | //  | 
7  |  | //       providing dump/load/mmap_load  | 
8  |  | //  | 
9  |  | // Licensed under the Apache License, Version 2.0 (the "License");  | 
10  |  | // you may not use this file except in compliance with the License.  | 
11  |  | // You may obtain a copy of the License at  | 
12  |  | //  | 
13  |  | //      https://www.apache.org/licenses/LICENSE-2.0  | 
14  |  | //  | 
15  |  | // Unless required by applicable law or agreed to in writing, software  | 
16  |  | // distributed under the License is distributed on an "AS IS" BASIS,  | 
17  |  | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  | 
18  |  | // See the License for the specific language governing permissions and  | 
19  |  | // limitations under the License.  | 
20  |  | // ---------------------------------------------------------------------------  | 
21  |  |  | 
22  |  | #include <iostream>  | 
23  |  | #include <fstream>  | 
24  |  | #include <functional>  | 
25  |  | #include "phmap.h"  | 
26  |  | namespace phmap  | 
27  |  | { | 
28  |  |  | 
29  |  | namespace type_traits_internal { | 
30  |  |  | 
31  |  | #if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801  | 
32  |  |     template<typename T> struct IsTriviallyCopyable : public std::integral_constant<bool, __has_trivial_copy(T)> {}; | 
33  |  | #else  | 
34  |  |     template<typename T> struct IsTriviallyCopyable : public std::is_trivially_copyable<T> {}; | 
35  |  | #endif  | 
36  |  |  | 
37  |  | template <class T1, class T2>  | 
38  |  | struct IsTriviallyCopyable<std::pair<T1, T2>> { | 
39  |  |     static constexpr bool value = IsTriviallyCopyable<T1>::value && IsTriviallyCopyable<T2>::value;  | 
40  |  | };  | 
41  |  | }  | 
42  |  |  | 
43  |  | namespace priv { | 
44  |  |  | 
45  |  | #if !defined(PHMAP_NON_DETERMINISTIC) && !defined(PHMAP_DISABLE_DUMP)  | 
46  |  |  | 
47  |  | static constexpr size_t s_version_base = std::numeric_limits<size_t>::max() - 10;  | 
48  |  | static constexpr size_t s_version = s_version_base;  | 
49  |  | // ------------------------------------------------------------------------  | 
50  |  | // dump/load for raw_hash_set  | 
51  |  | // ------------------------------------------------------------------------  | 
52  |  | template <class Policy, class Hash, class Eq, class Alloc>  | 
53  |  | template<typename OutputArchive>  | 
54  | 595  | bool raw_hash_set<Policy, Hash, Eq, Alloc>::phmap_dump(OutputArchive& ar) const { | 
55  | 595  |     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,  | 
56  | 595  |                     "value_type should be trivially copyable");  | 
57  |  |  | 
58  | 595  |     ar.saveBinary(&s_version, sizeof(size_t));  | 
59  | 595  |     ar.saveBinary(&size_, sizeof(size_t));  | 
60  | 595  |     ar.saveBinary(&capacity_, sizeof(size_t));  | 
61  | 595  |     if (size_ == 0)  | 
62  | 37  |         return true;  | 
63  | 558  |     ar.saveBinary(ctrl_,  sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1));  | 
64  | 558  |     ar.saveBinary(slots_, sizeof(slot_type) * capacity_);  | 
65  | 558  |     ar.saveBinary(&growth_left(), sizeof(size_t));  | 
66  | 558  |     return true;  | 
67  | 595  | }  | 
68  |  |  | 
69  |  | template <class Policy, class Hash, class Eq, class Alloc>  | 
70  |  | template<typename InputArchive>  | 
71  | 595  | bool raw_hash_set<Policy, Hash, Eq, Alloc>::phmap_load(InputArchive& ar) { | 
72  | 595  |     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,  | 
73  | 595  |                     "value_type should be trivially copyable");  | 
74  | 595  |     raw_hash_set<Policy, Hash, Eq, Alloc>().swap(*this); // clear any existing content  | 
75  |  |  | 
76  | 595  |     size_t version = 0;  | 
77  | 595  |     ar.loadBinary(&version, sizeof(size_t));  | 
78  | 595  |     if (version < s_version_base) { | 
79  |  |         // we didn't store the version, version actually contains the size  | 
80  | 462  |         size_ = version;  | 
81  | 462  |     } else { | 
82  | 133  |         ar.loadBinary(&size_, sizeof(size_t));  | 
83  | 133  |     }  | 
84  | 595  |     ar.loadBinary(&capacity_, sizeof(size_t));  | 
85  |  |  | 
86  | 595  |     if (capacity_) { | 
87  |  |         // allocate memory for ctrl_ and slots_  | 
88  | 133  |         initialize_slots(capacity_);  | 
89  | 133  |     }  | 
90  | 595  |     if (size_ == 0)  | 
91  | 462  |         return true;  | 
92  | 133  |     ar.loadBinary(ctrl_,  sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1));  | 
93  | 133  |     ar.loadBinary(slots_, sizeof(slot_type) * capacity_);  | 
94  | 133  |     if (version >= s_version_base) { | 
95  |  |         // growth_left should be restored after calling initialize_slots() which resets it.  | 
96  | 133  |         ar.loadBinary(&growth_left(), sizeof(size_t));  | 
97  | 133  |     } else { | 
98  | 0  |        drop_deletes_without_resize();  | 
99  | 0  |     }  | 
100  | 133  |     return true;  | 
101  | 595  | }  | 
102  |  |  | 
103  |  | // ------------------------------------------------------------------------  | 
104  |  | // dump/load for parallel_hash_set  | 
105  |  | // ------------------------------------------------------------------------  | 
106  |  | template <size_t N,  | 
107  |  |           template <class, class, class, class> class RefSet,  | 
108  |  |           class Mtx_,  | 
109  |  |           class Policy, class Hash, class Eq, class Alloc>  | 
110  |  | template<typename OutputArchive>  | 
111  |  | bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::phmap_dump(OutputArchive& ar) const { | 
112  |  |     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,  | 
113  |  |                   "value_type should be trivially copyable");  | 
114  |  |  | 
115  |  |     size_t submap_count = subcnt();  | 
116  |  |     ar.saveBinary(&submap_count, sizeof(size_t));  | 
117  |  |     for (size_t i = 0; i < sets_.size(); ++i) { | 
118  |  |         auto& inner = sets_[i];  | 
119  |  |         typename Lockable::UniqueLock m(const_cast<Inner&>(inner));  | 
120  |  |         if (!inner.set_.phmap_dump(ar)) { | 
121  |  |             std::cerr << "Failed to dump submap " << i << std::endl;  | 
122  |  |             return false;  | 
123  |  |         }  | 
124  |  |     }  | 
125  |  |     return true;  | 
126  |  | }  | 
127  |  |  | 
128  |  | template <size_t N,  | 
129  |  |           template <class, class, class, class> class RefSet,  | 
130  |  |           class Mtx_,  | 
131  |  |           class Policy, class Hash, class Eq, class Alloc>  | 
132  |  | template<typename InputArchive>  | 
133  |  | bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::phmap_load(InputArchive& ar) { | 
134  |  |     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,  | 
135  |  |                   "value_type should be trivially copyable");  | 
136  |  |  | 
137  |  |     size_t submap_count = 0;  | 
138  |  |     ar.loadBinary(&submap_count, sizeof(size_t));  | 
139  |  |     if (submap_count != subcnt()) { | 
140  |  |         std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; | 
141  |  |         return false;  | 
142  |  |     }  | 
143  |  |  | 
144  |  |     for (size_t i = 0; i < submap_count; ++i) {             | 
145  |  |         auto& inner = sets_[i];  | 
146  |  |         typename Lockable::UniqueLock m(const_cast<Inner&>(inner));  | 
147  |  |         if (!inner.set_.phmap_load(ar)) { | 
148  |  |             std::cerr << "Failed to load submap " << i << std::endl;  | 
149  |  |             return false;  | 
150  |  |         }  | 
151  |  |     }  | 
152  |  |     return true;  | 
153  |  | }  | 
154  |  |  | 
155  |  | #endif // !defined(PHMAP_NON_DETERMINISTIC) && !defined(PHMAP_DISABLE_DUMP)  | 
156  |  |  | 
157  |  | } // namespace priv  | 
158  |  |  | 
159  |  |  | 
160  |  |  | 
161  |  | // ------------------------------------------------------------------------  | 
162  |  | // BinaryArchive  | 
163  |  | //       File is closed when archive object is destroyed  | 
164  |  | // ------------------------------------------------------------------------  | 
165  |  |  | 
166  |  | // ------------------------------------------------------------------------  | 
167  |  | // ------------------------------------------------------------------------  | 
168  |  | class BinaryOutputArchive { | 
169  |  | public:  | 
170  | 595  |     BinaryOutputArchive(const char *file_path) { | 
171  | 595  |       os_ = new std::ofstream(file_path, std::ofstream::out |  | 
172  | 595  |                                              std::ofstream::trunc |  | 
173  | 595  |                                              std::ofstream::binary);  | 
174  | 595  |       destruct_ = [this]() { delete os_; }; | 
175  | 595  |     }  | 
176  |  |  | 
177  | 0  |     BinaryOutputArchive(std::ostream &os) : os_(&os) {} | 
178  |  |  | 
179  | 595  |     ~BinaryOutputArchive() { | 
180  | 595  |         if (destruct_) { | 
181  | 595  |             destruct_();  | 
182  | 595  |         }  | 
183  | 595  |     }  | 
184  |  |       | 
185  |  |     BinaryOutputArchive(const BinaryOutputArchive&) = delete;  | 
186  |  |     BinaryOutputArchive& operator=(const BinaryOutputArchive&) = delete;  | 
187  |  |  | 
188  | 3.45k  |     bool saveBinary(const void *p, size_t sz) { | 
189  | 3.45k  |         os_->write(reinterpret_cast<const char*>(p), (std::streamsize)sz);  | 
190  | 3.45k  |         return true;  | 
191  | 3.45k  |     }  | 
192  |  |  | 
193  |  |     template<typename V>  | 
194  |  |     typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type  | 
195  |  |     saveBinary(const V& v) { | 
196  |  |         os_->write(reinterpret_cast<const char *>(&v), sizeof(V));  | 
197  |  |         return true;  | 
198  |  |     }  | 
199  |  |  | 
200  |  |     template<typename Map>  | 
201  |  |     auto saveBinary(const Map& v) -> decltype(v.phmap_dump(*this), bool())  | 
202  |  |     { | 
203  |  |         return v.phmap_dump(*this);  | 
204  |  |     }  | 
205  |  |  | 
206  |  | private:  | 
207  |  |     std::ostream* os_;  | 
208  |  |     std::function<void()> destruct_;  | 
209  |  | };  | 
210  |  |  | 
211  |  |  | 
212  |  | class BinaryInputArchive { | 
213  |  | public:  | 
214  | 595  |     BinaryInputArchive(const char * file_path) { | 
215  | 595  |       is_ = new std::ifstream(file_path,  | 
216  | 595  |                               std::ifstream::in | std::ifstream::binary);  | 
217  | 595  |       destruct_ = [this]() { delete is_; }; | 
218  | 595  |     }  | 
219  |  |  | 
220  | 0  |     BinaryInputArchive(std::istream& is) : is_(&is) {} | 
221  |  |       | 
222  | 595  |     ~BinaryInputArchive() { | 
223  | 595  |         if (destruct_) { | 
224  | 595  |             destruct_();  | 
225  | 595  |         }  | 
226  | 595  |     }  | 
227  |  |  | 
228  |  |     BinaryInputArchive(const BinaryInputArchive&) = delete;  | 
229  |  |     BinaryInputArchive& operator=(const BinaryInputArchive&) = delete;  | 
230  |  |  | 
231  | 1.72k  |     bool loadBinary(void* p, size_t sz) { | 
232  | 1.72k  |         is_->read(reinterpret_cast<char*>(p),  (std::streamsize)sz);  | 
233  | 1.72k  |         return true;  | 
234  | 1.72k  |     }  | 
235  |  |  | 
236  |  |     template<typename V>  | 
237  |  |     typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type  | 
238  |  |     loadBinary(V* v) { | 
239  |  |         is_->read(reinterpret_cast<char *>(v), sizeof(V));  | 
240  |  |         return true;  | 
241  |  |     }  | 
242  |  |  | 
243  |  |     template<typename Map>  | 
244  |  |     auto loadBinary(Map* v) -> decltype(v->phmap_load(*this), bool())  | 
245  |  |     { | 
246  |  |         return v->phmap_load(*this);  | 
247  |  |     }  | 
248  |  |       | 
249  |  | private:  | 
250  |  |     std::istream* is_;  | 
251  |  |     std::function<void()> destruct_;  | 
252  |  | };  | 
253  |  |  | 
254  |  | } // namespace phmap  | 
255  |  |  | 
256  |  |  | 
257  |  | #ifdef CEREAL_SIZE_TYPE  | 
258  |  |  | 
259  |  | template <class T>  | 
260  |  | using PhmapTrivCopyable = typename phmap::type_traits_internal::IsTriviallyCopyable<T>;  | 
261  |  |       | 
262  |  | namespace cereal  | 
263  |  | { | 
264  |  |     // Overload Cereal serialization code for phmap::flat_hash_map  | 
265  |  |     // -----------------------------------------------------------  | 
266  |  |     template <class K, class V, class Hash, class Eq, class A>  | 
267  |  |     void save(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryOutputArchive>::type &ar,  | 
268  |  |               phmap::flat_hash_map<K, V, Hash, Eq, A> const &hmap)  | 
269  |  |     { | 
270  |  |         hmap.phmap_dump(ar);  | 
271  |  |     }  | 
272  |  |  | 
273  |  |     template <class K, class V, class Hash, class Eq, class A>  | 
274  |  |     void load(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryInputArchive>::type &ar,   | 
275  |  |               phmap::flat_hash_map<K, V, Hash, Eq, A>  &hmap)  | 
276  |  |     { | 
277  |  |         hmap.phmap_load(ar);  | 
278  |  |     }  | 
279  |  |  | 
280  |  |  | 
281  |  |     // Overload Cereal serialization code for phmap::parallel_flat_hash_map  | 
282  |  |     // --------------------------------------------------------------------  | 
283  |  |     template <class K, class V, class Hash, class Eq, class A, size_t N, class Mtx_>  | 
284  |  |     void save(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryOutputArchive>::type &ar,  | 
285  |  |               phmap::parallel_flat_hash_map<K, V, Hash, Eq, A, N, Mtx_> const &hmap)  | 
286  |  |     { | 
287  |  |         hmap.phmap_dump(ar);  | 
288  |  |     }  | 
289  |  |  | 
290  |  |     template <class K, class V, class Hash, class Eq, class A, size_t N, class Mtx_>  | 
291  |  |     void load(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryInputArchive>::type &ar,   | 
292  |  |               phmap::parallel_flat_hash_map<K, V, Hash, Eq, A, N, Mtx_>  &hmap)  | 
293  |  |     { | 
294  |  |         hmap.phmap_load(ar);  | 
295  |  |     }  | 
296  |  |  | 
297  |  |     // Overload Cereal serialization code for phmap::flat_hash_set  | 
298  |  |     // -----------------------------------------------------------  | 
299  |  |     template <class K, class Hash, class Eq, class A>  | 
300  |  |     void save(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryOutputArchive>::type &ar,  | 
301  |  |               phmap::flat_hash_set<K, Hash, Eq, A> const &hset)  | 
302  |  |     { | 
303  |  |         hset.phmap_dump(ar);  | 
304  |  |     }  | 
305  |  |  | 
306  |  |     template <class K, class Hash, class Eq, class A>  | 
307  |  |     void load(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryInputArchive>::type &ar,   | 
308  |  |               phmap::flat_hash_set<K, Hash, Eq, A>  &hset)  | 
309  |  |     { | 
310  |  |         hset.phmap_load(ar);  | 
311  |  |     }  | 
312  |  |  | 
313  |  |     // Overload Cereal serialization code for phmap::parallel_flat_hash_set  | 
314  |  |     // --------------------------------------------------------------------  | 
315  |  |     template <class K, class Hash, class Eq, class A, size_t N, class Mtx_>  | 
316  |  |     void save(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryOutputArchive>::type &ar,  | 
317  |  |               phmap::parallel_flat_hash_set<K, Hash, Eq, A, N, Mtx_> const &hset)  | 
318  |  |     { | 
319  |  |         hset.phmap_dump(ar);  | 
320  |  |     }  | 
321  |  |  | 
322  |  |     template <class K, class Hash, class Eq, class A, size_t N, class Mtx_>  | 
323  |  |     void load(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryInputArchive>::type &ar,   | 
324  |  |               phmap::parallel_flat_hash_set<K, Hash, Eq, A, N, Mtx_>  &hset)  | 
325  |  |     { | 
326  |  |         hset.phmap_load(ar);  | 
327  |  |     }  | 
328  |  | }  | 
329  |  |  | 
330  |  | #endif  | 
331  |  |  | 
332  |  |  | 
333  |  |  | 
334  |  |  | 
335  |  | #endif // phmap_dump_h_guard_  |