Line data Source code
1 : #include "fd_ssarchive.h"
2 :
3 : #include "../../../util/log/fd_log.h"
4 :
5 : #include <errno.h>
6 : #include <dirent.h>
7 : #include <stdlib.h>
8 : #include <unistd.h>
9 :
10 : struct fd_ssarchive_entry {
11 : ulong slot;
12 : ulong base_slot;
13 : int is_zstd;
14 : char path[ PATH_MAX ];
15 : uchar hash[ FD_HASH_FOOTPRINT ];
16 : };
17 : typedef struct fd_ssarchive_entry fd_ssarchive_entry_t;
18 :
19 : #define SORT_NAME sort_ssarchive_entries
20 0 : #define SORT_KEY_T fd_ssarchive_entry_t
21 0 : #define SORT_BEFORE(a,b) ( (a).slot>(b).slot )
22 : #include "../../../util/tmpl/fd_sort.c"
23 :
24 : #define FD_SSARCHIVE_MAX_ENTRIES (512UL)
25 :
26 : int
27 : fd_ssarchive_parse_filename( char const * _name,
28 : ulong * full_slot,
29 : ulong * incremental_slot,
30 : uchar hash[ static FD_HASH_FOOTPRINT ],
31 0 : int * is_zstd ) {
32 0 : char name[ PATH_MAX ];
33 0 : fd_cstr_ncpy( name, _name, sizeof(name) );
34 :
35 0 : char * ptr = name;
36 0 : int is_incremental;
37 0 : if( !strncmp( ptr, "incremental-snapshot-", 21UL ) ) {
38 0 : is_incremental = 1;
39 0 : ptr += 21UL;
40 0 : } else if( !strncmp( ptr, "snapshot-", 9UL ) ) {
41 0 : is_incremental = 0;
42 0 : ptr += 9UL;
43 0 : } else {
44 0 : return -1;
45 0 : }
46 :
47 0 : char * next = strchr( ptr, '-' );
48 0 : if( FD_UNLIKELY( !next ) ) return -1;
49 0 : *next = '\0';
50 0 : char * endptr;
51 0 : *full_slot = strtoul( ptr, &endptr, 10 );
52 0 : if( FD_UNLIKELY( *endptr!='\0' || endptr==ptr || *full_slot==ULONG_MAX ) ) return -1;
53 :
54 0 : if( is_incremental ) {
55 0 : ptr = next + 1;
56 0 : next = strchr( ptr, '-' );
57 0 : if( FD_UNLIKELY( !next ) ) return -1;
58 0 : *next = '\0';
59 0 : *incremental_slot = strtoul( ptr, &endptr, 10 );
60 0 : if( FD_UNLIKELY( *endptr!='\0' || endptr==ptr || *incremental_slot==ULONG_MAX ) ) return -1;
61 0 : } else {
62 0 : *incremental_slot = ULONG_MAX;
63 0 : }
64 :
65 0 : ptr = next + 1;
66 0 : next = strchr( ptr, '.' );
67 0 : if( FD_UNLIKELY( !next ) ) return -1;
68 0 : *next = '\0';
69 0 : ulong sz = (ulong)(next - ptr);
70 0 : if( FD_UNLIKELY( sz>FD_BASE58_ENCODED_32_LEN ) ) return -1;
71 0 : uchar * result = fd_base58_decode_32( ptr, hash );
72 0 : if( FD_UNLIKELY( result!=hash ) ) return -1;
73 :
74 0 : ptr = next + 1;
75 :
76 0 : if( FD_LIKELY( 0==strcmp( ptr, "tar.zst" ) ) ) *is_zstd = 1;
77 0 : else if ( FD_LIKELY( 0==strcmp( ptr, "tar" ) ) ) *is_zstd = 0;
78 0 : else return -1;
79 :
80 0 : return 0;
81 0 : }
82 :
83 : int
84 : fd_ssarchive_latest_pair( char const * directory,
85 : int incremental_snapshot,
86 : ulong * full_slot,
87 : ulong * incremental_slot,
88 : char full_path[ static PATH_MAX ],
89 : char incremental_path[ static PATH_MAX ],
90 : int * full_is_zstd,
91 : int * incremental_is_zstd,
92 : uchar full_hash[ static FD_HASH_FOOTPRINT ],
93 0 : uchar incremental_hash[ static FD_HASH_FOOTPRINT ] ) {
94 0 : *full_slot = ULONG_MAX;
95 0 : *incremental_slot = ULONG_MAX;
96 :
97 0 : DIR * dir = opendir( directory );
98 0 : if( FD_UNLIKELY( !dir ) ) {
99 0 : if( FD_LIKELY( errno==ENOENT ) ) return -1;
100 0 : FD_LOG_ERR(( "opendir() failed `%s` (%i-%s)", directory, errno, fd_io_strerror( errno ) ));
101 0 : }
102 :
103 0 : fd_ssarchive_entry_t full_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
104 0 : fd_ssarchive_entry_t incremental_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
105 0 : ulong full_snapshots_cnt = 0UL;
106 0 : ulong incremental_snapshots_cnt = 0UL;
107 :
108 0 : struct dirent * entry;
109 0 : errno = 0;
110 0 : while(( entry = readdir( dir ) )) {
111 0 : if( FD_LIKELY( !strcmp( entry->d_name, "." ) || !strcmp( entry->d_name, ".." ) ) ) continue;
112 :
113 0 : int is_zstd;
114 0 : ulong entry_full_slot, entry_incremental_slot;
115 0 : uchar decoded_hash[ FD_HASH_FOOTPRINT ];
116 0 : if( FD_UNLIKELY( -1==fd_ssarchive_parse_filename( entry->d_name, &entry_full_slot, &entry_incremental_slot, decoded_hash, &is_zstd ) ) ) {
117 0 : FD_LOG_INFO(( "unrecognized snapshot file `%s/%s` in snapshots directory", directory, entry->d_name ));
118 0 : continue;
119 0 : }
120 :
121 0 : if( FD_LIKELY( entry_incremental_slot==ULONG_MAX ) ) {
122 0 : if( FD_UNLIKELY( full_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
123 0 : continue;
124 0 : }
125 :
126 0 : full_snapshots[ full_snapshots_cnt ].slot = entry_full_slot;
127 0 : full_snapshots[ full_snapshots_cnt ].base_slot = ULONG_MAX;
128 0 : full_snapshots[ full_snapshots_cnt ].is_zstd = is_zstd;
129 0 : if( FD_UNLIKELY( !fd_cstr_printf_check( full_snapshots[ full_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) ) ) {
130 0 : FD_LOG_ERR(( "snapshot path too long `%s/%s`", directory, entry->d_name ));
131 0 : }
132 0 : fd_memcpy( full_snapshots[ full_snapshots_cnt ].hash, decoded_hash, FD_HASH_FOOTPRINT );
133 0 : full_snapshots_cnt++;
134 0 : } else {
135 0 : if( FD_UNLIKELY( incremental_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
136 0 : continue;
137 0 : }
138 :
139 0 : incremental_snapshots[ incremental_snapshots_cnt ].slot = entry_incremental_slot;
140 0 : incremental_snapshots[ incremental_snapshots_cnt ].base_slot = entry_full_slot;
141 0 : incremental_snapshots[ incremental_snapshots_cnt ].is_zstd = is_zstd;
142 0 : if( FD_UNLIKELY( !fd_cstr_printf_check( incremental_snapshots[ incremental_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) ) ) {
143 0 : FD_LOG_ERR(( "snapshot path too long `%s/%s`", directory, entry->d_name ));
144 0 : }
145 0 : fd_memcpy( incremental_snapshots[ incremental_snapshots_cnt ].hash, decoded_hash, FD_HASH_FOOTPRINT );
146 0 : incremental_snapshots_cnt++;
147 0 : }
148 0 : }
149 :
150 0 : if( FD_UNLIKELY( -1==closedir( dir ) ) ) FD_LOG_ERR(( "closedir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
151 0 : if( FD_UNLIKELY( errno && errno!=ENOENT ) ) FD_LOG_ERR(( "readdir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
152 :
153 0 : if( FD_LIKELY( incremental_snapshot ) ) {
154 0 : if( FD_UNLIKELY( incremental_snapshots_cnt==0UL && full_snapshots_cnt==0UL ) ) return -1;
155 0 : if( FD_UNLIKELY( full_snapshots_cnt==0UL ) ) return -1;
156 :
157 0 : sort_ssarchive_entries_inplace( incremental_snapshots, incremental_snapshots_cnt );
158 0 : sort_ssarchive_entries_inplace( full_snapshots, full_snapshots_cnt );
159 :
160 0 : if( FD_UNLIKELY( incremental_snapshots_cnt==0UL ) ) {
161 0 : FD_LOG_WARNING(("no incremental snapshots found in `%s`, falling back to latest full snapshot", directory ));
162 0 : *full_slot = full_snapshots[ 0UL ].slot;
163 0 : *full_is_zstd = full_snapshots[ 0UL ].is_zstd;
164 0 : *incremental_slot = ULONG_MAX;
165 0 : *incremental_is_zstd = 0;
166 0 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ 0UL ].path ) );
167 0 : fd_memcpy( full_hash, full_snapshots[ 0UL ].hash, FD_HASH_FOOTPRINT );
168 0 : memset( incremental_hash, 0, FD_HASH_FOOTPRINT );
169 0 : return 0;
170 0 : }
171 :
172 0 : for( ulong i=0UL; i<incremental_snapshots_cnt; i++ ) {
173 0 : ulong base_slot = incremental_snapshots[ i ].base_slot;
174 0 : for( ulong j=0; j<full_snapshots_cnt; j++ ) {
175 0 : if( FD_LIKELY( full_snapshots[ j ].slot==base_slot ) ) {
176 0 : *full_slot = base_slot;
177 0 : *incremental_slot = incremental_snapshots[ i ].slot;
178 0 : *full_is_zstd = full_snapshots[ j ].is_zstd;
179 0 : *incremental_is_zstd = incremental_snapshots[ i ].is_zstd;
180 0 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ j ].path ) );
181 0 : FD_TEST( fd_cstr_printf_check( incremental_path, PATH_MAX, NULL, "%s", incremental_snapshots[ i ].path ) );
182 0 : fd_memcpy( full_hash, full_snapshots[ j ].hash, FD_HASH_FOOTPRINT );
183 0 : fd_memcpy( incremental_hash, incremental_snapshots[ i ].hash, FD_HASH_FOOTPRINT );
184 0 : return 0;
185 0 : } else if( FD_LIKELY( full_snapshots[ j ].slot<base_slot ) ) {
186 : /* full snapshots are sorted in descending order, so if we reach a
187 : full snapshot with slot smaller than the incremental snapshot's
188 : base slot, we can stop searching. */
189 0 : break;
190 0 : }
191 0 : }
192 0 : }
193 :
194 : /* if we reach here, it means all incrementals are dangling (they
195 : don't build off any full snapshot). fallback to a full
196 : snapshot in that case. */
197 0 : *full_slot = full_snapshots[ 0UL ].slot;
198 0 : *full_is_zstd = full_snapshots[ 0UL ].is_zstd;
199 0 : *incremental_slot = ULONG_MAX;
200 0 : *incremental_is_zstd = 0;
201 0 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ 0UL ].path ) );
202 0 : incremental_path[ 0UL ] = '\0';
203 0 : fd_memcpy( full_hash, full_snapshots[ 0UL ].hash, FD_HASH_FOOTPRINT );
204 0 : memset( incremental_hash, 0, FD_HASH_FOOTPRINT );
205 0 : return 0;
206 :
207 0 : } else {
208 0 : if( FD_UNLIKELY( full_snapshots_cnt==0UL ) ) return -1;
209 :
210 0 : sort_ssarchive_entries_inplace( full_snapshots, full_snapshots_cnt );
211 :
212 0 : *full_slot = full_snapshots[ 0UL ].slot;
213 0 : *full_is_zstd = full_snapshots[ 0UL ].is_zstd;
214 0 : *incremental_slot = ULONG_MAX;
215 0 : *incremental_is_zstd = 0;
216 0 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ 0UL ].path ) );
217 0 : incremental_path[ 0UL ] = '\0';
218 0 : fd_memcpy( full_hash, full_snapshots[ 0UL ].hash, FD_HASH_FOOTPRINT );
219 0 : memset( incremental_hash, 0, FD_HASH_FOOTPRINT );
220 0 : return 0;
221 0 : }
222 0 : }
223 :
224 : void
225 : fd_ssarchive_remove_old_snapshots( char const * directory,
226 : uint max_full_snapshots_to_keep,
227 0 : uint max_incremental_snapshots_to_keep ) {
228 0 : ulong full_snapshots_cnt = 0UL;
229 0 : ulong incremental_snapshots_cnt = 0UL;
230 0 : fd_ssarchive_entry_t full_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
231 0 : fd_ssarchive_entry_t incremental_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
232 :
233 0 : DIR * dir = opendir( directory );
234 0 : if( FD_UNLIKELY( !dir ) ) {
235 0 : if( FD_LIKELY( errno==ENOENT ) ) return;
236 0 : FD_LOG_ERR(( "opendir() failed `%s` (%i-%s)", directory, errno, fd_io_strerror( errno ) ));
237 0 : }
238 :
239 0 : struct dirent * entry;
240 :
241 0 : errno = 0;
242 0 : while(( entry = readdir( dir ) )) {
243 0 : if( FD_LIKELY( !strcmp( entry->d_name, "." ) || !strcmp( entry->d_name, ".." ) ) ) continue;
244 :
245 0 : int is_zstd;
246 0 : ulong entry_full_slot, entry_incremental_slot;
247 0 : uchar decoded_hash[ FD_HASH_FOOTPRINT ];
248 0 : if( FD_UNLIKELY( -1==fd_ssarchive_parse_filename( entry->d_name, &entry_full_slot, &entry_incremental_slot, decoded_hash, &is_zstd ) ) ) {
249 0 : FD_LOG_INFO(( "unrecognized snapshot file `%s/%s` in snapshots directory", directory, entry->d_name ));
250 0 : continue;
251 0 : }
252 :
253 0 : if( FD_LIKELY( entry_incremental_slot==ULONG_MAX ) ) {
254 0 : FD_TEST( entry_full_slot!=ULONG_MAX );
255 :
256 0 : if( FD_UNLIKELY( full_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
257 0 : continue;
258 0 : }
259 :
260 0 : full_snapshots[ full_snapshots_cnt ].slot = entry_full_slot;
261 0 : full_snapshots[ full_snapshots_cnt ].base_slot = ULONG_MAX;
262 0 : full_snapshots[ full_snapshots_cnt ].is_zstd = is_zstd;
263 0 : FD_TEST( fd_cstr_printf_check( full_snapshots[ full_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) );
264 0 : full_snapshots_cnt++;
265 0 : } else {
266 :
267 0 : if( FD_UNLIKELY( incremental_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
268 0 : continue;
269 0 : }
270 :
271 0 : incremental_snapshots[ incremental_snapshots_cnt ].slot = entry_incremental_slot;
272 0 : incremental_snapshots[ incremental_snapshots_cnt ].base_slot = entry_full_slot;
273 0 : incremental_snapshots[ incremental_snapshots_cnt ].is_zstd = is_zstd;
274 0 : FD_TEST( fd_cstr_printf_check( incremental_snapshots[ incremental_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) );
275 0 : incremental_snapshots_cnt++;
276 0 : }
277 0 : }
278 :
279 0 : if( FD_UNLIKELY( errno && errno!=ENOENT ) ) FD_LOG_ERR(( "readdir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
280 0 : if( FD_UNLIKELY( -1==closedir( dir ) ) ) FD_LOG_ERR(( "closedir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
281 :
282 0 : if( FD_LIKELY( full_snapshots_cnt>max_full_snapshots_to_keep ) ) {
283 0 : sort_ssarchive_entries_inplace( full_snapshots, full_snapshots_cnt );
284 0 : for( ulong i=max_full_snapshots_to_keep; i<full_snapshots_cnt; i++ ) {
285 0 : if( FD_UNLIKELY( -1==unlink( full_snapshots[ i ].path ) ) ) {
286 0 : FD_LOG_ERR(( "unlink(%s) failed (%i-%s)", full_snapshots[ i ].path, errno, fd_io_strerror( errno ) ));
287 0 : }
288 0 : }
289 0 : }
290 :
291 0 : if( FD_LIKELY( incremental_snapshots_cnt>max_incremental_snapshots_to_keep ) ) {
292 0 : sort_ssarchive_entries_inplace( incremental_snapshots, incremental_snapshots_cnt );
293 0 : for( ulong i=max_incremental_snapshots_to_keep; i<incremental_snapshots_cnt; i++ ) {
294 0 : if( FD_UNLIKELY( -1==unlink( incremental_snapshots[ i ].path ) ) ) {
295 0 : FD_LOG_ERR(( "unlink(%s) failed (%i-%s)", incremental_snapshots[ i ].path, errno, fd_io_strerror( errno ) ));
296 0 : }
297 0 : }
298 0 : }
299 0 : }
|
