Coverage Report

Created: 2025-10-12 06:12

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/hdf5/src/H5FDonion.c
Line
Count
Source
1
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2
 * Copyright by The HDF Group.                                               *
3
 * All rights reserved.                                                      *
4
 *                                                                           *
5
 * This file is part of HDF5.  The full HDF5 copyright notice, including     *
6
 * terms governing use, modification, and redistribution, is contained in    *
7
 * the LICENSE file, which can be found at the root of the source code       *
8
 * distribution tree, or in https://www.hdfgroup.org/licenses.               *
9
 * If you do not have access to either file, you may request a copy from     *
10
 * help@hdfgroup.org.                                                        *
11
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
12
13
/*
14
 * Onion Virtual File Driver (VFD)
15
 *
16
 * Purpose:     Provide in-file provenance and revision/version control.
17
 */
18
19
#include "H5FDmodule.h" /* This source code file is part of the H5FD module */
20
21
#include "H5private.h"      /* Generic Functions           */
22
#include "H5Eprivate.h"     /* Error handling              */
23
#include "H5Fprivate.h"     /* Files                       */
24
#include "H5FDsec2.h"       /* Sec2 file driver            */
25
#include "H5FDpkg.h"        /* File drivers                */
26
#include "H5FDonion_priv.h" /* Onion file driver internals */
27
#include "H5FLprivate.h"    /* Free Lists                  */
28
#include "H5Iprivate.h"     /* IDs                         */
29
#include "H5MMprivate.h"    /* Memory management           */
30
31
/* The driver identification number, initialized at runtime */
32
hid_t H5FD_ONION_id_g = H5I_INVALID_HID;
33
34
/******************************************************************************
35
 *
36
 * Structure:   H5FD_onion_t
37
 *
38
 * Purpose:     Store information required to manage an onionized file.
39
 *              This structure is created when such a file is "opened" and
40
 *              discarded when it is "closed".
41
 *
42
 * pu
43
 *
44
 *      Instance of H5FD_t which contains fields common to all VFDs.
45
 *      It must be the first item in this structure, since at higher levels,
46
 *      this structure will be treated as an instance of H5FD_t.
47
 *
48
 * fa
49
 *
50
 *      Instance of `H5FD_onion_fapl_info_t` containing the configuration data
51
 *      needed to "open" the HDF5 file.
52
 *
53
 * original_file
54
 *
55
 *      VFD handle for the original HDF5 file.
56
 *
57
 * onion_file
58
 *
59
 *      VFD handle for the onion file.
60
 *      NULL if not set to use the single, separate storage target.
61
 *
62
 * recovery_file
63
 *
64
 *      VFD handle for the history recovery file. This file is a backup of
65
 *      the existing history when an existing onion file is opened in RW mode.
66
 *
67
 * recovery_file_name
68
 *
69
 *      String allocated and populated on file-open in write mode and freed on
70
 *      file-close, stores the path/name of the 'recovery' file. The file
71
 *      created at this location is to be removed upon successful file-close
72
 *      from write mode.
73
 *
74
 * is_open_rw
75
 *
76
 *      Remember whether the file was opened in a read-write mode.
77
 *
78
 * align_history_on_pages
79
 *
80
 *      Remember whether onion-writes must be aligned to page boundaries.
81
 *
82
 * header
83
 *
84
 *      In-memory copy of the onion history data header.
85
 *
86
 * history
87
 *
88
 *      In-memory copy of the onion history.
89
 *
90
 * curr_rev_record
91
 *
92
 *      Record for the currently open revision.
93
 *
94
 * rev_index
95
 *
96
 *      Index for maintaining modified pages (RW mode only).
97
 *      Pointer is NULL when the file is not opened in write mode.
98
 *      Pointer is allocated on open and must be freed on close.
99
 *      Contents must be merged with the revision record's archival index prior
100
 *      to commitment of history to backing store.
101
 *
102
 * onion_eof
103
 *
104
 *      Last byte in the onion file.
105
 *
106
 * origin_eof
107
 *
108
 *     Size of the original HDF5 file.
109
 *
110
 * logical_eoa
111
 *
112
 *     Address of first byte past addressed space in the logical 'file'
113
 *     presented by this VFD.
114
 *
115
 * logical_eof
116
 *
117
 *     Address of first byte past last byte in the logical 'file' presented
118
 *     by this VFD.
119
 *     Must be copied into the revision record on close to write onion data.
120
 *
121
 ******************************************************************************
122
 */
123
typedef struct H5FD_onion_t {
124
    H5FD_t                 pub;
125
    H5FD_onion_fapl_info_t fa;
126
    bool                   is_open_rw;
127
    bool                   align_history_on_pages;
128
129
    /* Onion-related files */
130
    H5FD_t *original_file;
131
    H5FD_t *onion_file;
132
    H5FD_t *recovery_file;
133
    char   *recovery_file_name;
134
135
    /* Onion data structures */
136
    H5FD_onion_header_t          header;
137
    H5FD_onion_history_t         history;
138
    H5FD_onion_revision_record_t curr_rev_record;
139
    H5FD_onion_revision_index_t *rev_index;
140
141
    /* End of addresses and files */
142
    haddr_t onion_eof;
143
    haddr_t origin_eof;
144
    haddr_t logical_eoa;
145
    haddr_t logical_eof;
146
} H5FD_onion_t;
147
148
H5FL_DEFINE_STATIC(H5FD_onion_t);
149
150
0
#define H5FD_CTL_GET_NUM_REVISIONS 20001
151
152
/* Prototypes */
153
static herr_t  H5FD__onion_close(H5FD_t *);
154
static haddr_t H5FD__onion_get_eoa(const H5FD_t *, H5FD_mem_t);
155
static haddr_t H5FD__onion_get_eof(const H5FD_t *, H5FD_mem_t);
156
static H5FD_t *H5FD__onion_open(const char *, unsigned int, hid_t, haddr_t);
157
static herr_t  H5FD__onion_read(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, void *);
158
static herr_t  H5FD__onion_set_eoa(H5FD_t *, H5FD_mem_t, haddr_t);
159
static herr_t  H5FD__onion_write(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, const void *);
160
161
static herr_t  H5FD__onion_open_rw(H5FD_onion_t *, unsigned int, haddr_t, bool new_open);
162
static herr_t  H5FD__onion_sb_encode(H5FD_t *_file, char *name /*out*/, unsigned char *buf /*out*/);
163
static herr_t  H5FD__onion_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf);
164
static hsize_t H5FD__onion_sb_size(H5FD_t *_file);
165
static herr_t  H5FD__onion_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags,
166
                               const void H5_ATTR_UNUSED *input, void H5_ATTR_UNUSED **output);
167
static herr_t  H5FD__get_onion_revision_count(H5FD_t *file, uint64_t *revision_count);
168
169
/* Temporary */
170
H5_DLL herr_t H5FD__onion_write_final_history(H5FD_onion_t *file);
171
172
static const H5FD_class_t H5FD_onion_g = {
173
    H5FD_CLASS_VERSION,             /* struct version       */
174
    H5FD_ONION_VALUE,               /* value                */
175
    "onion",                        /* name                 */
176
    H5FD_MAXADDR,                   /* maxaddr              */
177
    H5F_CLOSE_WEAK,                 /* fc_degree            */
178
    NULL,                           /* terminate            */
179
    H5FD__onion_sb_size,            /* sb_size              */
180
    H5FD__onion_sb_encode,          /* sb_encode            */
181
    H5FD__onion_sb_decode,          /* sb_decode            */
182
    sizeof(H5FD_onion_fapl_info_t), /* fapl_size            */
183
    NULL,                           /* fapl_get             */
184
    NULL,                           /* fapl_copy            */
185
    NULL,                           /* fapl_free            */
186
    0,                              /* dxpl_size            */
187
    NULL,                           /* dxpl_copy            */
188
    NULL,                           /* dxpl_free            */
189
    H5FD__onion_open,               /* open                 */
190
    H5FD__onion_close,              /* close                */
191
    NULL,                           /* cmp                  */
192
    NULL,                           /* query                */
193
    NULL,                           /* get_type_map         */
194
    NULL,                           /* alloc                */
195
    NULL,                           /* free                 */
196
    H5FD__onion_get_eoa,            /* get_eoa              */
197
    H5FD__onion_set_eoa,            /* set_eoa              */
198
    H5FD__onion_get_eof,            /* get_eof              */
199
    NULL,                           /* get_handle           */
200
    H5FD__onion_read,               /* read                 */
201
    H5FD__onion_write,              /* write                */
202
    NULL,                           /* read_vector          */
203
    NULL,                           /* write_vector         */
204
    NULL,                           /* read_selection       */
205
    NULL,                           /* write_selection      */
206
    NULL,                           /* flush                */
207
    NULL,                           /* truncate             */
208
    NULL,                           /* lock                 */
209
    NULL,                           /* unlock               */
210
    NULL,                           /* del                  */
211
    H5FD__onion_ctl,                /* ctl                  */
212
    H5FD_FLMAP_DICHOTOMY            /* fl_map               */
213
};
214
215
/*-----------------------------------------------------------------------------
216
 * Function:    H5FD__onion_register
217
 *
218
 * Purpose:     Register the driver with the library.
219
 *
220
 * Return:      SUCCEED/FAIL
221
 *
222
 *-----------------------------------------------------------------------------
223
 */
224
herr_t
225
H5FD__onion_register(void)
226
2
{
227
2
    herr_t ret_value = SUCCEED; /* Return value */
228
229
2
    FUNC_ENTER_PACKAGE
230
231
2
    if (H5I_VFL != H5I_get_type(H5FD_ONION_id_g))
232
2
        if ((H5FD_ONION_id_g = H5FD_register(&H5FD_onion_g, sizeof(H5FD_class_t), false)) < 0)
233
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTREGISTER, FAIL, "unable to register onion driver");
234
235
2
done:
236
2
    FUNC_LEAVE_NOAPI(ret_value)
237
2
} /* end H5FD__onion_register() */
238
239
/*-----------------------------------------------------------------------------
240
 * Function:    H5FD__onion_unregister
241
 *
242
 * Purpose:     Reset library driver info.
243
 *
244
 * Returns:     SUCCEED (Can't fail)
245
 *
246
 *-----------------------------------------------------------------------------
247
 */
248
herr_t
249
H5FD__onion_unregister(void)
250
2
{
251
2
    FUNC_ENTER_PACKAGE_NOERR
252
253
    /* Reset VFL ID */
254
2
    H5FD_ONION_id_g = H5I_INVALID_HID;
255
256
2
    FUNC_LEAVE_NOAPI(SUCCEED)
257
2
} /* end H5FD__onion_unregister() */
258
259
/*-----------------------------------------------------------------------------
260
 * Function:    H5Pget_fapl_onion
261
 *
262
 * Purpose:     Copy the Onion configuration information from the FAPL at
263
 *              `fapl_id` to the destination pointer `fa_out`.
264
 *
265
 * Return:      Success: Non-negative value (SUCCEED).
266
 *              Failure: Negative value (FAIL).
267
 *-----------------------------------------------------------------------------
268
 */
269
herr_t
270
H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out)
271
0
{
272
0
    const H5FD_onion_fapl_info_t *info_ptr  = NULL;
273
0
    H5P_genplist_t               *plist     = NULL;
274
0
    herr_t                        ret_value = SUCCEED;
275
276
0
    FUNC_ENTER_API(FAIL)
277
278
0
    if (NULL == fa_out)
279
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info-out pointer");
280
281
0
    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
282
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID");
283
284
0
    if (H5FD_ONION != H5P_peek_driver(plist))
285
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Incorrect VFL driver");
286
287
0
    if (NULL == (info_ptr = (const H5FD_onion_fapl_info_t *)H5P_peek_driver_info(plist)))
288
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad VFL driver info");
289
290
0
    H5MM_memcpy(fa_out, info_ptr, sizeof(H5FD_onion_fapl_info_t));
291
292
0
done:
293
0
    FUNC_LEAVE_API(ret_value)
294
295
0
} /* end H5Pget_fapl_onion() */
296
297
/*-----------------------------------------------------------------------------
298
 * Function:    H5Pset_fapl_onion
299
 *
300
 * Purpose      Set the file access property list at `fapl_id` to use the
301
 *              Onion virtual file driver with the given configuration.
302
 *              The info structure may be modified or deleted after this call,
303
 *              as its contents are copied into the FAPL.
304
 *
305
 * Return:      Success: Non-negative value (SUCCEED).
306
 *              Failure: Negative value (FAIL).
307
 *-----------------------------------------------------------------------------
308
 */
309
herr_t
310
H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa)
311
0
{
312
0
    H5P_genplist_t *fapl           = NULL;
313
0
    H5P_genplist_t *backing_fapl   = NULL;
314
0
    hid_t           backing_vfd_id = H5I_INVALID_HID;
315
0
    herr_t          ret_value      = SUCCEED;
316
317
0
    FUNC_ENTER_API(FAIL)
318
319
0
    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
320
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID");
321
0
    if (NULL == fa)
322
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info pointer");
323
0
    if (H5FD_ONION_FAPL_INFO_VERSION_CURR != fa->version)
324
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info version");
325
0
    if (!POWER_OF_TWO(fa->page_size))
326
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size");
327
0
    if (fa->page_size < 1)
328
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size");
329
330
0
    if (H5P_DEFAULT == fa->backing_fapl_id) {
331
0
        if (NULL == (backing_fapl = H5P_object_verify(H5P_FILE_ACCESS_DEFAULT, H5P_FILE_ACCESS, true)))
332
0
            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id");
333
0
    }
334
0
    else {
335
0
        if (NULL == (backing_fapl = H5P_object_verify(fa->backing_fapl_id, H5P_FILE_ACCESS, true)))
336
0
            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id");
337
0
    }
338
339
    /* The only backing fapl that is currently supported is sec2 */
340
0
    if ((backing_vfd_id = H5P_peek_driver(backing_fapl)) < 0)
341
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "Can't get VFD from fapl");
342
0
    if (backing_vfd_id != H5FD_SEC2)
343
0
        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "Onion VFD only supports sec2 backing store");
344
345
0
    if (H5P_set_driver(fapl, H5FD_ONION, (const void *)fa, NULL) < 0)
346
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "Can't set the onion VFD");
347
348
0
done:
349
0
    FUNC_LEAVE_API(ret_value)
350
0
} /* end H5Pset_fapl_onion() */
351
352
/*-------------------------------------------------------------------------
353
 * Function:    H5FD__onion_sb_size
354
 *
355
 * Purpose:     Returns the size of the private information to be stored in
356
 *              the superblock.
357
 *
358
 * Return:      Success:    The super block driver data size
359
 *              Failure:    never fails
360
 *-------------------------------------------------------------------------
361
 */
362
static hsize_t
363
H5FD__onion_sb_size(H5FD_t *_file)
364
0
{
365
0
    H5FD_onion_t *file      = (H5FD_onion_t *)_file;
366
0
    hsize_t       ret_value = 0;
367
368
0
    FUNC_ENTER_PACKAGE_NOERR
369
370
    /* Sanity check */
371
0
    assert(file);
372
0
    assert(file->original_file);
373
374
0
    if (file->original_file)
375
0
        ret_value = H5FD_sb_size(file->original_file);
376
377
0
    FUNC_LEAVE_NOAPI(ret_value)
378
0
} /* end H5FD__onion_sb_size */
379
380
/*-------------------------------------------------------------------------
381
 * Function:    H5FD__onion_sb_encode
382
 *
383
 * Purpose:     Encodes the superblock information for this driver
384
 *
385
 * Return:      SUCCEED/FAIL
386
 *-------------------------------------------------------------------------
387
 */
388
static herr_t
389
H5FD__onion_sb_encode(H5FD_t *_file, char *name /*out*/, unsigned char *buf /*out*/)
390
0
{
391
0
    H5FD_onion_t *file      = (H5FD_onion_t *)_file;
392
0
    herr_t        ret_value = SUCCEED; /* Return value */
393
394
0
    FUNC_ENTER_PACKAGE
395
396
    /* Sanity check */
397
0
    assert(file);
398
0
    assert(file->original_file);
399
400
0
    if (file->original_file && H5FD_sb_encode(file->original_file, name, buf) < 0)
401
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTENCODE, FAIL, "unable to encode the superblock in R/W file");
402
403
0
done:
404
0
    FUNC_LEAVE_NOAPI(ret_value)
405
0
} /* end H5FD__onion_sb_encode */
406
407
/*-------------------------------------------------------------------------
408
 * Function:    H5FD__onion_sb_decode
409
 *
410
 * Purpose:     Decodes the superblock information for this driver
411
 *
412
 * Return:      SUCCEED/FAIL
413
 *-------------------------------------------------------------------------
414
 */
415
static herr_t
416
H5FD__onion_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf)
417
0
{
418
0
    H5FD_onion_t *file      = (H5FD_onion_t *)_file;
419
0
    herr_t        ret_value = SUCCEED; /* Return value */
420
421
0
    FUNC_ENTER_PACKAGE
422
423
    /* Sanity check */
424
0
    assert(file);
425
0
    assert(file->original_file);
426
427
0
    if (H5FD_sb_load(file->original_file, name, buf) < 0)
428
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "unable to decode the superblock in R/W file");
429
430
0
done:
431
0
    FUNC_LEAVE_NOAPI(ret_value)
432
0
} /* end H5FD__onion_sb_decode */
433
434
/*-----------------------------------------------------------------------------
435
 * Write in-memory revision record to appropriate backing file.
436
 * Update information in other in-memory components.
437
 *-----------------------------------------------------------------------------
438
 */
439
static herr_t
440
H5FD__onion_commit_new_revision_record(H5FD_onion_t *file)
441
0
{
442
0
    uint32_t                      checksum  = 0; /* required */
443
0
    size_t                        size      = 0;
444
0
    haddr_t                       phys_addr = 0; /* offset in history file to record start */
445
0
    unsigned char                *buf       = NULL;
446
0
    herr_t                        ret_value = SUCCEED;
447
0
    H5FD_onion_revision_record_t *rec       = &file->curr_rev_record;
448
0
    H5FD_onion_history_t         *history   = &file->history;
449
0
    H5FD_onion_record_loc_t      *new_list  = NULL;
450
451
0
    time_t     rawtime;
452
0
    struct tm *info;
453
454
0
    FUNC_ENTER_PACKAGE
455
456
0
    time(&rawtime);
457
0
    info = gmtime(&rawtime);
458
0
    strftime(rec->time_of_creation, sizeof(rec->time_of_creation), "%Y%m%dT%H%M%SZ", info);
459
460
0
    rec->logical_eof = file->logical_eof;
461
462
0
    if ((true == file->is_open_rw) && (H5FD__onion_merge_revision_index_into_archival_index(
463
0
                                           file->rev_index, &file->curr_rev_record.archival_index) < 0))
464
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "unable to update index to write");
465
466
0
    if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_REVISION_RECORD + (size_t)rec->comment_size +
467
0
                                   (H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * rec->archival_index.n_entries))))
468
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer for encoded revision record");
469
470
0
    if (0 == (size = H5FD__onion_revision_record_encode(rec, buf, &checksum)))
471
0
        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding revision record");
472
473
0
    phys_addr = file->onion_eof;
474
0
    if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, phys_addr + size) < 0)
475
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA for new revision record");
476
0
    if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, phys_addr, size, buf) < 0)
477
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write new revision record");
478
479
0
    file->onion_eof = phys_addr + size;
480
0
    if (true == file->align_history_on_pages)
481
0
        file->onion_eof = (file->onion_eof + (file->header.page_size - 1)) & (~(file->header.page_size - 1));
482
483
    /* Update history info to accommodate new revision */
484
485
0
    if (history->n_revisions == 0) {
486
0
        unsigned char *ptr = buf; /* reuse buffer space to compute checksum */
487
488
0
        assert(history->record_locs == NULL);
489
0
        history->n_revisions = 1;
490
0
        if (NULL == (history->record_locs = H5MM_calloc(sizeof(H5FD_onion_record_loc_t))))
491
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate temporary record pointer list");
492
493
0
        history->record_locs[0].phys_addr   = phys_addr;
494
0
        history->record_locs[0].record_size = size;
495
0
        UINT64ENCODE(ptr, phys_addr);
496
0
        UINT64ENCODE(ptr, size);
497
0
        history->record_locs[0].checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
498
        /* TODO: size-reset belongs where? */
499
0
        file->header.history_size += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER;
500
0
    } /* end if no extant revisions in history */
501
0
    else {
502
0
        unsigned char *ptr = buf; /* reuse buffer space to compute checksum */
503
504
0
        assert(history->record_locs != NULL);
505
506
0
        if (NULL == (new_list = H5MM_calloc((history->n_revisions + 1) * sizeof(H5FD_onion_record_loc_t))))
507
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to resize record pointer list");
508
0
        H5MM_memcpy(new_list, history->record_locs, sizeof(H5FD_onion_record_loc_t) * history->n_revisions);
509
0
        H5MM_xfree(history->record_locs);
510
0
        history->record_locs                                   = new_list;
511
0
        new_list                                               = NULL;
512
0
        history->record_locs[history->n_revisions].phys_addr   = phys_addr;
513
0
        history->record_locs[history->n_revisions].record_size = size;
514
0
        UINT64ENCODE(ptr, phys_addr);
515
0
        UINT64ENCODE(ptr, size);
516
0
        history->record_locs[history->n_revisions].checksum =
517
0
            H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
518
519
0
        file->header.history_size += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER;
520
0
        history->n_revisions += 1;
521
0
    } /* end if one or more revisions present in history */
522
523
0
    file->header.history_addr = file->onion_eof;
524
525
0
done:
526
0
    H5MM_xfree(buf);
527
0
    H5MM_xfree(new_list);
528
529
0
    FUNC_LEAVE_NOAPI(ret_value)
530
0
} /* end H5FD__onion_commit_new_revision_record() */
531
532
/*-----------------------------------------------------------------------------
533
 * Function:    H5FD__onion_close
534
 *
535
 * Purpose:     Close an onionized file
536
 *
537
 * Return:      SUCCEED/FAIL
538
 *-----------------------------------------------------------------------------
539
 */
540
static herr_t
541
H5FD__onion_close(H5FD_t *_file)
542
0
{
543
0
    H5FD_onion_t *file      = (H5FD_onion_t *)_file;
544
0
    herr_t        ret_value = SUCCEED;
545
546
0
    FUNC_ENTER_PACKAGE
547
548
0
    assert(file);
549
550
0
    if (H5FD_ONION_STORE_TARGET_ONION == file->fa.store_target) {
551
552
0
        assert(file->onion_file);
553
554
0
        if (file->is_open_rw) {
555
556
0
            assert(file->recovery_file);
557
558
0
            if (H5FD__onion_commit_new_revision_record(file) < 0)
559
0
                HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write revision record to backing store");
560
561
0
            if (H5FD__onion_write_final_history(file) < 0)
562
0
                HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write history to backing store");
563
564
            /* Unset write-lock flag and write header */
565
0
            if (file->is_open_rw)
566
0
                file->header.flags &= (uint32_t)~H5FD_ONION_HEADER_FLAG_WRITE_LOCK;
567
0
            if (H5FD__onion_write_header(&(file->header), file->onion_file) < 0)
568
0
                HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write updated header to backing store");
569
0
        }
570
0
    }
571
0
    else
572
0
        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid history target");
573
574
0
done:
575
576
    /* Destroy things as best we can, even if there were earlier errors */
577
0
    if (file->original_file)
578
0
        if (H5FD_close(file->original_file) < 0)
579
0
            HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing canon file");
580
0
    if (file->onion_file)
581
0
        if (H5FD_close(file->onion_file) < 0)
582
0
            HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing onion file");
583
0
    if (file->recovery_file) {
584
0
        if (H5FD_close(file->recovery_file) < 0)
585
0
            HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing recovery file");
586
        /* TODO: Use the VFD's del callback instead of remove (this requires
587
         *       storing a copy of the fapl that was used to open it)
588
         */
589
0
        HDremove(file->recovery_file_name);
590
0
    }
591
0
    if (file->rev_index)
592
0
        if (H5FD__onion_revision_index_destroy(file->rev_index) < 0)
593
0
            HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close revision index");
594
595
0
    H5MM_xfree(file->recovery_file_name);
596
0
    H5MM_xfree(file->history.record_locs);
597
0
    H5MM_xfree(file->curr_rev_record.comment);
598
0
    H5MM_xfree(file->curr_rev_record.archival_index.list);
599
600
0
    file = H5FL_FREE(H5FD_onion_t, file);
601
602
0
    FUNC_LEAVE_NOAPI(ret_value)
603
0
} /* end H5FD__onion_close() */
604
605
/*-----------------------------------------------------------------------------
606
 * Function:    H5FD__onion_get_eoa
607
 *
608
 * Purpose:     Get end-of-address address.
609
 *
610
 * Return:      Address of first byte past the addressed space
611
 *-----------------------------------------------------------------------------
612
 */
613
static haddr_t
614
H5FD__onion_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
615
0
{
616
0
    const H5FD_onion_t *file = (const H5FD_onion_t *)_file;
617
618
0
    FUNC_ENTER_PACKAGE_NOERR
619
620
0
    FUNC_LEAVE_NOAPI(file->logical_eoa)
621
0
} /* end H5FD__onion_get_eoa() */
622
623
/*-----------------------------------------------------------------------------
624
 * Function:    H5FD__onion_get_eof
625
 *
626
 * Purpose:     Get end-of-file address.
627
 *
628
 * Return:      Address of first byte past the file-end.
629
 *-----------------------------------------------------------------------------
630
 */
631
static haddr_t
632
H5FD__onion_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
633
0
{
634
0
    const H5FD_onion_t *file = (const H5FD_onion_t *)_file;
635
636
0
    FUNC_ENTER_PACKAGE_NOERR
637
638
0
    FUNC_LEAVE_NOAPI(file->logical_eof)
639
0
} /* end H5FD__onion_get_eof() */
640
641
/*-----------------------------------------------------------------------------
642
 * Sanitize the backing FAPL ID
643
 *-----------------------------------------------------------------------------
644
 */
645
static inline hid_t
646
H5FD__onion_get_legit_fapl_id(hid_t fapl_id)
647
0
{
648
0
    if (H5P_DEFAULT == fapl_id)
649
0
        return H5P_FILE_ACCESS_DEFAULT;
650
0
    else if (true == H5P_isa_class(fapl_id, H5P_FILE_ACCESS))
651
0
        return fapl_id;
652
0
    else
653
0
        return H5I_INVALID_HID;
654
0
}
655
656
/*-----------------------------------------------------------------------------
657
 * Function:    H5FD_onion_create_truncate_onion
658
 *
659
 * Purpose:     Create/truncate HDF5 and onion data for a fresh file
660
 *
661
 *      Special open operation required to instantiate the canonical file and
662
 *      history simultaneously. If successful, the required backing files are
663
 *      craeated and given initial population on the backing store, and the Onion
664
 *      virtual file handle is set; open effects a write-mode open.
665
 *
666
 *      Cannot create 'template' history and proceed with normal write-mode open,
667
 *      as this would in effect create an empty first revision, making the history
668
 *      unintuitive. (create file -> initialize and commit empty first revision
669
 *      (revision 0); any data written to file during the 'create' open, as seen by
670
 *      the user, would be in the second revision (revision 1).)
671
 *
672
 * Return:      SUCCEED/FAIL
673
 *-----------------------------------------------------------------------------
674
 */
675
static herr_t
676
H5FD__onion_create_truncate_onion(H5FD_onion_t *file, const char *filename, const char *name_onion,
677
                                  const char *recovery_file_nameery, unsigned int flags, haddr_t maxaddr)
678
0
{
679
0
    hid_t                         backing_fapl_id = H5I_INVALID_HID;
680
0
    H5FD_onion_header_t          *hdr             = NULL;
681
0
    H5FD_onion_history_t         *history         = NULL;
682
0
    H5FD_onion_revision_record_t *rec             = NULL;
683
0
    unsigned char                *buf             = NULL;
684
0
    size_t                        size            = 0;
685
0
    herr_t                        ret_value       = SUCCEED;
686
687
0
    FUNC_ENTER_PACKAGE
688
689
0
    assert(file != NULL);
690
691
0
    hdr     = &file->header;
692
0
    history = &file->history;
693
0
    rec     = &file->curr_rev_record;
694
695
0
    hdr->flags = H5FD_ONION_HEADER_FLAG_WRITE_LOCK;
696
0
    if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & file->fa.creation_flags)
697
0
        hdr->flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT;
698
699
0
    hdr->origin_eof = 0;
700
701
0
    backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id);
702
0
    if (H5I_INVALID_HID == backing_fapl_id)
703
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid backing FAPL ID");
704
705
    /* Create backing files for onion history */
706
0
    if (H5FD_open(false, &file->original_file, filename, flags, backing_fapl_id, maxaddr) < 0)
707
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "cannot open the backing file");
708
0
    if (H5FD_open(false, &file->onion_file, name_onion, flags, backing_fapl_id, maxaddr) < 0)
709
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "cannot open the backing onion file");
710
0
    if (H5FD_open(false, &file->recovery_file, recovery_file_nameery, flags, backing_fapl_id, maxaddr) < 0)
711
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "cannot open the backing file");
712
713
    /* Write "empty" .h5 file contents (signature ONIONEOF) */
714
0
    if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, 8) < 0)
715
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't extend EOA");
716
0
    if (H5FD_write(file->original_file, H5FD_MEM_DRAW, 0, 8, "ONIONEOF") < 0)
717
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "cannot write header to the backing h5 file");
718
719
    /* Write nascent history (with no revisions) to "recovery" */
720
0
    if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY)))
721
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer");
722
0
    size = H5FD__onion_history_encode(history, buf, &history->checksum);
723
0
    if (H5FD_ONION_ENCODED_SIZE_HISTORY != size)
724
0
        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history");
725
0
    if (H5FD_set_eoa(file->recovery_file, H5FD_MEM_DRAW, size) < 0)
726
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't extend EOA");
727
0
    if (H5FD_write(file->recovery_file, H5FD_MEM_DRAW, 0, size, buf) < 0)
728
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "cannot write history to the backing recovery file");
729
0
    hdr->history_size = size; /* record for later use */
730
0
    H5MM_xfree(buf);
731
0
    buf = NULL;
732
733
    /* Write history header with "no" history.
734
     * Size of the "recovery" history recorded for later use on close.
735
     */
736
0
    if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER)))
737
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer");
738
0
    size = H5FD__onion_header_encode(hdr, buf, &hdr->checksum);
739
0
    if (H5FD_ONION_ENCODED_SIZE_HEADER != size)
740
0
        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history header");
741
0
    if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, size) < 0)
742
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't extend EOA");
743
0
    if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, size, buf) < 0)
744
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "cannot write header to the backing onion file");
745
0
    file->onion_eof = (haddr_t)size;
746
0
    if (true == file->align_history_on_pages)
747
0
        file->onion_eof = (file->onion_eof + (hdr->page_size - 1)) & (~(hdr->page_size - 1));
748
749
0
    rec->archival_index.list = NULL;
750
751
0
    if (NULL == (file->rev_index = H5FD__onion_revision_index_init(file->fa.page_size)))
752
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize revision index");
753
754
0
done:
755
0
    H5MM_xfree(buf);
756
757
0
    if (FAIL == ret_value)
758
0
        HDremove(recovery_file_nameery); /* destroy new temp file, if 'twas created */
759
760
0
    FUNC_LEAVE_NOAPI(ret_value)
761
0
} /* end H5FD__onion_create_truncate_onion() */
762
763
static herr_t
764
H5FD__onion_remove_unused_symbols(char *s)
765
0
{
766
0
    char *d = s;
767
768
0
    FUNC_ENTER_PACKAGE_NOERR
769
770
0
    do {
771
0
        while (*d == '{' || *d == '}' || *d == ' ') {
772
0
            ++d;
773
0
        }
774
0
    } while ((*s++ = *d++));
775
776
0
    FUNC_LEAVE_NOAPI(SUCCEED)
777
0
}
778
779
static herr_t
780
H5FD__onion_parse_config_str(const char *config_str, H5FD_onion_fapl_info_t *fa)
781
0
{
782
0
    char  *config_str_copy = NULL;
783
0
    herr_t ret_value       = SUCCEED;
784
785
0
    FUNC_ENTER_PACKAGE
786
787
0
    if (!strcmp(config_str, ""))
788
0
        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "configure string can't be empty");
789
790
    /* Initialize to the default values */
791
0
    fa->version          = H5FD_ONION_FAPL_INFO_VERSION_CURR;
792
0
    fa->backing_fapl_id  = H5P_DEFAULT;
793
0
    fa->page_size        = 4;
794
0
    fa->store_target     = H5FD_ONION_STORE_TARGET_ONION;
795
0
    fa->revision_num     = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
796
0
    fa->force_write_open = 0;
797
0
    fa->creation_flags   = 0;
798
0
    strcpy(fa->comment, "initial comment");
799
800
    /* If a single integer is passed in as a string, it's a shortcut for the tools
801
     * (h5repack, h5diff, h5dump).  Otherwise, the string should have curly brackets,
802
     * e.g. {revision_num: 2; page_size: 4;}
803
     */
804
0
    if (config_str[0] != '{')
805
0
        fa->revision_num = (uint64_t)strtoull(config_str, NULL, 10);
806
0
    else {
807
0
        char *token1 = NULL, *token2 = NULL;
808
809
        /* Duplicate the configure string since strtok will mess with it */
810
0
        if (NULL == (config_str_copy = H5MM_strdup(config_str)))
811
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't duplicate configure string");
812
813
        /* Remove the curly brackets and space from the configure string */
814
0
        H5FD__onion_remove_unused_symbols(config_str_copy);
815
816
        /* The configure string can't be empty after removing the curly brackets */
817
0
        if (!strcmp(config_str_copy, ""))
818
0
            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "configure string can't be empty");
819
820
0
        token1 = strtok(config_str_copy, ":");
821
0
        token2 = strtok(NULL, ";");
822
823
0
        do {
824
0
            if (token1 && token2) {
825
0
                if (!strcmp(token1, "version")) {
826
0
                    if (!strcmp(token2, "H5FD_ONION_FAPL_INFO_VERSION_CURR"))
827
0
                        fa->version = H5FD_ONION_FAPL_INFO_VERSION_CURR;
828
0
                }
829
0
                else if (!strcmp(token1, "backing_fapl_id")) {
830
0
                    if (!strcmp(token2, "H5P_DEFAULT"))
831
0
                        fa->backing_fapl_id = H5P_DEFAULT;
832
0
                    else if (!strcmp(token2, "H5I_INVALID_HID"))
833
0
                        fa->backing_fapl_id = H5I_INVALID_HID;
834
0
                    else
835
0
                        fa->backing_fapl_id = strtoll(token2, NULL, 10);
836
0
                }
837
0
                else if (!strcmp(token1, "page_size")) {
838
0
                    fa->page_size = (uint32_t)strtoul(token2, NULL, 10);
839
0
                }
840
0
                else if (!strcmp(token1, "revision_num")) {
841
0
                    if (!strcmp(token2, "H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST"))
842
0
                        fa->revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
843
0
                    else
844
0
                        fa->revision_num = (uint64_t)strtoull(token2, NULL, 10);
845
0
                }
846
0
                else if (!strcmp(token1, "force_write_open")) {
847
0
                    fa->force_write_open = (uint8_t)strtoul(token2, NULL, 10);
848
0
                }
849
0
                else if (!strcmp(token1, "creation_flags")) {
850
0
                    fa->creation_flags = (uint8_t)strtoul(token2, NULL, 10);
851
0
                }
852
0
                else if (!strcmp(token1, "comment")) {
853
0
                    strcpy(fa->comment, token2);
854
0
                }
855
0
                else
856
0
                    HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unknown token in the configure string: %s",
857
0
                                token1);
858
0
            }
859
860
0
            token1 = strtok(NULL, ":");
861
0
            token2 = strtok(NULL, ";");
862
0
        } while (token1);
863
0
    }
864
865
0
    if (H5P_DEFAULT == fa->backing_fapl_id || H5I_INVALID_HID == fa->backing_fapl_id) {
866
0
        H5P_genclass_t *pclass; /* Property list class to modify */
867
868
0
        if (NULL == (pclass = (H5P_genclass_t *)H5I_object_verify(H5P_FILE_ACCESS, H5I_GENPROP_CLS)))
869
0
            HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, FAIL, "not a property list class");
870
871
        /* Create the new property list */
872
0
        if ((fa->backing_fapl_id = H5P_create_id(pclass, true)) < 0)
873
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, FAIL, "unable to create property list");
874
0
    }
875
876
0
done:
877
0
    H5MM_free(config_str_copy);
878
879
0
    FUNC_LEAVE_NOAPI(ret_value)
880
0
}
881
882
/*-----------------------------------------------------------------------------
883
 * Function:    H5FD__onion_open
884
 *
885
 * Purpose:     Open an onionized file
886
 *
887
 * Return:      Success:    A pointer to a new file data structure
888
 *              Failure:    NULL
889
 *-----------------------------------------------------------------------------
890
 */
891
static H5FD_t *
892
H5FD__onion_open(const char *filename, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
893
0
{
894
0
    H5P_genplist_t               *plist                 = NULL;
895
0
    H5FD_onion_t                 *file                  = NULL;
896
0
    const H5FD_onion_fapl_info_t *fa                    = NULL;
897
0
    H5FD_onion_fapl_info_t       *new_fa                = NULL;
898
0
    const char                   *config_str            = NULL;
899
0
    double                        log2_page_size        = 0.0;
900
0
    hid_t                         backing_fapl_id       = H5I_INVALID_HID;
901
0
    char                         *name_onion            = NULL;
902
0
    char                         *recovery_file_nameery = NULL;
903
0
    bool                          new_open              = false;
904
0
    haddr_t                       canon_eof             = 0;
905
0
    H5FD_t                       *ret_value             = NULL;
906
907
0
    FUNC_ENTER_PACKAGE
908
909
    /* Check arguments */
910
0
    if (!filename || !*filename)
911
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name");
912
0
    if (0 == maxaddr || HADDR_UNDEF == maxaddr)
913
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr");
914
0
    assert(H5P_DEFAULT != fapl_id);
915
0
    if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
916
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list");
917
918
    /* This VFD can be invoked by either H5Pset_fapl_onion() or
919
     * H5Pset_driver_by_name(). When invoked by the former, there will be
920
     * driver info to peek at.
921
     */
922
0
    fa = (const H5FD_onion_fapl_info_t *)H5P_peek_driver_info(plist);
923
924
0
    if (NULL == fa) {
925
0
        if (NULL == (config_str = H5P_peek_driver_config_str(plist)))
926
0
            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "missing VFL driver configure string");
927
928
        /* Allocate a new onion fapl info struct and fill it from the
929
         * configuration string
930
         */
931
0
        if (NULL == (new_fa = H5MM_calloc(sizeof(H5FD_onion_fapl_info_t))))
932
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate memory for onion fapl info struct");
933
0
        if (H5FD__onion_parse_config_str(config_str, new_fa) < 0)
934
0
            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "failed to parse configure string");
935
936
0
        fa = new_fa;
937
0
    }
938
939
    /* Check for unsupported target values */
940
0
    if (H5FD_ONION_STORE_TARGET_ONION != fa->store_target)
941
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid store target");
942
943
    /* Allocate space for the file struct */
944
0
    if (NULL == (file = H5FL_CALLOC(H5FD_onion_t)))
945
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct");
946
947
    /* Allocate space for onion VFD file names */
948
0
    if (NULL == (name_onion = H5MM_malloc(sizeof(char) * (strlen(filename) + 7))))
949
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate onion name string");
950
0
    snprintf(name_onion, strlen(filename) + 7, "%s.onion", filename);
951
952
0
    if (NULL == (recovery_file_nameery = H5MM_malloc(sizeof(char) * (strlen(name_onion) + 10))))
953
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate recovery name string");
954
0
    snprintf(recovery_file_nameery, strlen(name_onion) + 10, "%s.recovery", name_onion);
955
0
    file->recovery_file_name = recovery_file_nameery;
956
957
0
    if (NULL == (file->recovery_file_name = H5MM_malloc(sizeof(char) * (strlen(name_onion) + 10))))
958
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate recovery name string");
959
0
    snprintf(file->recovery_file_name, strlen(name_onion) + 10, "%s.recovery", name_onion);
960
961
    /* Translate H5P_DEFAULT to a real fapl ID, if necessary */
962
0
    backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id);
963
0
    if (H5I_INVALID_HID == backing_fapl_id)
964
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid backing FAPL ID");
965
966
    /* Initialize file structure fields */
967
968
0
    H5MM_memcpy(&(file->fa), fa, sizeof(H5FD_onion_fapl_info_t));
969
970
0
    file->header.version   = H5FD_ONION_HEADER_VERSION_CURR;
971
0
    file->header.page_size = file->fa.page_size; /* guarded on FAPL-set */
972
973
0
    file->history.version = H5FD_ONION_HISTORY_VERSION_CURR;
974
975
0
    file->curr_rev_record.version                = H5FD_ONION_REVISION_RECORD_VERSION_CURR;
976
0
    file->curr_rev_record.archival_index.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR;
977
978
    /* Check that the page size is a power of two */
979
0
    if ((fa->page_size == 0) || ((fa->page_size & (fa->page_size - 1)) != 0))
980
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "page size is not a power of two");
981
982
    /* Assign the page size */
983
0
    log2_page_size                                      = log2((double)(fa->page_size));
984
0
    file->curr_rev_record.archival_index.page_size_log2 = (uint32_t)log2_page_size;
985
986
    /* Proceed with open. */
987
988
0
    if ((H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) {
989
990
        /* Create a new onion file from scratch */
991
992
        /* Set flags */
993
0
        if (fa->creation_flags & H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT) {
994
0
            file->header.flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT;
995
0
            file->align_history_on_pages = true;
996
0
        }
997
998
        /* Truncate and create everything as necessary */
999
0
        if (H5FD__onion_create_truncate_onion(file, filename, name_onion, file->recovery_file_name, flags,
1000
0
                                              maxaddr) < 0)
1001
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to create/truncate onionized files");
1002
0
        file->is_open_rw = true;
1003
0
    }
1004
0
    else {
1005
1006
        /* Opening an existing onion file */
1007
1008
        /* Open the existing file using the specified fapl */
1009
0
        if (H5FD_open(false, &file->original_file, filename, flags, backing_fapl_id, maxaddr) < 0)
1010
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open canonical file (does not exist?)");
1011
1012
        /* Try to open any existing onion file */
1013
0
        if (H5FD_open(true, &file->onion_file, name_onion, flags, backing_fapl_id, maxaddr) < 0)
1014
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "cannot try opening the backing onion file");
1015
1016
        /* If that didn't work, create a new onion file */
1017
        /* TODO: Move to a new function */
1018
0
        if (NULL == file->onion_file) {
1019
0
            if (H5F_ACC_RDWR & flags) {
1020
0
                H5FD_onion_header_t          *hdr        = NULL;
1021
0
                H5FD_onion_history_t         *history    = NULL;
1022
0
                H5FD_onion_revision_record_t *rec        = NULL;
1023
0
                unsigned char                *head_buf   = NULL;
1024
0
                unsigned char                *hist_buf   = NULL;
1025
0
                size_t                        size       = 0;
1026
0
                size_t                        saved_size = 0;
1027
1028
0
                assert(file != NULL);
1029
1030
0
                hdr     = &file->header;
1031
0
                history = &file->history;
1032
0
                rec     = &file->curr_rev_record;
1033
1034
0
                new_open = true;
1035
1036
0
                if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & file->fa.creation_flags) {
1037
0
                    hdr->flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT;
1038
0
                    file->align_history_on_pages = true;
1039
0
                }
1040
1041
0
                if (HADDR_UNDEF == (canon_eof = H5FD_get_eof(file->original_file, H5FD_MEM_DEFAULT)))
1042
0
                    HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "cannot get size of canonical file");
1043
0
                if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, canon_eof) < 0)
1044
0
                    HGOTO_ERROR(H5E_VFL, H5E_CANTSET, NULL, "can't extend EOA");
1045
0
                hdr->origin_eof   = canon_eof;
1046
0
                file->logical_eof = canon_eof;
1047
1048
0
                backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id);
1049
1050
0
                if (H5I_INVALID_HID == backing_fapl_id)
1051
0
                    HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid backing FAPL ID");
1052
1053
                /* Create backing files for onion history */
1054
0
                if (H5FD_open(false, &file->onion_file, name_onion,
1055
0
                              (H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC), backing_fapl_id, maxaddr) < 0)
1056
0
                    HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "cannot open the backing onion file");
1057
1058
                /* Write history header with "no" history */
1059
0
                hdr->history_size = H5FD_ONION_ENCODED_SIZE_HISTORY; /* record for later use */
1060
0
                hdr->history_addr =
1061
0
                    H5FD_ONION_ENCODED_SIZE_HEADER + 1; /* TODO: comment these 2 or do some other way */
1062
0
                head_buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER);
1063
0
                if (NULL == head_buf)
1064
0
                    HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate buffer");
1065
0
                size = H5FD__onion_header_encode(hdr, head_buf, &hdr->checksum);
1066
0
                if (H5FD_ONION_ENCODED_SIZE_HEADER != size)
1067
0
                    HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't encode history header");
1068
1069
0
                hist_buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY);
1070
0
                if (NULL == hist_buf)
1071
0
                    HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate buffer");
1072
0
                saved_size                = size;
1073
0
                history->n_revisions      = 0;
1074
0
                size                      = H5FD__onion_history_encode(history, hist_buf, &history->checksum);
1075
0
                file->header.history_size = size; /* record for later use */
1076
0
                if (H5FD_ONION_ENCODED_SIZE_HISTORY != size)
1077
0
                    HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't encode history");
1078
0
                if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, saved_size + size + 1) < 0)
1079
0
                    HGOTO_ERROR(H5E_VFL, H5E_CANTSET, NULL, "can't extend EOA");
1080
1081
0
                if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, saved_size, head_buf) < 0)
1082
0
                    HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL,
1083
0
                                "cannot write header to the backing onion file");
1084
1085
0
                file->onion_eof = (haddr_t)saved_size;
1086
0
                if (true == file->align_history_on_pages)
1087
0
                    file->onion_eof = (file->onion_eof + (hdr->page_size - 1)) & (~(hdr->page_size - 1));
1088
1089
0
                rec->archival_index.list = NULL;
1090
1091
0
                file->header.history_addr = file->onion_eof;
1092
1093
                /* Write nascent history (with no revisions) to the backing onion file */
1094
0
                if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, saved_size + 1, size, hist_buf) < 0)
1095
0
                    HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL,
1096
0
                                "cannot write history to the backing onion file");
1097
1098
0
                file->header.history_size = size; /* record for later use */
1099
1100
0
                H5MM_xfree(head_buf);
1101
0
                H5MM_xfree(hist_buf);
1102
0
            }
1103
0
            else
1104
0
                HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open onion file (does not exist?).");
1105
0
        }
1106
1107
0
        if (HADDR_UNDEF == (canon_eof = H5FD_get_eof(file->original_file, H5FD_MEM_DEFAULT)))
1108
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "cannot get size of canonical file");
1109
0
        if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, canon_eof) < 0)
1110
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTSET, NULL, "can't extend EOA");
1111
1112
        /* Get the history header from the onion file */
1113
0
        if (H5FD__onion_ingest_header(&file->header, file->onion_file, 0) < 0)
1114
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get history header from backing store");
1115
0
        file->align_history_on_pages =
1116
0
            (file->header.flags & H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT) ? true : false;
1117
1118
        /* Opening a file twice in write mode is an error */
1119
0
        if (H5FD_ONION_HEADER_FLAG_WRITE_LOCK & file->header.flags)
1120
0
            HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "Can't open file already opened in write-mode");
1121
0
        else {
1122
            /* Read in the history from the onion file */
1123
0
            if (H5FD__onion_ingest_history(&file->history, file->onion_file, file->header.history_addr,
1124
0
                                           file->header.history_size) < 0)
1125
0
                HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get history from backing store");
1126
1127
            /* Sanity check on revision ID */
1128
0
            if (fa->revision_num > file->history.n_revisions &&
1129
0
                fa->revision_num != H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST)
1130
0
                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "target revision ID out of range");
1131
1132
0
            if (fa->revision_num == 0)
1133
0
                file->curr_rev_record.logical_eof = canon_eof;
1134
0
            else if (file->history.n_revisions > 0 &&
1135
0
                     H5FD__onion_ingest_revision_record(
1136
0
                         &file->curr_rev_record, file->onion_file, &file->history,
1137
0
                         MIN(fa->revision_num - 1, (file->history.n_revisions - 1))) < 0)
1138
0
                HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get revision record from backing store");
1139
1140
0
            if (H5F_ACC_RDWR & flags)
1141
0
                if (H5FD__onion_open_rw(file, flags, maxaddr, new_open) < 0)
1142
0
                    HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "can't write-open write-locked file");
1143
0
        }
1144
0
    } /* End if opening existing file */
1145
1146
    /* Copy comment from FAPL info, if one is given */
1147
0
    if ((H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) {
1148
        /* Free the old comment */
1149
0
        file->curr_rev_record.comment = H5MM_xfree(file->curr_rev_record.comment);
1150
1151
        /* The buffer is of size H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN + 1
1152
         *
1153
         * We're getting this buffer from a fixed-size array in a struct, which
1154
         * will be garbage and not null-terminated if the user isn't careful.
1155
         * Be careful of this and do strndup first to ensure strdup gets a
1156
         * null-termianted string (HDF5 doesn't provide a strnlen call if you
1157
         * don't have one).
1158
         */
1159
0
        if (NULL ==
1160
0
            (file->curr_rev_record.comment = H5MM_strndup(fa->comment, H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN)))
1161
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to duplicate comment string");
1162
1163
        /* TODO: Lengths of strings should be size_t */
1164
0
        file->curr_rev_record.comment_size = (uint32_t)strlen(fa->comment) + 1;
1165
0
    }
1166
0
    file->origin_eof  = file->header.origin_eof;
1167
0
    file->logical_eof = MAX(file->curr_rev_record.logical_eof, file->logical_eof);
1168
0
    file->logical_eoa = 0;
1169
1170
0
    file->onion_eof = H5FD_get_eoa(file->onion_file, H5FD_MEM_DRAW);
1171
0
    if (true == file->align_history_on_pages)
1172
0
        file->onion_eof = (file->onion_eof + (file->header.page_size - 1)) & (~(file->header.page_size - 1));
1173
1174
0
    ret_value = (H5FD_t *)file;
1175
1176
0
done:
1177
0
    H5MM_xfree(name_onion);
1178
0
    H5MM_xfree(recovery_file_nameery);
1179
1180
0
    if (config_str && new_fa)
1181
0
        if (fa && fa->backing_fapl_id)
1182
0
            if (H5I_GENPROP_LST == H5I_get_type(fa->backing_fapl_id))
1183
0
                H5I_dec_app_ref(fa->backing_fapl_id);
1184
1185
0
    if ((NULL == ret_value) && file) {
1186
0
        if (file->original_file)
1187
0
            if (H5FD_close(file->original_file) < 0)
1188
0
                HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing canon");
1189
0
        if (file->onion_file)
1190
0
            if (H5FD_close(file->onion_file) < 0)
1191
0
                HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing onion");
1192
0
        if (file->recovery_file)
1193
0
            if (H5FD_close(file->recovery_file) < 0)
1194
0
                HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing recov");
1195
0
        if (file->rev_index)
1196
0
            if (H5FD__onion_revision_index_destroy(file->rev_index) < 0)
1197
0
                HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy revision index");
1198
1199
0
        H5MM_xfree(file->history.record_locs);
1200
0
        H5MM_xfree(file->recovery_file_name);
1201
0
        H5MM_xfree(file->curr_rev_record.comment);
1202
1203
0
        H5FL_FREE(H5FD_onion_t, file);
1204
0
    }
1205
1206
0
    H5MM_xfree(new_fa);
1207
1208
0
    FUNC_LEAVE_NOAPI(ret_value)
1209
0
} /* end H5FD__onion_open() */
1210
1211
/*-----------------------------------------------------------------------------
1212
 * Function:    H5FD__onion_open_rw
1213
 *
1214
 * Purpose:     Complete onion file-open, handling process for write mode.
1215
 *
1216
 *              Creates recovery file if one does not exist.
1217
 *              Initializes 'live' revision index.
1218
 *              Force write-open is not yet supported (recovery provision) TODO
1219
 *              Establishes write-lock in history header (sets lock flag).
1220
 *
1221
 * Return:      SUCCEED/FAIL
1222
 *-----------------------------------------------------------------------------
1223
 */
1224
static herr_t
1225
H5FD__onion_open_rw(H5FD_onion_t *file, unsigned int flags, haddr_t maxaddr, bool new_open)
1226
0
{
1227
0
    unsigned char *buf       = NULL;
1228
0
    size_t         size      = 0;
1229
0
    uint32_t       checksum  = 0;
1230
0
    herr_t         ret_value = SUCCEED;
1231
1232
0
    FUNC_ENTER_PACKAGE
1233
1234
    /* Guard against simultaneous write-open.
1235
     * TODO: support recovery open with force-write-open flag in FAPL info.
1236
     */
1237
1238
0
    if (file->header.flags & H5FD_ONION_HEADER_FLAG_WRITE_LOCK)
1239
0
        HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "can't write-open write-locked file");
1240
1241
    /* Copy history to recovery file */
1242
0
    if (H5FD_open(false, &file->recovery_file, file->recovery_file_name,
1243
0
                  (flags | H5F_ACC_CREAT | H5F_ACC_TRUNC), file->fa.backing_fapl_id, maxaddr) < 0)
1244
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "unable to create recovery file");
1245
1246
0
    if (0 == (size = H5FD__onion_write_history(&file->history, file->recovery_file, 0, 0)))
1247
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write history to recovery file");
1248
0
    if (size != file->header.history_size)
1249
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "written history differed from expected size");
1250
1251
    /* Set write-lock flag in onion header */
1252
0
    if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER)))
1253
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate space for encoded buffer");
1254
0
    file->header.flags |= H5FD_ONION_HEADER_FLAG_WRITE_LOCK;
1255
0
    if (0 == (size = H5FD__onion_header_encode(&file->header, buf, &checksum)))
1256
0
        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding history header");
1257
0
    if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, size, buf) < 0)
1258
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write updated history header");
1259
1260
    /* Prepare revision index and finalize write-mode open */
1261
0
    if (NULL == (file->rev_index = H5FD__onion_revision_index_init(file->fa.page_size)))
1262
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize revision index");
1263
0
    file->curr_rev_record.parent_revision_num = file->curr_rev_record.revision_num;
1264
0
    if (!new_open)
1265
0
        file->curr_rev_record.revision_num += 1;
1266
0
    file->is_open_rw = true;
1267
1268
0
done:
1269
0
    if (FAIL == ret_value) {
1270
0
        if (file->recovery_file != NULL) {
1271
0
            if (H5FD_close(file->recovery_file) < 0)
1272
0
                HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close recovery file");
1273
0
            file->recovery_file = NULL;
1274
0
        }
1275
1276
0
        if (file->rev_index != NULL) {
1277
0
            if (H5FD__onion_revision_index_destroy(file->rev_index) < 0)
1278
0
                HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't destroy revision index");
1279
0
            file->rev_index = NULL;
1280
0
        }
1281
0
    }
1282
1283
0
    H5MM_xfree(buf);
1284
1285
0
    FUNC_LEAVE_NOAPI(ret_value)
1286
0
} /* end H5FD__onion_open_rw() */
1287
1288
/*-----------------------------------------------------------------------------
1289
 * Function:    H5FD__onion_read
1290
 *
1291
 * Purpose:     Read bytes from an onionized file
1292
 *
1293
 * Return:      SUCCEED/FAIL
1294
 *-----------------------------------------------------------------------------
1295
 */
1296
static herr_t
1297
H5FD__onion_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len,
1298
                 void *_buf_out)
1299
0
{
1300
0
    H5FD_onion_t  *file           = (H5FD_onion_t *)_file;
1301
0
    uint64_t       page_0         = 0;
1302
0
    size_t         n_pages        = 0;
1303
0
    uint32_t       page_size      = 0;
1304
0
    uint32_t       page_size_log2 = 0;
1305
0
    size_t         bytes_to_read  = len;
1306
0
    unsigned char *buf_out        = (unsigned char *)_buf_out;
1307
0
    herr_t         ret_value      = SUCCEED;
1308
1309
0
    FUNC_ENTER_PACKAGE
1310
1311
0
    assert(file != NULL);
1312
0
    assert(buf_out != NULL);
1313
1314
0
    if ((uint64_t)(offset + len) > file->logical_eoa)
1315
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Read extends beyond addressed space");
1316
1317
0
    if (0 == len)
1318
0
        goto done;
1319
1320
0
    page_size      = file->header.page_size;
1321
0
    page_size_log2 = file->curr_rev_record.archival_index.page_size_log2;
1322
0
    page_0         = offset >> page_size_log2;
1323
0
    n_pages        = (len + page_size - 1) >> page_size_log2;
1324
1325
    /* Read, page-by-page */
1326
0
    for (size_t i = 0; i < n_pages; i++) {
1327
0
        const H5FD_onion_index_entry_t *entry_out     = NULL;
1328
0
        haddr_t                         page_gap_head = 0; /* start of page to start of buffer */
1329
0
        haddr_t                         page_gap_tail = 0; /* end of buffer to end of page */
1330
0
        size_t                          page_readsize = 0;
1331
0
        uint64_t                        page_i        = page_0 + i;
1332
1333
0
        if (0 == i) {
1334
0
            page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1);
1335
            /* Check if we need to add an additional page to make up for the page_gap_head */
1336
0
            if (page_gap_head > 0 &&
1337
0
                (page_gap_head + (bytes_to_read % page_size) > page_size || bytes_to_read % page_size == 0)) {
1338
0
                n_pages++;
1339
0
            }
1340
0
        }
1341
1342
0
        if (n_pages - 1 == i)
1343
0
            page_gap_tail = page_size - bytes_to_read - page_gap_head;
1344
1345
0
        page_readsize = (size_t)page_size - page_gap_head - page_gap_tail;
1346
1347
0
        if (true == file->is_open_rw && file->fa.revision_num != 0 &&
1348
0
            H5FD__onion_revision_index_find(file->rev_index, page_i, &entry_out)) {
1349
            /* Page exists in 'live' revision index */
1350
0
            if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr + page_gap_head,
1351
0
                          page_readsize, buf_out) < 0)
1352
0
                HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get working file data");
1353
0
        }
1354
0
        else if (file->fa.revision_num != 0 &&
1355
0
                 H5FD__onion_archival_index_find(&file->curr_rev_record.archival_index, page_i, &entry_out)) {
1356
            /* Page exists in archival index */
1357
0
            if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr + page_gap_head,
1358
0
                          page_readsize, buf_out) < 0)
1359
0
                HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get previously-amended file data");
1360
0
        }
1361
0
        else {
1362
            /* Page does not exist in either index */
1363
1364
            /* Casts prevent truncation */
1365
0
            haddr_t addr_start   = (haddr_t)page_i * (haddr_t)page_size + (haddr_t)page_gap_head;
1366
0
            haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 : file->origin_eof - addr_start;
1367
0
            haddr_t read_size    = MIN(overlap_size, page_readsize);
1368
1369
            /* Get all original bytes in page range */
1370
0
            if ((read_size > 0) && H5FD_read(file->original_file, type, addr_start, read_size, buf_out) < 0) {
1371
0
                HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get original file data");
1372
0
            }
1373
1374
            /* Fill with 0s any gaps after end of original bytes
1375
             * and before end of page.
1376
             */
1377
0
            for (size_t j = read_size; j < page_readsize; j++)
1378
0
                buf_out[j] = 0;
1379
0
        }
1380
1381
0
        buf_out += page_readsize;
1382
0
        bytes_to_read -= page_readsize;
1383
0
    } /* end for each page in range */
1384
1385
0
    assert(0 == bytes_to_read);
1386
1387
0
done:
1388
0
    FUNC_LEAVE_NOAPI(ret_value)
1389
0
} /* end H5FD__onion_read() */
1390
1391
/*-----------------------------------------------------------------------------
1392
 * Function:    H5FD__onion_set_eoa
1393
 *
1394
 * Purpose:     Set end-of-address marker of the logical file.
1395
 *
1396
 * Return:      SUCCEED/FAIL
1397
 *-----------------------------------------------------------------------------
1398
 */
1399
static herr_t
1400
H5FD__onion_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr)
1401
0
{
1402
0
    H5FD_onion_t *file = (H5FD_onion_t *)_file;
1403
1404
0
    FUNC_ENTER_PACKAGE_NOERR
1405
1406
0
    file->logical_eoa = addr;
1407
1408
0
    FUNC_LEAVE_NOAPI(SUCCEED)
1409
0
} /* end H5FD__onion_set_eoa() */
1410
1411
/*-----------------------------------------------------------------------------
1412
 * Function:    H5FD__onion_write
1413
 *
1414
 * Purpose:     Write bytes to an onionized file
1415
 *
1416
 * Return:      SUCCEED/FAIL
1417
 *-----------------------------------------------------------------------------
1418
 */
1419
static herr_t
1420
H5FD__onion_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len,
1421
                  const void *_buf)
1422
0
{
1423
0
    H5FD_onion_t        *file           = (H5FD_onion_t *)_file;
1424
0
    uint64_t             page_0         = 0;
1425
0
    size_t               n_pages        = 0;
1426
0
    unsigned char       *page_buf       = NULL;
1427
0
    uint32_t             page_size      = 0;
1428
0
    uint32_t             page_size_log2 = 0;
1429
0
    size_t               bytes_to_write = len;
1430
0
    const unsigned char *buf            = (const unsigned char *)_buf;
1431
0
    herr_t               ret_value      = SUCCEED;
1432
1433
0
    FUNC_ENTER_PACKAGE
1434
1435
0
    assert(file != NULL);
1436
0
    assert(buf != NULL);
1437
0
    assert(file->rev_index != NULL);
1438
0
    assert((uint64_t)(offset + len) <= file->logical_eoa);
1439
1440
0
    if (false == file->is_open_rw)
1441
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Write not allowed if file not opened in write mode");
1442
1443
0
    if (0 == len)
1444
0
        goto done;
1445
1446
0
    page_size      = file->header.page_size;
1447
0
    page_size_log2 = file->curr_rev_record.archival_index.page_size_log2;
1448
0
    page_0         = offset >> page_size_log2;
1449
0
    n_pages        = (len + page_size - 1) >> page_size_log2;
1450
1451
0
    if (NULL == (page_buf = H5MM_calloc(page_size * sizeof(unsigned char))))
1452
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "cannot allocate temporary buffer");
1453
1454
    /* Write, page-by-page */
1455
0
    for (size_t i = 0; i < n_pages; i++) {
1456
0
        const unsigned char            *write_buf = buf;
1457
0
        H5FD_onion_index_entry_t        new_entry;
1458
0
        const H5FD_onion_index_entry_t *entry_out     = NULL;
1459
0
        haddr_t                         page_gap_head = 0; /* start of page to start of buffer */
1460
0
        haddr_t                         page_gap_tail = 0; /* end of buffer to end of page */
1461
0
        size_t                          page_n_used   = 0; /* nbytes from buffer for this page-write */
1462
0
        uint64_t                        page_i        = page_0 + i;
1463
1464
0
        if (0 == i) {
1465
0
            page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1);
1466
            /* If we have a page_gap_head and the number of bytes to write is
1467
             * evenly divisible by the page size we need to add an additional
1468
             * page to make up for the page_gap_head
1469
             */
1470
0
            if (page_gap_head > 0 && (page_gap_head + (bytes_to_write % page_size) > page_size ||
1471
0
                                      bytes_to_write % page_size == 0)) {
1472
0
                n_pages++;
1473
0
            }
1474
0
        }
1475
0
        if (n_pages - 1 == i)
1476
0
            page_gap_tail = page_size - bytes_to_write - page_gap_head;
1477
0
        page_n_used = page_size - page_gap_head - page_gap_tail;
1478
1479
        /* Modify page in revision index, if present */
1480
0
        if (H5FD__onion_revision_index_find(file->rev_index, page_i, &entry_out)) {
1481
0
            if (page_gap_head | page_gap_tail) {
1482
                /* Copy existing page verbatim. */
1483
0
                if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, page_buf) < 0)
1484
0
                    HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get working file data");
1485
                /* Overlay delta from input buffer onto page buffer. */
1486
0
                H5MM_memcpy(page_buf + page_gap_head, buf, page_n_used);
1487
0
                write_buf = page_buf;
1488
0
            } /* end if partial page */
1489
1490
0
            if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, write_buf) < 0)
1491
0
                HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write amended page data to backing file");
1492
1493
0
            buf += page_n_used; /* overflow never touched */
1494
0
            bytes_to_write -= page_n_used;
1495
1496
0
            continue;
1497
0
        } /* end if page exists in 'live' revision index */
1498
1499
0
        if (page_gap_head || page_gap_tail) {
1500
            /* Fill gaps with existing data or zeroes. */
1501
0
            if (H5FD__onion_archival_index_find(&file->curr_rev_record.archival_index, page_i, &entry_out)) {
1502
                /* Page exists in archival index */
1503
1504
                /* Copy existing page verbatim */
1505
0
                if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, page_buf) < 0)
1506
0
                    HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get previously-amended data");
1507
0
            }
1508
0
            else {
1509
0
                haddr_t addr_start   = (haddr_t)(page_i * page_size);
1510
0
                haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 : file->origin_eof - addr_start;
1511
0
                haddr_t read_size    = MIN(overlap_size, page_size);
1512
1513
                /* Get all original bytes in page range */
1514
0
                if ((read_size > 0) &&
1515
0
                    H5FD_read(file->original_file, type, addr_start, read_size, page_buf) < 0) {
1516
0
                    HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get original file data");
1517
0
                }
1518
1519
                /* Fill with 0s any gaps after end of original bytes
1520
                 * or start of page and before start of new data.
1521
                 */
1522
0
                for (size_t j = read_size; j < page_gap_head; j++)
1523
0
                    page_buf[j] = 0;
1524
1525
                /* Fill with 0s any gaps after end of original bytes
1526
                 * or end of new data and before end of page.
1527
                 */
1528
0
                for (size_t j = MAX(read_size, page_size - page_gap_tail); j < page_size; j++)
1529
0
                    page_buf[j] = 0;
1530
0
            } /* end if page exists in neither index */
1531
1532
            /* Copy input buffer to temporary page buffer */
1533
0
            assert((page_size - page_gap_head) >= page_n_used);
1534
0
            H5MM_memcpy(page_buf + page_gap_head, buf, page_n_used);
1535
0
            write_buf = page_buf;
1536
1537
0
        } /* end if data range does not span entire page */
1538
1539
0
        new_entry.logical_page = page_i;
1540
0
        new_entry.phys_addr    = file->onion_eof;
1541
1542
0
        if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, file->onion_eof + page_size) < 0)
1543
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA for new page amendment");
1544
1545
0
        if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, file->onion_eof, page_size, write_buf) < 0)
1546
0
            HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write amended page data to backing file");
1547
1548
0
        if (H5FD__onion_revision_index_insert(file->rev_index, &new_entry) < 0)
1549
0
            HGOTO_ERROR(H5E_VFL, H5E_CANTINSERT, FAIL, "can't insert new index entry into revision index");
1550
1551
0
        file->onion_eof += page_size;
1552
0
        buf += page_n_used; /* possible overflow never touched */
1553
0
        bytes_to_write -= page_n_used;
1554
1555
0
    } /* end for each page to write */
1556
1557
0
    assert(0 == bytes_to_write);
1558
1559
0
    file->logical_eof = MAX(file->logical_eof, (offset + len));
1560
1561
0
done:
1562
0
    H5MM_xfree(page_buf);
1563
1564
0
    FUNC_LEAVE_NOAPI(ret_value)
1565
0
} /* end H5FD__onion_write() */
1566
1567
/*-------------------------------------------------------------------------
1568
 * Function:    H5FD__onion_ctl
1569
 *
1570
 * Purpose:     Onion VFD version of the ctl callback.
1571
 *
1572
 *              The desired operation is specified by the op_code
1573
 *              parameter.
1574
 *
1575
 *              The flags parameter controls management of op_codes that
1576
 *              are unknown to the callback
1577
 *
1578
 *              The input and output parameters allow op_code specific
1579
 *              input and output
1580
 *
1581
 * Return:      SUCCEED/FAIL
1582
 *-------------------------------------------------------------------------
1583
 */
1584
static herr_t
1585
H5FD__onion_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
1586
                void **output)
1587
0
{
1588
0
    H5FD_onion_t *file      = (H5FD_onion_t *)_file;
1589
0
    herr_t        ret_value = SUCCEED;
1590
1591
0
    FUNC_ENTER_PACKAGE
1592
1593
    /* Sanity checks */
1594
0
    assert(file);
1595
1596
0
    switch (op_code) {
1597
0
        case H5FD_CTL_GET_NUM_REVISIONS:
1598
0
            if (!output || !*output)
1599
0
                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "the output parameter is null");
1600
1601
0
            **((uint64_t **)output) = file->history.n_revisions;
1602
0
            break;
1603
        /* Unknown op code */
1604
0
        default:
1605
0
            if (flags & H5FD_CTL_FAIL_IF_UNKNOWN_FLAG)
1606
0
                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set");
1607
0
            break;
1608
0
    }
1609
1610
0
done:
1611
0
    FUNC_LEAVE_NOAPI(ret_value)
1612
0
} /* end H5FD__onion_ctl() */
1613
1614
/*-------------------------------------------------------------------------
1615
 * Function:    H5FDget_onion_revision_count
1616
 *
1617
 * Purpose:     Get the number of revisions in an onion file
1618
 *
1619
 * Return:      SUCCEED/FAIL
1620
 *-------------------------------------------------------------------------
1621
 */
1622
herr_t
1623
H5FDonion_get_revision_count(const char *filename, hid_t fapl_id, uint64_t *revision_count /*out*/)
1624
0
{
1625
0
    H5P_genplist_t *plist     = NULL;
1626
0
    H5FD_t         *file      = NULL;
1627
0
    herr_t          ret_value = SUCCEED;
1628
1629
0
    FUNC_ENTER_API(FAIL)
1630
1631
    /* Check args */
1632
0
    if (!filename || !strcmp(filename, ""))
1633
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid file name");
1634
0
    if (!revision_count)
1635
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "revision count can't be null");
1636
1637
    /* Make sure using the correct driver */
1638
0
    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
1639
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid FAPL ID");
1640
0
    if (H5FD_ONION != H5P_peek_driver(plist))
1641
0
        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a Onion VFL driver");
1642
1643
    /* Open the file with the driver */
1644
0
    if (H5FD_open(false, &file, filename, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF) < 0)
1645
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "unable to open file with onion driver");
1646
1647
    /* Call the private function */
1648
0
    if (H5FD__get_onion_revision_count(file, revision_count) < 0)
1649
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "failed to get the number of revisions");
1650
1651
0
done:
1652
    /* Close H5FD_t structure pointer */
1653
0
    if (file && H5FD_close(file) < 0)
1654
0
        HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file");
1655
1656
0
    FUNC_LEAVE_API(ret_value)
1657
0
}
1658
1659
/*-------------------------------------------------------------------------
1660
 * Function:    H5FD__get_onion_revision_count
1661
 *
1662
 * Purpose:     Private version of H5FDget_onion_revision_count()
1663
 *
1664
 * Return:      SUCCEED/FAIL
1665
 *-------------------------------------------------------------------------
1666
 */
1667
static herr_t
1668
H5FD__get_onion_revision_count(H5FD_t *file, uint64_t *revision_count)
1669
0
{
1670
0
    uint64_t op_code;
1671
0
    uint64_t flags;
1672
0
    herr_t   ret_value = SUCCEED;
1673
1674
0
    FUNC_ENTER_PACKAGE
1675
1676
0
    assert(file);
1677
0
    assert(revision_count);
1678
1679
0
    op_code = H5FD_CTL_GET_NUM_REVISIONS;
1680
0
    flags   = H5FD_CTL_FAIL_IF_UNKNOWN_FLAG;
1681
1682
    /* Get the number of revisions via the ctl callback */
1683
0
    if (H5FD_ctl(file, op_code, flags, NULL, (void **)&revision_count) < 0)
1684
0
        HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed");
1685
1686
0
done:
1687
0
    FUNC_LEAVE_NOAPI(ret_value)
1688
0
}
1689
1690
/*-----------------------------------------------------------------------------
1691
 * Function:    H5FD__onion_write_final_history
1692
 *
1693
 * Purpose:     Write final history to appropriate backing file on file close
1694
 *
1695
 * Return:      SUCCEED/FAIL
1696
 *-----------------------------------------------------------------------------
1697
 */
1698
herr_t
1699
H5FD__onion_write_final_history(H5FD_onion_t *file)
1700
0
{
1701
0
    size_t size      = 0;
1702
0
    herr_t ret_value = SUCCEED;
1703
1704
0
    FUNC_ENTER_PACKAGE
1705
1706
    /* TODO: history EOF may not be correct (under what circumstances?) */
1707
0
    if (0 == (size = H5FD__onion_write_history(&(file->history), file->onion_file, file->onion_eof,
1708
0
                                               file->onion_eof)))
1709
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write final history");
1710
1711
0
    if (size != file->header.history_size)
1712
0
        HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "written history differed from expected size");
1713
1714
    /* Is last write operation to history file; no need to extend to page
1715
     * boundary if set to page-align.
1716
     */
1717
0
    file->onion_eof += size;
1718
1719
0
done:
1720
0
    FUNC_LEAVE_NOAPI(ret_value)
1721
0
} /* end H5FD__onion_write_final_history() */