/src/c-blosc2/blosc/blosc2-stdio.c

Source (jump to first uncovered line)
/*********************************************************************
  Blosc - Blocked Shuffling and Compression Library

  Copyright (c) 2021  Blosc Development Team <blosc@blosc.org>
  https://blosc.org
  License: BSD 3-Clause (see LICENSE.txt)

  See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/

#if defined(__linux__)
  /* Must be defined before anything else is included */
  #define _GNU_SOURCE
#endif

#include "blosc2/blosc2-stdio.h"
#include "blosc2.h"

#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <errno.h>
#include <inttypes.h>

#if defined(_WIN32)
  #include <memoryapi.h>
  // See https://github.com/Blosc/python-blosc2/issues/359
  #define fseek _fseeki64
  #define ftell _ftelli64
#else
  #include <sys/mman.h>
#endif


void *blosc2_stdio_open(const char *urlpath, const char *mode, void *params) {
  BLOSC_UNUSED_PARAM(params);
  FILE *file = fopen(urlpath, mode);
  if (file == NULL)
    return NULL;
  blosc2_stdio_file *my_fp = malloc(sizeof(blosc2_stdio_file));
  my_fp->file = file;
  return my_fp;
}

int blosc2_stdio_close(void *stream) {
  blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream;
  int err = fclose(my_fp->file);
  free(my_fp);
  return err;
}

int64_t blosc2_stdio_size(void *stream) {
  blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream;

  fseek(my_fp->file, 0, SEEK_END);
  int64_t size = ftell(my_fp->file);
  fseek(my_fp->file, 0, SEEK_SET);

  return size;
}

int64_t blosc2_stdio_write(const void *ptr, int64_t size, int64_t nitems, int64_t position, void *stream) {
  blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream;
  fseek(my_fp->file, position, SEEK_SET);

  size_t nitems_ = fwrite(ptr, (size_t) size, (size_t) nitems, my_fp->file);
  return (int64_t) nitems_;
}

int64_t blosc2_stdio_read(void **ptr, int64_t size, int64_t nitems, int64_t position, void *stream) {
  blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream;
  fseek(my_fp->file, position, SEEK_SET);

  void* data_ptr = *ptr;
  size_t nitems_ = fread(data_ptr, (size_t) size, (size_t) nitems, my_fp->file);
  return (int64_t) nitems_;
}

int blosc2_stdio_truncate(void *stream, int64_t size) {
  blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream;
  int rc;
#if defined(_MSC_VER)
  rc = _chsize_s(_fileno(my_fp->file), size);
#else
  rc = ftruncate(fileno(my_fp->file), size);
#endif
  return rc;
}

int blosc2_stdio_destroy(void* params) {
  BLOSC_UNUSED_PARAM(params);
  return 0;
}

#if defined(_WIN32)
void _print_last_error() {
    DWORD last_error = GetLastError();
    if(last_error == 0) {
        return;
    }

    LPSTR msg = NULL;
    FormatMessage(
      FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
      NULL,
      last_error,
      MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
      (LPSTR)&msg,
      0,
      NULL
    );

    printf("Message for the error %lu:\n%s\n", last_error, msg);
    LocalFree(msg);
}
#endif

void *blosc2_stdio_mmap_open(const char *urlpath, const char *mode, void* params) {
  BLOSC_UNUSED_PARAM(mode);

  blosc2_stdio_mmap *mmap_file = (blosc2_stdio_mmap *) params;
  if (mmap_file->addr != NULL) {
    if (strcmp(mmap_file->urlpath, urlpath) != 0) {
      BLOSC_TRACE_ERROR(
        "The memory-mapped file is already opened with the path %s and hence cannot be reopened with the path %s. This "
        "happens if you try to open a sframe (sparse frame); please note that memory-mapped files are not supported "
        "for sframes.",
        mmap_file->urlpath,
        urlpath
      );
      return NULL;
    }

    /* A memory-mapped file is only opened once */
    return mmap_file;
  }

  // Keep the original path to ensure that all future file openings are with the same path
  mmap_file->urlpath = malloc(strlen(urlpath) + 1);
  strcpy(mmap_file->urlpath, urlpath);

  /* mmap_file->mode mapping is similar to Numpy's memmap
  (https://github.com/numpy/numpy/blob/main/numpy/_core/memmap.py) and CPython
  (https://github.com/python/cpython/blob/main/Modules/mmapmodule.c) */
#if defined(_WIN32)
  char* open_mode;
  bool use_initial_mapping_size;
  if (strcmp(mmap_file->mode, "r") == 0) {
    mmap_file->access_flags = PAGE_READONLY;
    mmap_file->map_flags = FILE_MAP_READ;
    mmap_file->is_memory_only = false;
    open_mode = "rb";
    use_initial_mapping_size = false;
  } else if (strcmp(mmap_file->mode, "r+") == 0) {
    mmap_file->access_flags = PAGE_READWRITE;
    mmap_file->map_flags = FILE_MAP_WRITE;
    mmap_file->is_memory_only = false;
    open_mode = "rb+";
    use_initial_mapping_size = true;
  } else if (strcmp(mmap_file->mode, "w+") == 0) {
    mmap_file->access_flags = PAGE_READWRITE;
    mmap_file->map_flags = FILE_MAP_WRITE;
    mmap_file->is_memory_only = false;
    open_mode = "wb+";
    use_initial_mapping_size = true;
  } else if (strcmp(mmap_file->mode, "c") == 0) {
    mmap_file->access_flags = PAGE_WRITECOPY;
    mmap_file->map_flags = FILE_MAP_COPY;
    mmap_file->is_memory_only = true;
    open_mode = "rb";
    use_initial_mapping_size = false;
  } else {
    BLOSC_TRACE_ERROR("Mode %s not supported for memory-mapped files.", mmap_file->mode);
    return NULL;
  }
#else
  char* open_mode;
  bool use_initial_mapping_size;
  if (strcmp(mmap_file->mode, "r") == 0) {
    mmap_file->access_flags = PROT_READ;
    mmap_file->map_flags = MAP_SHARED;
    mmap_file->is_memory_only = false;
    open_mode = "rb";
    use_initial_mapping_size = false;
  } else if (strcmp(mmap_file->mode, "r+") == 0) {
    mmap_file->access_flags = PROT_READ | PROT_WRITE;
    mmap_file->map_flags = MAP_SHARED;
    mmap_file->is_memory_only = false;
    open_mode = "rb+";
    use_initial_mapping_size = true;
  } else if (strcmp(mmap_file->mode, "w+") == 0) {
    mmap_file->access_flags = PROT_READ | PROT_WRITE;
    mmap_file->map_flags = MAP_SHARED;
    mmap_file->is_memory_only = false;
    open_mode = "wb+";
    use_initial_mapping_size = true;
  } else if (strcmp(mmap_file->mode, "c") == 0) {
    mmap_file->access_flags = PROT_READ | PROT_WRITE;
    mmap_file->map_flags = MAP_PRIVATE;
    mmap_file->is_memory_only = true;
    open_mode = "rb";
    use_initial_mapping_size = true;
  } else {
    BLOSC_TRACE_ERROR("Mode %s not supported for memory-mapped files.", mmap_file->mode);
    return NULL;
  }
#endif

  mmap_file->file = fopen(urlpath, open_mode);
  if (mmap_file->file == NULL) {
    BLOSC_TRACE_ERROR("Cannot open the file %s with mode %s.", urlpath, open_mode);
    return NULL;
  }

  /* Retrieve the size of the file */
  fseek(mmap_file->file, 0, SEEK_END);
  mmap_file->file_size = ftell(mmap_file->file);
  fseek(mmap_file->file, 0, SEEK_SET);

  /* The size of the mapping must be > 0 so we are using a large enough buffer for writing
  (which will be increased later if needed) */
  if (use_initial_mapping_size) {
    mmap_file->mapping_size = mmap_file->initial_mapping_size;
  }
  else {
    mmap_file->mapping_size = mmap_file->file_size;
  }

  if (mmap_file->file_size > mmap_file->mapping_size) {
    mmap_file->mapping_size = mmap_file->file_size;
  }

#if defined(_WIN32)
  mmap_file->fd = _fileno(mmap_file->file);

  /* Windows automatically expands the file size to the memory mapped file
  (https://learn.microsoft.com/en-us/windows/win32/memory/creating-a-file-mapping-object).
  In general, the size of the file is directly connected to the size of the mapping and cannot change. We cut the
  file size to the target size in the end after we close the mapping */
  HANDLE file_handle = (HANDLE) _get_osfhandle(mmap_file->fd);
  DWORD size_hi = (DWORD)(mmap_file->mapping_size >> 32);
  DWORD size_lo = (DWORD)(mmap_file->mapping_size & 0xFFFFFFFF);
  mmap_file->mmap_handle = CreateFileMapping(file_handle, NULL, mmap_file->access_flags, size_hi, size_lo, NULL);
  if (mmap_file->mmap_handle == NULL) {
    _print_last_error();
    BLOSC_TRACE_ERROR("Creating the memory mapping failed for the file %s.", urlpath);
    return NULL;
  }

  DWORD offset = 0;
  mmap_file->addr = (char*) MapViewOfFile(
    mmap_file->mmap_handle, mmap_file->map_flags, offset, offset, mmap_file->mapping_size);
  if (mmap_file->addr == NULL) {
    _print_last_error();
    BLOSC_TRACE_ERROR("Memory mapping failed for the file %s.", urlpath);

    if (!CloseHandle(mmap_file->mmap_handle)) {
      _print_last_error();
      BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
    }

    return NULL;
  }
#else
  mmap_file->fd = fileno(mmap_file->file);

  /* Offset where the mapping should start */
  int64_t offset = 0;
  mmap_file->addr = mmap(
    NULL, mmap_file->mapping_size, mmap_file->access_flags, mmap_file->map_flags, mmap_file->fd, offset);
  if (mmap_file->addr == MAP_FAILED) {
    BLOSC_TRACE_ERROR("Memory mapping failed for file %s (error: %s).", urlpath, strerror(errno));
    return NULL;
  }
#endif

  BLOSC_INFO(
    "Opened memory-mapped file %s in mode %s with an mapping size of %" PRId64 " bytes.",
    mmap_file->urlpath,
    mmap_file->mode,
    mmap_file->mapping_size
  );

  /* The mmap_file->mode parameter is only available during the opening call and cannot be used in any of the other
     I/O functions since this string is managed by the caller (e.g., from Python) and the memory of the string may not
     be available anymore at a later point. */
  mmap_file->mode = NULL;

  return mmap_file;
}

int blosc2_stdio_mmap_close(void *stream) {
  BLOSC_UNUSED_PARAM(stream);
  return 0;
}

int64_t blosc2_stdio_mmap_size(void *stream) {
  blosc2_stdio_mmap *mmap_file = (blosc2_stdio_mmap *) stream;
  return mmap_file->file_size;
}

int64_t blosc2_stdio_mmap_write(const void *ptr, int64_t size, int64_t nitems, int64_t position, void *stream) {
  blosc2_stdio_mmap *mmap_file = (blosc2_stdio_mmap *) stream;

  if (position < 0) {
    BLOSC_TRACE_ERROR("Cannot write to a negative position.");
    return 0;
  }

  int64_t n_bytes = size * nitems;
  if (n_bytes == 0) {
    return 0;
  }

  int64_t position_end = position + n_bytes;
  int64_t new_size = position_end > mmap_file->file_size ? position_end : mmap_file->file_size;

#if defined(_WIN32)
  if (mmap_file->file_size < new_size) {
    mmap_file->file_size = new_size;
  }

  if (mmap_file->mapping_size < mmap_file->file_size) {
    mmap_file->mapping_size = mmap_file->file_size * 2;

    /* We need to remap the file completely and cannot pass the previous used address on Windows */
    if (!UnmapViewOfFile(mmap_file->addr)) {
      _print_last_error();
      BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file.");
      return 0;
    }
    if (!CloseHandle(mmap_file->mmap_handle)) {
      _print_last_error();
      BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
      return 0;
    }

    HANDLE file_handle = (HANDLE) _get_osfhandle(mmap_file->fd);
    DWORD size_hi = (DWORD)(mmap_file->mapping_size >> 32);
    DWORD size_lo = (DWORD)(mmap_file->mapping_size & 0xFFFFFFFF);
    mmap_file->mmap_handle = CreateFileMapping(file_handle, NULL, mmap_file->access_flags, size_hi, size_lo, NULL);
    if (mmap_file->mmap_handle == NULL) {
      _print_last_error();
      BLOSC_TRACE_ERROR("Cannot remapt the memory-mapped file.");
      return 0;
    }

    DWORD offset = 0;
    char* new_address = (char*) MapViewOfFile(
      mmap_file->mmap_handle, mmap_file->map_flags, offset, offset, mmap_file->mapping_size);
    if (new_address == NULL) {
      _print_last_error();
      BLOSC_TRACE_ERROR("Cannot remapt the memory-mapped file");

      if (!CloseHandle(mmap_file->mmap_handle)) {
        _print_last_error();
        BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
      }

      return 0;
    }

    mmap_file->addr = new_address;
  }
#else
  if (mmap_file->file_size < new_size) {
    mmap_file->file_size = new_size;

    if (!mmap_file->is_memory_only) {
      int rc = ftruncate(mmap_file->fd, new_size);
      if (rc < 0) {
        BLOSC_TRACE_ERROR("Cannot extend the file size to %" PRId64 " bytes (error: %s).", new_size, strerror(errno));
        return 0;
      }
    }
  }

  if (mmap_file->mapping_size < mmap_file->file_size) {
    mmap_file->mapping_size = mmap_file->file_size * 2;

#if defined(__linux__)
    int64_t old_mapping_size = mmap_file->mapping_size;

    /* mremap is the best option as it also ensures that the old data is still available in c mode. Unfortunately, it
    is no POSIX standard and only available on Linux */
    char* new_address = mremap(mmap_file->addr, old_mapping_size, mmap_file->mapping_size, MREMAP_MAYMOVE);
#else
    if (mmap_file->is_memory_only) {
      BLOSC_TRACE_ERROR("Remapping a memory-mapping in c mode is only possible on Linux."
      "Please specify either a different mode or set initial_mapping_size to a large enough number.");
      return 0;
    }
    /* Extend the current mapping with the help of MAP_FIXED */
    int64_t offset = 0;
    char* new_address = mmap(
      mmap_file->addr,
      mmap_file->mapping_size,
      mmap_file->access_flags,
      mmap_file->map_flags | MAP_FIXED,
      mmap_file->fd,
      offset
    );
#endif

    if (new_address == MAP_FAILED) {
      BLOSC_TRACE_ERROR("Cannot remap the memory-mapped file (error: %s).", strerror(errno));
      if (munmap(mmap_file->addr, mmap_file->mapping_size) < 0) {
        BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file (error: %s).", strerror(errno));
      }

      return 0;
    }

    mmap_file->addr = new_address;
  }
#endif

  memcpy(mmap_file->addr + position, ptr, n_bytes);
  return nitems;
}

int64_t blosc2_stdio_mmap_read(void **ptr, int64_t size, int64_t nitems, int64_t position, void *stream) {
  blosc2_stdio_mmap *mmap_file = (blosc2_stdio_mmap *) stream;

  if (position < 0) {
    BLOSC_TRACE_ERROR("Cannot read from a negative position.");
    *ptr = NULL;
    return 0;
  }

  if (position + size * nitems > mmap_file->file_size) {
    BLOSC_TRACE_ERROR("Cannot read beyond the end of the memory-mapped file.");
    *ptr = NULL;
    return 0;
  }

  *ptr = mmap_file->addr + position;

  return nitems;
}

int blosc2_stdio_mmap_truncate(void *stream, int64_t size) {
  blosc2_stdio_mmap *mmap_file = (blosc2_stdio_mmap *) stream;

  if (mmap_file->file_size == size) {
    return 0;
  }

  mmap_file->file_size = size;

  /* No file operations in c mode */
  if (mmap_file->is_memory_only) {
    return 0;
  }

#if defined(_WIN32)
  /* On Windows, we can truncate the file only at the end after we released the mapping */
  return 0;
#else
  return ftruncate(mmap_file->fd, size);
#endif
}

int blosc2_stdio_mmap_destroy(void* params) {
  blosc2_stdio_mmap *mmap_file = (blosc2_stdio_mmap *) params;
  int err = 0;

#if defined(_WIN32)
  if (mmap_file->access_flags == PAGE_READWRITE) {
    /* Ensure modified pages are written to disk */
    if (!FlushViewOfFile(mmap_file->addr, mmap_file->file_size)) {
      _print_last_error();
      BLOSC_TRACE_ERROR("Cannot flush the memory-mapped view to disk.");
      err = -1;
    }
    HANDLE file_handle = (HANDLE) _get_osfhandle(mmap_file->fd);
    if (!FlushFileBuffers(file_handle)) {
      _print_last_error();
      BLOSC_TRACE_ERROR("Cannot flush the memory-mapped file to disk.");
      err = -1;
    }
  }

  if (!UnmapViewOfFile(mmap_file->addr)) {
    _print_last_error();
    BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file.");
    err = -1;
  }
  if (!CloseHandle(mmap_file->mmap_handle)) {
    _print_last_error();
    BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
    err = -1;
  }
  int rc = _chsize_s(mmap_file->fd, mmap_file->file_size);
  if (rc != 0) {
    BLOSC_TRACE_ERROR(
      "Cannot extend the file size to %" PRId64 " bytes (error: %s).", mmap_file->file_size, strerror(errno));
    err = -1;
  }
#else
  if ((mmap_file->access_flags & PROT_WRITE) && !mmap_file->is_memory_only) {
    /* Ensure modified pages are written to disk */
    /* This is important since not every munmap implementation flushes modified pages to disk
    (e.g.: https://nfs.sourceforge.net/#faq_d8) */
    int rc = msync(mmap_file->addr, mmap_file->file_size, MS_SYNC);
    if (rc < 0) {
      BLOSC_TRACE_ERROR("Cannot sync the memory-mapped file to disk (error: %s).", strerror(errno));
      err = -1;
    }
  }

  if (munmap(mmap_file->addr, mmap_file->mapping_size) < 0) {
    BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file (error: %s).", strerror(errno));
    err = -1;
  }
#endif
  /* Also closes the HANDLE on Windows */
  if (fclose(mmap_file->file) < 0) {
    BLOSC_TRACE_ERROR("Could not close the memory-mapped file.");
    err = -1;
  }

  free(mmap_file->urlpath);
  if (mmap_file->needs_free) {
    free(mmap_file);
  }

  return err;
}

Coverage Report

Created: 2025-08-24 06:50

Line	Count	Source (jump to first uncovered line)
1		/*********************************************************************
2		Blosc - Blocked Shuffling and Compression Library
3
4		Copyright (c) 2021 Blosc Development Team <blosc@blosc.org>
5		https://blosc.org
6		License: BSD 3-Clause (see LICENSE.txt)
7
8		See LICENSE.txt for details about copyright and rights to use.
9		**********************************************************************/
10
11		#if defined(__linux__)
12		/* Must be defined before anything else is included */
13		#define _GNU_SOURCE
14		#endif
15
16		#include "blosc2/blosc2-stdio.h"
17		#include "blosc2.h"
18
19		#include <stdlib.h>
20		#include <stdio.h>
21		#include <stdint.h>
22		#include <errno.h>
23		#include <inttypes.h>
24
25		#if defined(_WIN32)
26		#include <memoryapi.h>
27		// See https://github.com/Blosc/python-blosc2/issues/359
28		#define fseek _fseeki64
29		#define ftell _ftelli64
30		#else
31		#include <sys/mman.h>
32		#endif
33
34
35	0	void blosc2_stdio_open(const char urlpath, const char mode, void params) {
36	0	BLOSC_UNUSED_PARAM(params);
37	0	FILE *file = fopen(urlpath, mode);
38	0	if (file == NULL)
39	0	return NULL;
40	0	blosc2_stdio_file *my_fp = malloc(sizeof(blosc2_stdio_file));
41	0	my_fp->file = file;
42	0	return my_fp;
43	0	}
44
45	0	int blosc2_stdio_close(void *stream) {
46	0	blosc2_stdio_file my_fp = (blosc2_stdio_file ) stream;
47	0	int err = fclose(my_fp->file);
48	0	free(my_fp);
49	0	return err;
50	0	}
51
52	0	int64_t blosc2_stdio_size(void *stream) {
53	0	blosc2_stdio_file my_fp = (blosc2_stdio_file ) stream;
54
55	0	fseek(my_fp->file, 0, SEEK_END);
56	0	int64_t size = ftell(my_fp->file);
57	0	fseek(my_fp->file, 0, SEEK_SET);
58
59	0	return size;
60	0	}
61
62	0	int64_t blosc2_stdio_write(const void ptr, int64_t size, int64_t nitems, int64_t position, void stream) {
63	0	blosc2_stdio_file my_fp = (blosc2_stdio_file ) stream;
64	0	fseek(my_fp->file, position, SEEK_SET);
65
66	0	size_t nitems_ = fwrite(ptr, (size_t) size, (size_t) nitems, my_fp->file);
67	0	return (int64_t) nitems_;
68	0	}
69
70	0	int64_t blosc2_stdio_read(void *ptr, int64_t size, int64_t nitems, int64_t position, void stream) {
71	0	blosc2_stdio_file my_fp = (blosc2_stdio_file ) stream;
72	0	fseek(my_fp->file, position, SEEK_SET);
73
74	0	void* data_ptr = *ptr;
75	0	size_t nitems_ = fread(data_ptr, (size_t) size, (size_t) nitems, my_fp->file);
76	0	return (int64_t) nitems_;
77	0	}
78
79	0	int blosc2_stdio_truncate(void *stream, int64_t size) {
80	0	blosc2_stdio_file my_fp = (blosc2_stdio_file ) stream;
81	0	int rc;
82		#if defined(_MSC_VER)
83		rc = _chsize_s(_fileno(my_fp->file), size);
84		#else
85	0	rc = ftruncate(fileno(my_fp->file), size);
86	0	#endif
87	0	return rc;
88	0	}
89
90	2.29k	int blosc2_stdio_destroy(void* params) {
91	2.29k	BLOSC_UNUSED_PARAM(params);
92	2.29k	return 0;
93	2.29k	}
94
95		#if defined(_WIN32)
96		void _print_last_error() {
97		DWORD last_error = GetLastError();
98		if(last_error == 0) {
99		return;
100		}
101
102		LPSTR msg = NULL;
103		FormatMessage(
104		FORMAT_MESSAGE_ALLOCATE_BUFFER \| FORMAT_MESSAGE_FROM_SYSTEM \| FORMAT_MESSAGE_IGNORE_INSERTS,
105		NULL,
106		last_error,
107		MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
108		(LPSTR)&msg,
109		0,
110		NULL
111		);
112
113		printf("Message for the error %lu:\n%s\n", last_error, msg);
114		LocalFree(msg);
115		}
116		#endif
117
118	0	void blosc2_stdio_mmap_open(const char urlpath, const char mode, void params) {
119	0	BLOSC_UNUSED_PARAM(mode);
120
121	0	blosc2_stdio_mmap mmap_file = (blosc2_stdio_mmap ) params;
122	0	if (mmap_file->addr != NULL) {
123	0	if (strcmp(mmap_file->urlpath, urlpath) != 0) {
124	0	BLOSC_TRACE_ERROR(
125	0	"The memory-mapped file is already opened with the path %s and hence cannot be reopened with the path %s. This "
126	0	"happens if you try to open a sframe (sparse frame); please note that memory-mapped files are not supported "
127	0	"for sframes.",
128	0	mmap_file->urlpath,
129	0	urlpath
130	0	);
131	0	return NULL;
132	0	}
133
134		/* A memory-mapped file is only opened once */
135	0	return mmap_file;
136	0	}
137
138		// Keep the original path to ensure that all future file openings are with the same path
139	0	mmap_file->urlpath = malloc(strlen(urlpath) + 1);
140	0	strcpy(mmap_file->urlpath, urlpath);
141
142		/* mmap_file->mode mapping is similar to Numpy's memmap
143		(https://github.com/numpy/numpy/blob/main/numpy/_core/memmap.py) and CPython
144		(https://github.com/python/cpython/blob/main/Modules/mmapmodule.c) */
145		#if defined(_WIN32)
146		char* open_mode;
147		bool use_initial_mapping_size;
148		if (strcmp(mmap_file->mode, "r") == 0) {
149		mmap_file->access_flags = PAGE_READONLY;
150		mmap_file->map_flags = FILE_MAP_READ;
151		mmap_file->is_memory_only = false;
152		open_mode = "rb";
153		use_initial_mapping_size = false;
154		} else if (strcmp(mmap_file->mode, "r+") == 0) {
155		mmap_file->access_flags = PAGE_READWRITE;
156		mmap_file->map_flags = FILE_MAP_WRITE;
157		mmap_file->is_memory_only = false;
158		open_mode = "rb+";
159		use_initial_mapping_size = true;
160		} else if (strcmp(mmap_file->mode, "w+") == 0) {
161		mmap_file->access_flags = PAGE_READWRITE;
162		mmap_file->map_flags = FILE_MAP_WRITE;
163		mmap_file->is_memory_only = false;
164		open_mode = "wb+";
165		use_initial_mapping_size = true;
166		} else if (strcmp(mmap_file->mode, "c") == 0) {
167		mmap_file->access_flags = PAGE_WRITECOPY;
168		mmap_file->map_flags = FILE_MAP_COPY;
169		mmap_file->is_memory_only = true;
170		open_mode = "rb";
171		use_initial_mapping_size = false;
172		} else {
173		BLOSC_TRACE_ERROR("Mode %s not supported for memory-mapped files.", mmap_file->mode);
174		return NULL;
175		}
176		#else
177	0	char* open_mode;
178	0	bool use_initial_mapping_size;
179	0	if (strcmp(mmap_file->mode, "r") == 0) {
180	0	mmap_file->access_flags = PROT_READ;
181	0	mmap_file->map_flags = MAP_SHARED;
182	0	mmap_file->is_memory_only = false;
183	0	open_mode = "rb";
184	0	use_initial_mapping_size = false;
185	0	} else if (strcmp(mmap_file->mode, "r+") == 0) {
186	0	mmap_file->access_flags = PROT_READ \| PROT_WRITE;
187	0	mmap_file->map_flags = MAP_SHARED;
188	0	mmap_file->is_memory_only = false;
189	0	open_mode = "rb+";
190	0	use_initial_mapping_size = true;
191	0	} else if (strcmp(mmap_file->mode, "w+") == 0) {
192	0	mmap_file->access_flags = PROT_READ \| PROT_WRITE;
193	0	mmap_file->map_flags = MAP_SHARED;
194	0	mmap_file->is_memory_only = false;
195	0	open_mode = "wb+";
196	0	use_initial_mapping_size = true;
197	0	} else if (strcmp(mmap_file->mode, "c") == 0) {
198	0	mmap_file->access_flags = PROT_READ \| PROT_WRITE;
199	0	mmap_file->map_flags = MAP_PRIVATE;
200	0	mmap_file->is_memory_only = true;
201	0	open_mode = "rb";
202	0	use_initial_mapping_size = true;
203	0	} else {
204	0	BLOSC_TRACE_ERROR("Mode %s not supported for memory-mapped files.", mmap_file->mode);
205	0	return NULL;
206	0	}
207	0	#endif
208
209	0	mmap_file->file = fopen(urlpath, open_mode);
210	0	if (mmap_file->file == NULL) {
211	0	BLOSC_TRACE_ERROR("Cannot open the file %s with mode %s.", urlpath, open_mode);
212	0	return NULL;
213	0	}
214
215		/* Retrieve the size of the file */
216	0	fseek(mmap_file->file, 0, SEEK_END);
217	0	mmap_file->file_size = ftell(mmap_file->file);
218	0	fseek(mmap_file->file, 0, SEEK_SET);
219
220		/* The size of the mapping must be > 0 so we are using a large enough buffer for writing
221		(which will be increased later if needed) */
222	0	if (use_initial_mapping_size) {
223	0	mmap_file->mapping_size = mmap_file->initial_mapping_size;
224	0	}
225	0	else {
226	0	mmap_file->mapping_size = mmap_file->file_size;
227	0	}
228
229	0	if (mmap_file->file_size > mmap_file->mapping_size) {
230	0	mmap_file->mapping_size = mmap_file->file_size;
231	0	}
232
233		#if defined(_WIN32)
234		mmap_file->fd = _fileno(mmap_file->file);
235
236		/* Windows automatically expands the file size to the memory mapped file
237		(https://learn.microsoft.com/en-us/windows/win32/memory/creating-a-file-mapping-object).
238		In general, the size of the file is directly connected to the size of the mapping and cannot change. We cut the
239		file size to the target size in the end after we close the mapping */
240		HANDLE file_handle = (HANDLE) _get_osfhandle(mmap_file->fd);
241		DWORD size_hi = (DWORD)(mmap_file->mapping_size >> 32);
242		DWORD size_lo = (DWORD)(mmap_file->mapping_size & 0xFFFFFFFF);
243		mmap_file->mmap_handle = CreateFileMapping(file_handle, NULL, mmap_file->access_flags, size_hi, size_lo, NULL);
244		if (mmap_file->mmap_handle == NULL) {
245		_print_last_error();
246		BLOSC_TRACE_ERROR("Creating the memory mapping failed for the file %s.", urlpath);
247		return NULL;
248		}
249
250		DWORD offset = 0;
251		mmap_file->addr = (char*) MapViewOfFile(
252		mmap_file->mmap_handle, mmap_file->map_flags, offset, offset, mmap_file->mapping_size);
253		if (mmap_file->addr == NULL) {
254		_print_last_error();
255		BLOSC_TRACE_ERROR("Memory mapping failed for the file %s.", urlpath);
256
257		if (!CloseHandle(mmap_file->mmap_handle)) {
258		_print_last_error();
259		BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
260		}
261
262		return NULL;
263		}
264		#else
265	0	mmap_file->fd = fileno(mmap_file->file);
266
267		/* Offset where the mapping should start */
268	0	int64_t offset = 0;
269	0	mmap_file->addr = mmap(
270	0	NULL, mmap_file->mapping_size, mmap_file->access_flags, mmap_file->map_flags, mmap_file->fd, offset);
271	0	if (mmap_file->addr == MAP_FAILED) {
272	0	BLOSC_TRACE_ERROR("Memory mapping failed for file %s (error: %s).", urlpath, strerror(errno));
273	0	return NULL;
274	0	}
275	0	#endif
276
277	0	BLOSC_INFO(
278	0	"Opened memory-mapped file %s in mode %s with an mapping size of %" PRId64 " bytes.",
279	0	mmap_file->urlpath,
280	0	mmap_file->mode,
281	0	mmap_file->mapping_size
282	0	);
283
284		/* The mmap_file->mode parameter is only available during the opening call and cannot be used in any of the other
285		I/O functions since this string is managed by the caller (e.g., from Python) and the memory of the string may not
286		be available anymore at a later point. */
287	0	mmap_file->mode = NULL;
288
289	0	return mmap_file;
290	0	}
291
292	0	int blosc2_stdio_mmap_close(void *stream) {
293	0	BLOSC_UNUSED_PARAM(stream);
294	0	return 0;
295	0	}
296
297	0	int64_t blosc2_stdio_mmap_size(void *stream) {
298	0	blosc2_stdio_mmap mmap_file = (blosc2_stdio_mmap ) stream;
299	0	return mmap_file->file_size;
300	0	}
301
302	0	int64_t blosc2_stdio_mmap_write(const void ptr, int64_t size, int64_t nitems, int64_t position, void stream) {
303	0	blosc2_stdio_mmap mmap_file = (blosc2_stdio_mmap ) stream;
304
305	0	if (position < 0) {
306	0	BLOSC_TRACE_ERROR("Cannot write to a negative position.");
307	0	return 0;
308	0	}
309
310	0	int64_t n_bytes = size * nitems;
311	0	if (n_bytes == 0) {
312	0	return 0;
313	0	}
314
315	0	int64_t position_end = position + n_bytes;
316	0	int64_t new_size = position_end > mmap_file->file_size ? position_end : mmap_file->file_size;
317
318		#if defined(_WIN32)
319		if (mmap_file->file_size < new_size) {
320		mmap_file->file_size = new_size;
321		}
322
323		if (mmap_file->mapping_size < mmap_file->file_size) {
324		mmap_file->mapping_size = mmap_file->file_size * 2;
325
326		/* We need to remap the file completely and cannot pass the previous used address on Windows */
327		if (!UnmapViewOfFile(mmap_file->addr)) {
328		_print_last_error();
329		BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file.");
330		return 0;
331		}
332		if (!CloseHandle(mmap_file->mmap_handle)) {
333		_print_last_error();
334		BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
335		return 0;
336		}
337
338		HANDLE file_handle = (HANDLE) _get_osfhandle(mmap_file->fd);
339		DWORD size_hi = (DWORD)(mmap_file->mapping_size >> 32);
340		DWORD size_lo = (DWORD)(mmap_file->mapping_size & 0xFFFFFFFF);
341		mmap_file->mmap_handle = CreateFileMapping(file_handle, NULL, mmap_file->access_flags, size_hi, size_lo, NULL);
342		if (mmap_file->mmap_handle == NULL) {
343		_print_last_error();
344		BLOSC_TRACE_ERROR("Cannot remapt the memory-mapped file.");
345		return 0;
346		}
347
348		DWORD offset = 0;
349		char* new_address = (char*) MapViewOfFile(
350		mmap_file->mmap_handle, mmap_file->map_flags, offset, offset, mmap_file->mapping_size);
351		if (new_address == NULL) {
352		_print_last_error();
353		BLOSC_TRACE_ERROR("Cannot remapt the memory-mapped file");
354
355		if (!CloseHandle(mmap_file->mmap_handle)) {
356		_print_last_error();
357		BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
358		}
359
360		return 0;
361		}
362
363		mmap_file->addr = new_address;
364		}
365		#else
366	0	if (mmap_file->file_size < new_size) {
367	0	mmap_file->file_size = new_size;
368
369	0	if (!mmap_file->is_memory_only) {
370	0	int rc = ftruncate(mmap_file->fd, new_size);
371	0	if (rc < 0) {
372	0	BLOSC_TRACE_ERROR("Cannot extend the file size to %" PRId64 " bytes (error: %s).", new_size, strerror(errno));
373	0	return 0;
374	0	}
375	0	}
376	0	}
377
378	0	if (mmap_file->mapping_size < mmap_file->file_size) {
379	0	mmap_file->mapping_size = mmap_file->file_size * 2;
380
381	0	#if defined(__linux__)
382	0	int64_t old_mapping_size = mmap_file->mapping_size;
383
384		/* mremap is the best option as it also ensures that the old data is still available in c mode. Unfortunately, it
385		is no POSIX standard and only available on Linux */
386	0	char* new_address = mremap(mmap_file->addr, old_mapping_size, mmap_file->mapping_size, MREMAP_MAYMOVE);
387		#else
388		if (mmap_file->is_memory_only) {
389		BLOSC_TRACE_ERROR("Remapping a memory-mapping in c mode is only possible on Linux."
390		"Please specify either a different mode or set initial_mapping_size to a large enough number.");
391		return 0;
392		}
393		/* Extend the current mapping with the help of MAP_FIXED */
394		int64_t offset = 0;
395		char* new_address = mmap(
396		mmap_file->addr,
397		mmap_file->mapping_size,
398		mmap_file->access_flags,
399		mmap_file->map_flags \| MAP_FIXED,
400		mmap_file->fd,
401		offset
402		);
403		#endif
404
405	0	if (new_address == MAP_FAILED) {
406	0	BLOSC_TRACE_ERROR("Cannot remap the memory-mapped file (error: %s).", strerror(errno));
407	0	if (munmap(mmap_file->addr, mmap_file->mapping_size) < 0) {
408	0	BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file (error: %s).", strerror(errno));
409	0	}
410
411	0	return 0;
412	0	}
413
414	0	mmap_file->addr = new_address;
415	0	}
416	0	#endif
417
418	0	memcpy(mmap_file->addr + position, ptr, n_bytes);
419	0	return nitems;
420	0	}
421
422	0	int64_t blosc2_stdio_mmap_read(void *ptr, int64_t size, int64_t nitems, int64_t position, void stream) {
423	0	blosc2_stdio_mmap mmap_file = (blosc2_stdio_mmap ) stream;
424
425	0	if (position < 0) {
426	0	BLOSC_TRACE_ERROR("Cannot read from a negative position.");
427	0	*ptr = NULL;
428	0	return 0;
429	0	}
430
431	0	if (position + size * nitems > mmap_file->file_size) {
432	0	BLOSC_TRACE_ERROR("Cannot read beyond the end of the memory-mapped file.");
433	0	*ptr = NULL;
434	0	return 0;
435	0	}
436
437	0	*ptr = mmap_file->addr + position;
438
439	0	return nitems;
440	0	}
441
442	0	int blosc2_stdio_mmap_truncate(void *stream, int64_t size) {
443	0	blosc2_stdio_mmap mmap_file = (blosc2_stdio_mmap ) stream;
444
445	0	if (mmap_file->file_size == size) {
446	0	return 0;
447	0	}
448
449	0	mmap_file->file_size = size;
450
451		/* No file operations in c mode */
452	0	if (mmap_file->is_memory_only) {
453	0	return 0;
454	0	}
455
456		#if defined(_WIN32)
457		/* On Windows, we can truncate the file only at the end after we released the mapping */
458		return 0;
459		#else
460	0	return ftruncate(mmap_file->fd, size);
461	0	#endif
462	0	}
463
464	0	int blosc2_stdio_mmap_destroy(void* params) {
465	0	blosc2_stdio_mmap mmap_file = (blosc2_stdio_mmap ) params;
466	0	int err = 0;
467
468		#if defined(_WIN32)
469		if (mmap_file->access_flags == PAGE_READWRITE) {
470		/* Ensure modified pages are written to disk */
471		if (!FlushViewOfFile(mmap_file->addr, mmap_file->file_size)) {
472		_print_last_error();
473		BLOSC_TRACE_ERROR("Cannot flush the memory-mapped view to disk.");
474		err = -1;
475		}
476		HANDLE file_handle = (HANDLE) _get_osfhandle(mmap_file->fd);
477		if (!FlushFileBuffers(file_handle)) {
478		_print_last_error();
479		BLOSC_TRACE_ERROR("Cannot flush the memory-mapped file to disk.");
480		err = -1;
481		}
482		}
483
484		if (!UnmapViewOfFile(mmap_file->addr)) {
485		_print_last_error();
486		BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file.");
487		err = -1;
488		}
489		if (!CloseHandle(mmap_file->mmap_handle)) {
490		_print_last_error();
491		BLOSC_TRACE_ERROR("Cannot close the handle to the memory-mapped file.");
492		err = -1;
493		}
494		int rc = _chsize_s(mmap_file->fd, mmap_file->file_size);
495		if (rc != 0) {
496		BLOSC_TRACE_ERROR(
497		"Cannot extend the file size to %" PRId64 " bytes (error: %s).", mmap_file->file_size, strerror(errno));
498		err = -1;
499		}
500		#else
501	0	if ((mmap_file->access_flags & PROT_WRITE) && !mmap_file->is_memory_only) {
502		/* Ensure modified pages are written to disk */
503		/* This is important since not every munmap implementation flushes modified pages to disk
504		(e.g.: https://nfs.sourceforge.net/#faq_d8) */
505	0	int rc = msync(mmap_file->addr, mmap_file->file_size, MS_SYNC);
506	0	if (rc < 0) {
507	0	BLOSC_TRACE_ERROR("Cannot sync the memory-mapped file to disk (error: %s).", strerror(errno));
508	0	err = -1;
509	0	}
510	0	}
511
512	0	if (munmap(mmap_file->addr, mmap_file->mapping_size) < 0) {
513	0	BLOSC_TRACE_ERROR("Cannot unmap the memory-mapped file (error: %s).", strerror(errno));
514	0	err = -1;
515	0	}
516	0	#endif
517		/* Also closes the HANDLE on Windows */
518	0	if (fclose(mmap_file->file) < 0) {
519	0	BLOSC_TRACE_ERROR("Could not close the memory-mapped file.");
520	0	err = -1;
521	0	}
522
523	0	free(mmap_file->urlpath);
524	0	if (mmap_file->needs_free) {
525	0	free(mmap_file);
526	0	}
527
528	0	return err;
529	0	}