#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)

#define NOMINMAX

#include <windows.h>

#include <fcntl.h> // `O_RDNLY`

#include <sys/mman.h> // `mmap`

#include <sys/stat.h> // `stat`

#include <sys/types.h>

#ifdef _MSC_VER

#include <BaseTsd.h>

typedef SSIZE_T ssize_t;

#include <limits.h> // `SSIZE_MAX`

#include <unistd.h> // `ssize_t`

#ifndef SSIZE_MAX

#define SSIZE_MAX (SIZE_MAX / 2)

#ifdef _POSIX_C_SOURCE

#undef _POSIX_C_SOURCE

#include <Python.h> // Core CPython interfaces

#include <errno.h> // `errno`

#include <stdio.h> // `fopen`

#include <stdlib.h> // `rand`, `srand`

#include <time.h> // `time`

#include <stringzilla/stringzilla.h>

struct ArrowSchema {

};

struct ArrowArray {

};

typedef struct PyAPI {

} PyAPI;

#pragma region Forward Declarations

static PyTypeObject FileType;

static PyTypeObject StrType;

static PyTypeObject StrsType;

static PyTypeObject SplitIteratorType;

static PyTypeObject Utf8SplitLinesIteratorType;

static PyTypeObject Utf8SplitWhitespaceIteratorType;

static PyTypeObject Utf8WordBoundaryIteratorType;

static PyTypeObject Utf8CaseInsensitiveFindIteratorType;

static PyTypeObject HasherType;

static PyTypeObject Sha256Type;

static sz_string_view_t temporary_memory = {NULL, 0};

typedef struct {

PyObject ob_base;

#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)

HANDLE file_handle;

HANDLE mapping_handle;

int file_descriptor;

sz_string_view_t memory;

} File;

typedef struct {

PyObject ob_base;

PyObject *parent;

sz_string_view_t memory;

} Str;

typedef struct {

PyObject ob_base;

PyObject *text_obj; //< For reference counting

PyObject *separator_obj; //< For reference counting

sz_string_view_t text;

sz_string_view_t separator;

sz_find_t finder;

/// @brief How many bytes to skip after each successful find.

/// Generally equal to `needle_length`, or 1 for character sets.

sz_size_t match_length;

/// @brief Should we include the separator in the resulting slices?

sz_bool_t include_match;

/// @brief Should we enumerate the slices in normal or reverse order?

sz_bool_t is_reverse;

/// @brief Upper limit for the number of splits to report. Monotonically decreases during iteration.

sz_size_t max_parts;

/// @brief Indicates that we've already reported the tail of the split, and should return NULL next.

sz_bool_t reached_tail;

} SplitIterator;

typedef struct {

PyObject ob_base;

PyObject *text_obj; //< For reference counting

sz_cptr_t start; //< Current position (start of current segment)

sz_cptr_t end; //< End of original text (immutable)

sz_size_t match_length; //< Length of current segment to yield

/// @brief Should we include the newline characters in the resulting slices?

sz_bool_t keepends;

/// @brief Should we skip empty segments (trailing, leading, consecutive)?

sz_bool_t skip_empty;

} Utf8SplitLinesIterator;

typedef struct {

PyObject ob_base;

PyObject *text_obj; //< For reference counting

sz_cptr_t start; //< Current position in text

sz_cptr_t end; //< End of text (immutable)

sz_size_t match_length; //< Length of current segment to yield

/// @brief Should we skip empty segments (trailing, leading, consecutive)?

sz_bool_t skip_empty;

} Utf8SplitWhitespaceIterator;

typedef struct {

PyObject ob_base;

PyObject *text_obj; //< For reference counting

sz_cptr_t start; //< Start of current word

sz_cptr_t end; //< End of original text (immutable)

sz_cptr_t text_start; //< Start of original text (for reverse iteration)

/// @brief Should we skip empty segments (consecutive boundaries)?

sz_bool_t skip_empty;

} Utf8WordBoundaryIterator;

typedef struct {

PyObject ob_base;

PyObject *haystack_obj; //< Reference for garbage collection

PyObject *needle_obj; //< Reference for garbage collection (needle bytes must remain valid)

sz_cptr_t current; //< Current search position in haystack

sz_cptr_t haystack_end; //< End boundary of haystack

sz_string_view_t needle; //< Needle view (bytes and length)

/// @brief Reusable metadata for repeated searches with the same needle.

sz_utf8_case_insensitive_needle_metadata_t metadata;

/// @brief Whether to allow overlapping matches.

sz_bool_t include_overlapping;

} Utf8CaseInsensitiveFindIterator;

typedef struct {

PyObject ob_base;

enum {

STRS_U32_TAPE_VIEW = 0,

STRS_U64_TAPE_VIEW = 1,

STRS_U32_TAPE = 2,

STRS_U64_TAPE = 3,

STRS_FRAGMENTED = 4,

} layout;

union {

/**

struct u32_tape_view_t {

} u32_tape_view;

/**

struct u32_tape_t {

} u32_tape;

/**

struct u64_tape_view_t {

} u64_tape_view;

/**

struct u64_tape_t {

} u64_tape;

/**

struct fragmented_t {

} fragmented;

} data;

} Strs;

#pragma endregion

#pragma region Helpers

static sz_ptr_t temporary_memory_allocate(sz_size_t size, sz_string_view_t *existing) {

if (existing->length < size) {

sz_cptr_t new_start = realloc(existing->start, size);

if (!new_start) {

PyErr_Format(PyExc_MemoryError, "Unable to allocate memory for the Levenshtein matrix");

return NULL;

}

existing->start = new_start;

existing->length = size;

}

return existing->start;

}

static void temporary_memory_free(sz_ptr_t start, sz_size_t size, sz_string_view_t *existing) {}

static sz_cptr_t Strs_get_start_(void const *handle, sz_size_t i) {

Strs *strs = (Strs *)handle;

switch (strs->layout) {

case STRS_U32_TAPE: return strs->data.u32_tape.data + strs->data.u32_tape.offsets[i];

case STRS_U32_TAPE_VIEW: return strs->data.u32_tape_view.data + strs->data.u32_tape_view.offsets[i];

case STRS_U64_TAPE: return strs->data.u64_tape.data + strs->data.u64_tape.offsets[i];

case STRS_U64_TAPE_VIEW: return strs->data.u64_tape_view.data + strs->data.u64_tape_view.offsets[i];

case STRS_FRAGMENTED: return strs->data.fragmented.spans[i].start;

}

return NULL;

}

static sz_size_t Strs_get_length_(void const *handle, sz_size_t i) {

Strs *strs = (Strs *)handle;

switch (strs->layout) {

case STRS_U32_TAPE: return strs->data.u32_tape.offsets[i + 1] - strs->data.u32_tape.offsets[i];

case STRS_U32_TAPE_VIEW: return strs->data.u32_tape_view.offsets[i + 1] - strs->data.u32_tape_view.offsets[i];

case STRS_U64_TAPE: return strs->data.u64_tape.offsets[i + 1] - strs->data.u64_tape.offsets[i];

case STRS_U64_TAPE_VIEW: return strs->data.u64_tape_view.offsets[i + 1] - strs->data.u64_tape_view.offsets[i];

case STRS_FRAGMENTED: return strs->data.fragmented.spans[i].length;

}

return 0;

}

void reverse_offsets(sz_sorted_idx_t *array, sz_size_t length) {

sz_size_t i, j;

// Swap array[i] and array[j]

for (i = 0, j = length - 1; i < j; i++, j--) {

sz_sorted_idx_t temp = array[i];

array[i] = array[j];

array[j] = temp;

}

}

void reverse_haystacks(sz_string_view_t *array, sz_size_t length) {

sz_size_t i, j;

// Swap array[i] and array[j]

for (i = 0, j = length - 1; i < j; i++, j--) {

sz_string_view_t temp = array[i];

array[i] = array[j];

array[j] = temp;

}

}

void permute(sz_string_view_t *array, sz_sorted_idx_t *order, sz_size_t length) {

for (sz_size_t i = 0; i < length; ++i) {

if (i == order[i]) continue;

sz_string_view_t temp = array[i];

sz_size_t k = i, j;

while (i != (j = (sz_size_t)order[k])) {

array[k] = array[j];

order[k] = k;

k = j;

}

array[k] = temp;

order[k] = k;

}

}

SZ_INTERNAL sz_bool_t sz_py_is_mutable(PyObject *object) {

if (PyUnicode_Check(object)) {

PyErr_SetString(PyExc_TypeError, "str objects are immutable (use bytearray instead)");

return sz_false_k;

}

else if (PyBytes_Check(object)) {

PyErr_SetString(PyExc_TypeError, "bytes objects are immutable (use bytearray instead)");

return sz_false_k;

}

else if (PyMemoryView_Check(object)) {

Py_buffer *view = PyMemoryView_GET_BUFFER(object);

if (view->readonly) {

PyErr_SetString(PyExc_TypeError, "memoryview is read-only");

return sz_false_k;

}

}

// Everything else is optimistically considered mutable

return sz_true_k;

}

SZ_DYNAMIC sz_bool_t sz_py_export_string_like(PyObject *object, sz_cptr_t *start, sz_size_t *length) {

if (PyUnicode_Check(object)) {

// Handle Python `str` object

Py_ssize_t signed_length;

*start = PyUnicode_AsUTF8AndSize(object, &signed_length);

*length = (sz_size_t)signed_length;

return 1;

}

else if (PyBytes_Check(object)) {

// Handle Python `bytes` object

// https://docs.python.org/3/c-api/bytes.html

Py_ssize_t signed_length;

if (PyBytes_AsStringAndSize(object, (sz_ptr_t *)start, &signed_length) == -1) {

PyErr_SetString(PyExc_ValueError, "Couldn't access `bytes` buffer internals");

return 0;

}

*length = (sz_size_t)signed_length;

return 1;

}

else if (PyByteArray_Check(object)) {

// Handle Python mutable `bytearray` object

// https://docs.python.org/3/c-api/bytearray.html

*start = PyByteArray_AS_STRING(object);

*length = PyByteArray_GET_SIZE(object);

return 1;

}

else if (PyObject_TypeCheck(object, &StrType)) {

Str *str = (Str *)object;

*start = str->memory.start;

*length = str->memory.length;

return 1;

}

else if (PyObject_TypeCheck(object, &FileType)) {

File *file = (File *)object;

*start = file->memory.start;

*length = file->memory.length;

return 1;

}

else if (PyMemoryView_Check(object)) {

// Handle Python `memoryview` object

// https://docs.python.org/3/c-api/memoryview.html

// https://docs.python.org/3/c-api/buffer.html#c.Py_buffer

Py_buffer *view = PyMemoryView_GET_BUFFER(object);

// Make sure we are dealing with single-byte integral representations

if (view->itemsize != 1) {

PyErr_SetString(PyExc_ValueError, "Only single-byte integral types are supported");

return 0;

}

// Let's make sure the data is contiguous.

// This can be a bit trickier for high-dimensional arrays, but CPython has a built-in function for that.

// The flag 'C' stands for C-style-contiguous, which means that the last dimension is contiguous.

// The flag 'F' stands for Fortran-style-contiguous, which means that the first dimension is contiguous.

// The flag 'A' stands for any-contiguous, which only means there are no gaps between elements.

// For byte-level processing that's all we need.

if (!PyBuffer_IsContiguous(view, 'A')) {

PyErr_SetString(PyExc_ValueError, "The array must be contiguous");

return 0;

}

*start = (sz_cptr_t)view->buf;

*length = (sz_size_t)view->len;

return 1;

}

else {

PyErr_SetString(PyExc_TypeError, "Unsupported argument layout");

return 0;

}

}

sz_cptr_t sz_py_strs_sequence_member_start_if_fragmented(void const *sequence_punned, sz_size_t index) {

Strs *strs = (Strs *)sequence_punned;

sz_assert_(strs->layout == STRS_FRAGMENTED && "Expected a reordered Strs layout");

if (index < 0 || index >= strs->data.fragmented.count) {

PyErr_SetString(PyExc_IndexError, "Index out of bounds");

return NULL;

}

return strs->data.fragmented.spans[index].start;

}

sz_size_t sz_py_strs_sequence_member_length_if_fragmented(void const *sequence_punned, sz_size_t index) {

Strs *strs = (Strs *)sequence_punned;

sz_assert_(strs->layout == STRS_FRAGMENTED && "Expected a reordered Strs layout");

if (index < 0 || index >= strs->data.fragmented.count) {

PyErr_SetString(PyExc_IndexError, "Index out of bounds");

return 0;

}

return strs->data.fragmented.spans[index].length;

}

SZ_DYNAMIC sz_bool_t sz_py_export_strings_as_sequence(PyObject *object, sz_sequence_t *sequence) {

if (!sequence) return sz_false_k;

if (PyObject_TypeCheck(object, &StrsType)) {

Strs *strs = (Strs *)object;

sz_assert_(strs->layout == STRS_FRAGMENTED && "View as tapes!");

sequence->handle = strs;

sequence->count = strs->data.fragmented.count;

sequence->get_start = sz_py_strs_sequence_member_start_if_fragmented;

sequence->get_length = sz_py_strs_sequence_member_length_if_fragmented;

return sz_true_k;

}

return sz_false_k;

}

SZ_DYNAMIC sz_bool_t sz_py_export_strings_as_u32tape(PyObject *object, sz_cptr_t *data, sz_u32_t const **offsets,

sz_size_t *count) {

if (!data || !offsets || !count) return sz_false_k;

if (!PyObject_TypeCheck(object, &StrsType)) return sz_false_k;

Strs *strs = (Strs *)object;

if (strs->layout == STRS_U32_TAPE) {

*data = strs->data.u32_tape.data;

*offsets = strs->data.u32_tape.offsets;

*count = strs->data.u32_tape.count;

return sz_true_k;

}

else if (strs->layout == STRS_U32_TAPE_VIEW) {

*data = strs->data.u32_tape_view.data;

*offsets = strs->data.u32_tape_view.offsets;

*count = strs->data.u32_tape_view.count;

return sz_true_k;

}

else { return sz_false_k; }

}

SZ_DYNAMIC sz_bool_t sz_py_export_strings_as_u64tape(PyObject *object, sz_cptr_t *data, sz_u64_t const **offsets,

sz_size_t *count) {

if (!data || !offsets || !count) return sz_false_k;

if (!PyObject_TypeCheck(object, &StrsType)) return sz_false_k;

Strs *strs = (Strs *)object;

if (strs->layout == STRS_U64_TAPE) {

*data = strs->data.u64_tape.data;

*offsets = strs->data.u64_tape.offsets;

*count = strs->data.u64_tape.count;

return sz_true_k;

}

else if (strs->layout == STRS_U64_TAPE_VIEW) {

*data = strs->data.u64_tape_view.data;

*offsets = strs->data.u64_tape_view.offsets;

*count = strs->data.u64_tape_view.count;

return sz_true_k;

}

else { return sz_false_k; }

}

static sz_bool_t sz_py_replace_u32_tape_allocator(Strs *strs, sz_memory_allocator_t *old_allocator,

sz_memory_allocator_t *allocator) {

struct u32_tape_t *data = &strs->data.u32_tape;

sz_assert_(data->offsets && "Expected offsets to be allocated");

sz_size_t const string_data_size = (sz_size_t)data->offsets[data->count];

sz_size_t const offsets_size = (data->count + 1) * sizeof(sz_u32_t);

// Allocate new string data with new allocator

sz_ptr_t new_string_data =

string_data_size ? (sz_ptr_t)allocator->allocate(string_data_size, allocator->handle) : (sz_ptr_t)NULL;

if (string_data_size && !new_string_data) return sz_false_k;

memcpy(new_string_data, data->data, string_data_size);

// Allocate new offsets array

sz_u32_t *new_offsets =

offsets_size ? (sz_u32_t *)allocator->allocate(offsets_size, allocator->handle) : (sz_u32_t *)NULL;

if (offsets_size && !new_offsets) {

if (string_data_size) allocator->free(new_string_data, string_data_size, allocator->handle);

return sz_false_k;

}

memcpy(new_offsets, data->offsets, offsets_size);

// Free old memory with old allocator (tapes always own their data)

old_allocator->free(data->data, string_data_size, old_allocator->handle);

old_allocator->free(data->offsets, offsets_size, old_allocator->handle);

// Update pointers and allocator

data->data = new_string_data;

data->offsets = new_offsets;

data->allocator = *allocator;

return sz_true_k;

}

static sz_bool_t sz_py_replace_u64_tape_allocator(Strs *strs, sz_memory_allocator_t *old_allocator,

sz_memory_allocator_t *allocator) {

struct u64_tape_t *data = &strs->data.u64_tape;

sz_assert_(data->offsets && "Expected offsets to be allocated");

sz_size_t string_data_size = (sz_size_t)data->offsets[data->count];

sz_size_t offsets_size = (data->count + 1) * sizeof(sz_u64_t);

// Allocate new string data with new allocator

sz_ptr_t new_string_data =

string_data_size ? (sz_ptr_t)allocator->allocate(string_data_size, allocator->handle) : (sz_ptr_t)NULL;

if (string_data_size && !new_string_data) return sz_false_k;

memcpy(new_string_data, data->data, string_data_size);

// Allocate new offsets array

sz_u64_t *new_offsets =

offsets_size ? (sz_u64_t *)allocator->allocate(offsets_size, allocator->handle) : (sz_u64_t *)NULL;

if (offsets_size && !new_offsets) {

if (string_data_size) allocator->free(new_string_data, string_data_size, allocator->handle);

return sz_false_k;

}

memcpy(new_offsets, data->offsets, offsets_size);

// Free old memory with old allocator (tapes always own their data)

old_allocator->free(data->data, string_data_size, old_allocator->handle);

old_allocator->free(data->offsets, offsets_size, old_allocator->handle);

// Update pointers and allocator

data->data = new_string_data;

data->offsets = new_offsets;

data->allocator = *allocator;

return sz_true_k;

}

static sz_bool_t sz_py_replace_u32_tape_view_allocator(Strs *strs, sz_memory_allocator_t *allocator) {

// Convert view to tape by copying the data

struct u32_tape_view_t *view = &strs->data.u32_tape_view;

sz_u32_t const slice_start_offset = view->offsets[0];

sz_size_t const string_data_size = (sz_size_t)(view->offsets[view->count] - slice_start_offset);

sz_size_t const offsets_size = (view->count + 1) * sizeof(sz_u32_t);

// Allocate new string data with new allocator

sz_ptr_t new_string_data = NULL;

if (string_data_size > 0) {

new_string_data = (sz_ptr_t)allocator->allocate(string_data_size, allocator->handle);

if (!new_string_data) return sz_false_k;

memcpy(new_string_data, view->data + slice_start_offset, string_data_size);

}

// Allocate new offsets array and adjust to be relative to slice start

sz_u32_t *new_offsets = NULL;

if (offsets_size > 0) {

new_offsets = (sz_u32_t *)allocator->allocate(offsets_size, allocator->handle);

if (!new_offsets) {

if (string_data_size > 0) allocator->free(new_string_data, string_data_size, allocator->handle);

return sz_false_k;

}

for (sz_size_t i = 0; i <= view->count; ++i) new_offsets[i] = view->offsets[i] - slice_start_offset;

}

// Release parent reference if any

Py_XDECREF(view->parent);

// Convert to tape layout

strs->layout = STRS_U32_TAPE;

strs->data.u32_tape.count = view->count;

strs->data.u32_tape.data = new_string_data;

strs->data.u32_tape.offsets = new_offsets;

strs->data.u32_tape.allocator = *allocator;

return sz_true_k;

}

static sz_bool_t sz_py_replace_u64_tape_view_allocator(Strs *strs, sz_memory_allocator_t *allocator) {

// Convert view to tape by copying the data

struct u64_tape_view_t *view = &strs->data.u64_tape_view;

sz_u64_t const slice_start_offset = view->offsets[0];

sz_size_t const string_data_size = (sz_size_t)(view->offsets[view->count] - slice_start_offset);

sz_size_t const offsets_size = (view->count + 1) * sizeof(sz_u64_t);

// Allocate new string data with new allocator

sz_ptr_t new_string_data = NULL;

if (string_data_size > 0) {

new_string_data = (sz_ptr_t)allocator->allocate(string_data_size, allocator->handle);

if (!new_string_data) return sz_false_k;

memcpy(new_string_data, view->data + slice_start_offset, string_data_size);

}

// Allocate new offsets array and adjust to be relative to slice start

sz_u64_t *new_offsets = NULL;

if (offsets_size > 0) {

new_offsets = (sz_u64_t *)allocator->allocate(offsets_size, allocator->handle);

if (!new_offsets) {

if (string_data_size > 0) allocator->free(new_string_data, string_data_size, allocator->handle);

return sz_false_k;

}

for (sz_size_t i = 0; i <= view->count; ++i) new_offsets[i] = view->offsets[i] - slice_start_offset;

}

// Release parent reference if any

Py_XDECREF(view->parent);

// Convert to tape layout

strs->layout = STRS_U64_TAPE;

strs->data.u64_tape.count = view->count;

strs->data.u64_tape.data = new_string_data;

strs->data.u64_tape.offsets = new_offsets;

strs->data.u64_tape.allocator = *allocator;

return sz_true_k;

}

static sz_bool_t sz_py_replace_fragmented_allocator(Strs *strs, sz_memory_allocator_t *old_allocator,

sz_memory_allocator_t *allocator) {

struct fragmented_t *fragmented = &strs->data.fragmented;

sz_assert_(fragmented->spans && "Expected spans to be allocated");

// Calculate total size needed for consolidated tape

sz_size_t total_bytes = 0;

for (sz_size_t i = 0; i < fragmented->count; i++) total_bytes += fragmented->spans[i].length;

// Choose 32-bit or 64-bit tape based on size

sz_bool_t use_64bit = total_bytes >= UINT32_MAX;

// Skip allocation if there's no data to allocate (empty strings case)

if (total_bytes == 0) {

// Convert to empty tape layout

old_allocator->free(fragmented->spans, fragmented->count * sizeof(sz_string_view_t), old_allocator->handle);

Py_XDECREF(fragmented->parent);

strs->layout = STRS_U32_TAPE;

strs->data.u32_tape.count = fragmented->count;

strs->data.u32_tape.data = NULL;

strs->data.u32_tape.offsets = NULL;

strs->data.u32_tape.allocator = *allocator;

return sz_true_k;

}

// Allocate consolidated data buffer and offsets array

sz_ptr_t new_data = (sz_ptr_t)allocator->allocate(total_bytes, allocator->handle);

if (!new_data) return sz_false_k;

if (use_64bit) {

sz_u64_t *new_offsets =

(sz_u64_t *)allocator->allocate((fragmented->count + 1) * sizeof(sz_u64_t), allocator->handle);

if (!new_offsets) {

allocator->free(new_data, total_bytes, allocator->handle);

return sz_false_k;

}

// Copy fragmented data into consolidated buffer

sz_size_t current_offset = 0;

new_offsets[0] = 0;

for (sz_size_t i = 0; i < fragmented->count; i++) {

sz_size_t len = fragmented->spans[i].length;

if (len > 0) { memcpy(new_data + current_offset, fragmented->spans[i].start, len); }

current_offset += len;

new_offsets[i + 1] = current_offset;

}

// Free old fragmented data and convert to 64-bit tape

old_allocator->free(fragmented->spans, fragmented->count * sizeof(sz_string_view_t), old_allocator->handle);

Py_XDECREF(fragmented->parent);

strs->layout = STRS_U64_TAPE;

strs->data.u64_tape.count = fragmented->count;

strs->data.u64_tape.data = new_data;

strs->data.u64_tape.offsets = new_offsets;

strs->data.u64_tape.allocator = *allocator;

}

else {

sz_u32_t *new_offsets =

(sz_u32_t *)allocator->allocate((fragmented->count + 1) * sizeof(sz_u32_t), allocator->handle);

if (!new_offsets) {

allocator->free(new_data, total_bytes, allocator->handle);

return sz_false_k;

}

// Copy fragmented data into consolidated buffer

sz_size_t current_offset = 0;

new_offsets[0] = 0;

for (sz_size_t i = 0; i < fragmented->count; i++) {

sz_size_t len = fragmented->spans[i].length;

if (len > 0) { memcpy(new_data + current_offset, fragmented->spans[i].start, len); }

current_offset += len;

// Ensure we don't overflow 32-bit offset

if (current_offset > UINT32_MAX) {

allocator->free(new_data, total_bytes, allocator->handle);

allocator->free(new_offsets, (fragmented->count + 1) * sizeof(sz_u32_t), allocator->handle);

return sz_false_k;

}

new_offsets[i + 1] = (sz_u32_t)current_offset;

}

// Free old fragmented data and convert to 32-bit tape

old_allocator->free(fragmented->spans, fragmented->count * sizeof(sz_string_view_t), old_allocator->handle);

Py_XDECREF(fragmented->parent);

strs->layout = STRS_U32_TAPE;

strs->data.u32_tape.count = fragmented->count;

strs->data.u32_tape.data = new_data;

strs->data.u32_tape.offsets = new_offsets;

strs->data.u32_tape.allocator = *allocator;

}

return sz_true_k;

}

SZ_DYNAMIC sz_bool_t sz_py_replace_strings_allocator(PyObject *object, sz_memory_allocator_t *allocator) {

if (!object || !allocator) return sz_false_k;

if (!PyObject_TypeCheck(object, &StrsType)) return sz_false_k;

Strs *strs = (Strs *)object;

// Get the current allocator based on layout

sz_memory_allocator_t old_allocator;

switch (strs->layout) {

case STRS_U32_TAPE: old_allocator = strs->data.u32_tape.allocator; break;

case STRS_U64_TAPE: old_allocator = strs->data.u64_tape.allocator; break;

case STRS_FRAGMENTED: old_allocator = strs->data.fragmented.allocator; break;

case STRS_U32_TAPE_VIEW:

case STRS_U64_TAPE_VIEW:

// Traverse parent chain until we find an allocator

{

Strs *up = strs;

while (up && (up->layout == STRS_U32_TAPE_VIEW || up->layout == STRS_U64_TAPE_VIEW)) {

PyObject *parent =

(up->layout == STRS_U32_TAPE_VIEW) ? up->data.u32_tape_view.parent : up->data.u64_tape_view.parent;

if (!parent || !PyObject_TypeCheck(parent, &StrsType)) break;

up = (Strs *)parent;

}

// Extract allocator from the owning layout we found

if (up && up->layout == STRS_U32_TAPE) { old_allocator = up->data.u32_tape.allocator; }

else if (up && up->layout == STRS_U64_TAPE) { old_allocator = up->data.u64_tape.allocator; }

else if (up && up->layout == STRS_FRAGMENTED) { old_allocator = up->data.fragmented.allocator; }

else { sz_memory_allocator_init_default(&old_allocator); } // Final fallback

}

break;

default: sz_memory_allocator_init_default(&old_allocator); break;

}

// Check if the allocators are the same - no need to reallocate

if (sz_memory_allocator_equal(&old_allocator, allocator)) return sz_true_k;

// Handle different Strs layouts using dedicated functions

switch (strs->layout) {

case STRS_U32_TAPE: return sz_py_replace_u32_tape_allocator(strs, &old_allocator, allocator);

case STRS_U64_TAPE: return sz_py_replace_u64_tape_allocator(strs, &old_allocator, allocator);

case STRS_U32_TAPE_VIEW: return sz_py_replace_u32_tape_view_allocator(strs, allocator);

case STRS_U64_TAPE_VIEW: return sz_py_replace_u64_tape_view_allocator(strs, allocator);

case STRS_FRAGMENTED: return sz_py_replace_fragmented_allocator(strs, &old_allocator, allocator);

}

return sz_false_k; // Should never reach here

}

void wrap_current_exception(sz_cptr_t comment) {

// ? Prior to Python 3.12 we need to fetch and restore the exception state using

// ? `PyErr_Fetch` and `PyErr_Restore` to avoid overwriting the current exception.

// ? After Python 3.12 we can use `PyErr_GetRaisedException` and `PyErr_SetRaisedException`.

sz_unused_(comment);

}

typedef void (*get_string_at_offset_t)(Strs *, Py_ssize_t, Py_ssize_t, PyObject **, sz_cptr_t *, sz_size_t *);

void str_at_offset_u32_tape(Strs *strs, Py_ssize_t i, Py_ssize_t count, //

PyObject **memory_owner, sz_cptr_t *start, sz_size_t *length) {

// Apache Arrow format: offsets[i] to offsets[i+1] defines string i

sz_u32_t start_offset = strs->data.u32_tape.offsets[i];

sz_u32_t end_offset = strs->data.u32_tape.offsets[i + 1];

*start = strs->data.u32_tape.data + start_offset;

*length = end_offset - start_offset;

*memory_owner = strs; // Tapes own their data

}

void str_at_offset_u32_tape_view(Strs *strs, Py_ssize_t i, Py_ssize_t count, //

PyObject **memory_owner, sz_cptr_t *start, sz_size_t *length) {

// Apache Arrow format: offsets[i] to offsets[i+1] defines string i

sz_u32_t start_offset = strs->data.u32_tape_view.offsets[i];

sz_u32_t end_offset = strs->data.u32_tape_view.offsets[i + 1];

*start = strs->data.u32_tape_view.data + start_offset;

*length = end_offset - start_offset;

*memory_owner = strs->data.u32_tape_view.parent;

}

void str_at_offset_u64_tape(Strs *strs, Py_ssize_t i, Py_ssize_t count, //

PyObject **memory_owner, sz_cptr_t *start, sz_size_t *length) {

// Apache Arrow format: offsets[i] to offsets[i+1] defines string i

sz_u64_t start_offset = strs->data.u64_tape.offsets[i];

sz_u64_t end_offset = strs->data.u64_tape.offsets[i + 1];

*start = strs->data.u64_tape.data + start_offset;

*length = end_offset - start_offset;

*memory_owner = strs; // Tapes own their data

}

void str_at_offset_u64_tape_view(Strs *strs, Py_ssize_t i, Py_ssize_t count, //

PyObject **memory_owner, sz_cptr_t *start, sz_size_t *length) {

// Apache Arrow format: offsets[i] to offsets[i+1] defines string i

sz_u64_t start_offset = strs->data.u64_tape_view.offsets[i];

sz_u64_t end_offset = strs->data.u64_tape_view.offsets[i + 1];

*start = strs->data.u64_tape_view.data + start_offset;

*length = end_offset - start_offset;

*memory_owner = strs->data.u64_tape_view.parent;

}

void str_at_offset_fragmented(Strs *strs, Py_ssize_t i, Py_ssize_t count, //

PyObject **memory_owner, sz_cptr_t *start, sz_size_t *length) {

*start = strs->data.fragmented.spans[i].start;

*length = strs->data.fragmented.spans[i].length;

*memory_owner = strs->data.fragmented.parent;

}

get_string_at_offset_t str_at_offset_getter(Strs *strs) {

switch (strs->layout) {

case STRS_U32_TAPE: return str_at_offset_u32_tape;

case STRS_U32_TAPE_VIEW: return str_at_offset_u32_tape_view;

case STRS_U64_TAPE: return str_at_offset_u64_tape;

case STRS_U64_TAPE_VIEW: return str_at_offset_u64_tape_view;

case STRS_FRAGMENTED: return str_at_offset_fragmented;

default:

// Unsupported layout

PyErr_SetString(PyExc_TypeError, "Unsupported layout for conversion");

return NULL;

}

}

#pragma endregion

#pragma region Memory Mapping File

static void File_dealloc(File *self) {

#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)

if (self->memory.start) {