/src/readstat/src/sas/readstat_sas7bcat_read.c

Source
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <math.h>
#include "readstat_sas.h"
#include "../readstat_iconv.h"
#include "../readstat_convert.h"
#include "../readstat_malloc.h"

#define SAS_CATALOG_FIRST_INDEX_PAGE 1
#define SAS_CATALOG_USELESS_PAGES    3

typedef struct sas7bcat_ctx_s {
    readstat_metadata_handler      metadata_handler;
    readstat_value_label_handler   value_label_handler;
    void          *user_ctx;
    readstat_io_t *io;
    int            u64;
    int            pad1;
    int            bswap;
    int64_t        xlsr_size;
    int64_t        xlsr_offset;
    int64_t        xlsr_O_offset;
    int64_t        page_count;
    int64_t        page_size;
    int64_t        header_size;
    uint64_t      *block_pointers;
    int            block_pointers_used;
    int            block_pointers_capacity;
    const char    *input_encoding;
    const char    *output_encoding;
    iconv_t        converter;
} sas7bcat_ctx_t;

static void sas7bcat_ctx_free(sas7bcat_ctx_t *ctx) {
    if (ctx->converter)
        iconv_close(ctx->converter);
    if (ctx->block_pointers)
        free(ctx->block_pointers);

    free(ctx);
}

static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, size_t value_labels_len, 
        int label_count_used, int label_count_capacity, const char *name, sas7bcat_ctx_t *ctx) {
    readstat_error_t retval = READSTAT_OK;
    int i;
    const char *lbp1 = value_start;
    uint32_t *value_offset = readstat_calloc(label_count_used, sizeof(uint32_t));
    /* Doubles appear to be stored as big-endian, always */
    int bswap_doubles = machine_is_little_endian();
    int is_string = (name[0] == '$');
    char *label = NULL;

    if (value_offset == NULL) {
        retval = READSTAT_ERROR_MALLOC;
        goto cleanup;
    }

    /* Pass 1 -- find out the offset of the labels */
    for (i=0; i<label_count_capacity; i++) {
        if (&lbp1[4] - value_start > value_labels_len || sas_read2(&lbp1[2], ctx->bswap) < 0) {
            retval = READSTAT_ERROR_PARSE;
            goto cleanup;
        }
        if (i<label_count_used) {
            if (&lbp1[10+ctx->pad1+4] - value_start > value_labels_len) {
                retval = READSTAT_ERROR_PARSE;
                goto cleanup;
            }
            uint32_t label_pos = sas_read4(&lbp1[10+ctx->pad1], ctx->bswap);
            if (label_pos >= label_count_used) {
                retval = READSTAT_ERROR_PARSE;
                goto cleanup;
            }
            value_offset[label_pos] = lbp1 - value_start;
        }
        lbp1 += 6 + sas_read2(&lbp1[2], ctx->bswap);
    }

    const char *lbp2 = lbp1;

    /* Pass 2 -- parse pairs of values & labels */
    for (i=0; i<label_count_used && i<label_count_capacity; i++) {
        lbp1 = value_start + value_offset[i];

        if (&lbp1[30] - value_start > value_labels_len ||
                &lbp2[10] - value_start > value_labels_len) {
            retval = READSTAT_ERROR_PARSE;
            goto cleanup;
        }
        readstat_value_t value = { .type = is_string ? READSTAT_TYPE_STRING : READSTAT_TYPE_DOUBLE };
        char string_val[4*16+1];
        if (is_string) {
            size_t value_entry_len = 6 + sas_read2(&lbp1[2], ctx->bswap);
            retval = readstat_convert(string_val, sizeof(string_val),
                    &lbp1[value_entry_len-16], 16, ctx->converter);
            if (retval != READSTAT_OK)
                goto cleanup;

            value.v.string_value = string_val;
        } else {
            uint64_t val = sas_read8(&lbp1[22], bswap_doubles);
            double dval = NAN;
            if ((val | 0xFF0000000000) == 0xFFFFFFFFFFFF) {
                sas_assign_tag(&value, (val >> 40));
            } else {
                memcpy(&dval, &val, 8);
                if (dval > 0.0) {
                    val = ~val;
                    memcpy(&dval, &val, 8);
                } else {
                    dval *= -1.0;
                }
            }

            value.v.double_value = dval;
        }
        size_t label_len = sas_read2(&lbp2[8], ctx->bswap);
        if (&lbp2[10] > value_start + value_labels_len) {
            retval = READSTAT_ERROR_PARSE;
            goto cleanup;
        }
        /* Some labels seem to overflow the reported block length, truncate it */
        /* (Observed with formats.sasbcat from GSS2021, produced with 9.0401M6X64_SR12R2 */
        if (label_len > value_start + value_labels_len - &lbp2[10]) {
            label_len = value_start + value_labels_len - &lbp2[10];
        }
        if (ctx->value_label_handler) {
            label = realloc(label, 4 * label_len + 1);
            retval = readstat_convert(label, 4 * label_len + 1,
                    &lbp2[10], label_len, ctx->converter);
            if (retval != READSTAT_OK)
                goto cleanup;

            if (ctx->value_label_handler(name, value, label, ctx->user_ctx) != READSTAT_HANDLER_OK) {
                retval = READSTAT_ERROR_USER_ABORT;
                goto cleanup;
            }
        }

        lbp2 += 8 + 2 + label_len + 1;
    }

cleanup:
    free(label);
    free(value_offset);
    return retval;
}

static readstat_error_t sas7bcat_parse_block(const char *data, size_t data_size, sas7bcat_ctx_t *ctx) {
    readstat_error_t retval = READSTAT_OK;

    size_t pad = 0;
    uint64_t label_count_capacity = 0;
    uint64_t label_count_used = 0;
    int payload_offset = 106;
    uint16_t flags = 0;
    char name[4*32+1];

    if (data_size < payload_offset)
        goto cleanup;

    flags = sas_read2(&data[2], ctx->bswap);
    pad = (flags & 0x08) ? 4 : 0; // might be 0x10, not sure
    if (ctx->u64) {
        label_count_capacity = sas_read8(&data[42+pad], ctx->bswap);
        label_count_used = sas_read8(&data[50+pad], ctx->bswap);

        payload_offset += 32;
    } else {
        label_count_capacity = sas_read4(&data[38+pad], ctx->bswap);
        label_count_used = sas_read4(&data[42+pad], ctx->bswap);
    }

    if ((retval = readstat_convert(name, sizeof(name), &data[8], 8, ctx->converter)) != READSTAT_OK)
        goto cleanup;

    if (pad) {
        pad += 16;
    }

    if (((flags & 0x80) && !ctx->u64) || ((flags & 0x20) && ctx->u64)) { // has long name
        if (data_size < payload_offset + pad + 32)
            goto cleanup;

        retval = readstat_convert(name, sizeof(name), &data[payload_offset+pad], 32, ctx->converter);
        if (retval != READSTAT_OK)
            goto cleanup;
        pad += 32;
    }

    if (data_size < payload_offset + pad)
        goto cleanup;

    if (label_count_used == 0)
        goto cleanup;

    if ((retval = sas7bcat_parse_value_labels(&data[payload_offset+pad], data_size - payload_offset - pad,
                    label_count_used, label_count_capacity, name, ctx)) != READSTAT_OK)
        goto cleanup;

cleanup:
    return retval;
}

static readstat_error_t sas7bcat_augment_index(const char *index, size_t len, sas7bcat_ctx_t *ctx) {
    const char *xlsr = index;
    readstat_error_t retval = READSTAT_OK;
    while (xlsr + ctx->xlsr_size <= index + len) {
        if (memcmp(xlsr, "XLSR", 4) != 0) // some block pointers seem to have 8 bytes of extra padding
            xlsr += 8;
        if (memcmp(xlsr, "XLSR", 4) != 0)
            break;

        if (xlsr[ctx->xlsr_O_offset] == 'O') {
            uint64_t page = 0, pos = 0;
            if (ctx->u64) {
                page = sas_read8(&xlsr[8], ctx->bswap);
                pos = sas_read2(&xlsr[16], ctx->bswap);
            } else {
                page = sas_read4(&xlsr[4], ctx->bswap);
                pos = sas_read2(&xlsr[8], ctx->bswap);
            }
            ctx->block_pointers[ctx->block_pointers_used++] = (page << 32) + pos;
        }

        if (ctx->block_pointers_used == ctx->block_pointers_capacity) {
            ctx->block_pointers = readstat_realloc(ctx->block_pointers, (ctx->block_pointers_capacity *= 2) * sizeof(uint64_t));
            if (ctx->block_pointers == NULL) {
                retval = READSTAT_ERROR_MALLOC;
                goto cleanup;
            }
        }

        xlsr += ctx->xlsr_size;
    }
cleanup:
    return retval;
}

static int compare_block_pointers(const void *elem1, const void *elem2) {
    uint64_t v1 = *(const uint64_t *)elem1;
    uint64_t v2 = *(const uint64_t *)elem2;
    return v1 - v2;
}

static void sas7bcat_sort_index(sas7bcat_ctx_t *ctx) {
    if (ctx->block_pointers_used == 0)
        return;

    int i;
    for (i=1; i<ctx->block_pointers_used; i++) {
        if (ctx->block_pointers[i] < ctx->block_pointers[i-1]) {
            qsort(ctx->block_pointers, ctx->block_pointers_used, sizeof(uint64_t), &compare_block_pointers);
            break;
        }
    }
}

static void sas7bcat_uniq_index(sas7bcat_ctx_t *ctx) {
    if (ctx->block_pointers_used == 0)
        return;

    int i;
    int out_i = 1;
    for (i=1; i<ctx->block_pointers_used; i++) {
        if (ctx->block_pointers[i] != ctx->block_pointers[i-1]) {
            if (out_i != i) {
                ctx->block_pointers[out_i] = ctx->block_pointers[i];
            }
            out_i++;
        }
    }
    ctx->block_pointers_used = out_i;
}

static int sas7bcat_block_size(int start_page, int start_page_pos, sas7bcat_ctx_t *ctx, readstat_error_t *outError) {
    readstat_error_t retval = READSTAT_OK;
    readstat_io_t *io = ctx->io;
    int next_page = start_page;
    int next_page_pos = start_page_pos;
    int link_count = 0;

    int buffer_len = 0;
    int chain_link_len = 0;

    char chain_link[32];
    int chain_link_header_len = 16;
    if (ctx->u64) {
        chain_link_header_len = 32;
    }

    // calculate buffer size needed
    while (next_page > 0 && next_page_pos > 0 && next_page <= ctx->page_count && link_count++ < ctx->page_count) {
        if (io->seek(ctx->header_size+(next_page-1)*ctx->page_size+next_page_pos, READSTAT_SEEK_SET, io->io_ctx) == -1) {
            retval = READSTAT_ERROR_SEEK;
            goto cleanup;
        }
        if (io->read(chain_link, chain_link_header_len, io->io_ctx) < chain_link_header_len) {
            retval = READSTAT_ERROR_READ;
            goto cleanup;
        }

        if (ctx->u64) {
            next_page = sas_read4(&chain_link[0], ctx->bswap);
            next_page_pos = sas_read2(&chain_link[8], ctx->bswap);
            chain_link_len = sas_read2(&chain_link[10], ctx->bswap);
        } else {
            next_page = sas_read4(&chain_link[0], ctx->bswap);
            next_page_pos = sas_read2(&chain_link[4], ctx->bswap);
            chain_link_len = sas_read2(&chain_link[6], ctx->bswap);
        }

        buffer_len += chain_link_len;
    }

cleanup:
    if (outError)
        *outError = retval;

    return retval == READSTAT_OK ? buffer_len : -1;
}

static readstat_error_t sas7bcat_read_block(char *buffer, size_t buffer_len,
        int start_page, int start_page_pos, sas7bcat_ctx_t *ctx) {
    readstat_error_t retval = READSTAT_OK;
    readstat_io_t *io = ctx->io;
    int next_page = start_page;
    int next_page_pos = start_page_pos;
    int link_count = 0;

    int chain_link_len = 0;
    int buffer_offset = 0;

    char chain_link[32];
    int chain_link_header_len = 16;
    if (ctx->u64) {
        chain_link_header_len = 32;
    }

    while (next_page > 0 && next_page_pos > 0 && next_page <= ctx->page_count && link_count++ < ctx->page_count) {
        if (io->seek(ctx->header_size+(next_page-1)*ctx->page_size+next_page_pos, READSTAT_SEEK_SET, io->io_ctx) == -1) {
            retval = READSTAT_ERROR_SEEK;
            goto cleanup;
        }
        if (io->read(chain_link, chain_link_header_len, io->io_ctx) < chain_link_header_len) {
            retval = READSTAT_ERROR_READ;
            goto cleanup;
        }
        if (ctx->u64) {
            next_page = sas_read4(&chain_link[0], ctx->bswap);
            next_page_pos = sas_read2(&chain_link[8], ctx->bswap);
            chain_link_len = sas_read2(&chain_link[10], ctx->bswap);
        } else {
            next_page = sas_read4(&chain_link[0], ctx->bswap);
            next_page_pos = sas_read2(&chain_link[4], ctx->bswap);
            chain_link_len = sas_read2(&chain_link[6], ctx->bswap);
        }
        if (buffer_offset + chain_link_len > buffer_len) {
            retval = READSTAT_ERROR_PARSE;
            goto cleanup;
        }
        if (io->read(buffer + buffer_offset, chain_link_len, io->io_ctx) < chain_link_len) {
            retval = READSTAT_ERROR_READ;
            goto cleanup;
        }
        buffer_offset += chain_link_len;
    }
cleanup:

    return retval;
}

readstat_error_t readstat_parse_sas7bcat(readstat_parser_t *parser, const char *path, void *user_ctx) {
    readstat_error_t retval = READSTAT_OK;
    readstat_io_t *io = parser->io;
    int64_t i;
    char *page = NULL;
    char *buffer = NULL;

    sas7bcat_ctx_t *ctx = calloc(1, sizeof(sas7bcat_ctx_t));
    sas_header_info_t *hinfo = calloc(1, sizeof(sas_header_info_t));

    ctx->block_pointers = malloc((ctx->block_pointers_capacity = 200) * sizeof(uint64_t));

    ctx->value_label_handler = parser->handlers.value_label;
    ctx->metadata_handler = parser->handlers.metadata;
    ctx->input_encoding = parser->input_encoding;
    ctx->output_encoding = parser->output_encoding;
    ctx->user_ctx = user_ctx;
    ctx->io = io;

    if (io->open(path, io->io_ctx) == -1) {
        retval = READSTAT_ERROR_OPEN;
        goto cleanup;
    }

    if ((retval = sas_read_header(io, hinfo, parser->handlers.error, user_ctx)) != READSTAT_OK) {
        goto cleanup;
    }

    ctx->u64 = hinfo->u64;
    ctx->pad1 = hinfo->pad1;
    ctx->bswap = machine_is_little_endian() ^ hinfo->little_endian;
    ctx->header_size = hinfo->header_size;
    ctx->page_count = hinfo->page_count;
    ctx->page_size = hinfo->page_size;
    if (ctx->input_encoding == NULL) {
        ctx->input_encoding = hinfo->encoding;
    }

    ctx->xlsr_size = 212 + ctx->pad1;
    ctx->xlsr_offset = 856 + 2 * ctx->pad1;
    ctx->xlsr_O_offset = 50 + ctx->pad1;
    if (ctx->u64) {
        ctx->xlsr_offset += 144;
        ctx->xlsr_size += 72;
        ctx->xlsr_O_offset += 24;
    }

    if (ctx->input_encoding && ctx->output_encoding && strcmp(ctx->input_encoding, ctx->output_encoding) != 0) {
        iconv_t converter = iconv_open(ctx->output_encoding, ctx->input_encoding);
        if (converter == (iconv_t)-1) {
            retval = READSTAT_ERROR_UNSUPPORTED_CHARSET;
            goto cleanup;
        }
        ctx->converter = converter;
    }

    if (ctx->metadata_handler) {
        char table_name[4*32+1];
        readstat_metadata_t metadata = { 
            .file_encoding = ctx->input_encoding, /* orig encoding? */
            .modified_time = hinfo->modification_time,
            .creation_time = hinfo->creation_time,
            .file_format_version = hinfo->major_version,
            .endianness = hinfo->little_endian ? READSTAT_ENDIAN_LITTLE : READSTAT_ENDIAN_BIG,
            .is64bit = ctx->u64
        };
        retval = readstat_convert(table_name, sizeof(table_name),
                hinfo->table_name, sizeof(hinfo->table_name), ctx->converter);
        if (retval != READSTAT_OK)
            goto cleanup;

        metadata.table_name = table_name;

        if (ctx->metadata_handler(&metadata, ctx->user_ctx) != READSTAT_HANDLER_OK) {
            retval = READSTAT_ERROR_USER_ABORT;
            goto cleanup;
        }
    }

    if ((page = readstat_malloc(ctx->page_size)) == NULL) {
        retval = READSTAT_ERROR_MALLOC;
        goto cleanup;
    }
    if (io->seek(ctx->header_size+SAS_CATALOG_FIRST_INDEX_PAGE*ctx->page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) {
        retval = READSTAT_ERROR_SEEK;
        goto cleanup;
    }
    if (io->read(page, ctx->page_size, io->io_ctx) < ctx->page_size) {
        retval = READSTAT_ERROR_READ;
        goto cleanup;
    }

    retval = sas7bcat_augment_index(&page[ctx->xlsr_offset], ctx->page_size - ctx->xlsr_offset, ctx);
    if (retval != READSTAT_OK)
        goto cleanup;

    // Pass 1 -- find the XLSR entries
    for (i=SAS_CATALOG_USELESS_PAGES; i<ctx->page_count; i++) {
        if (io->seek(ctx->header_size+i*ctx->page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) {
            retval = READSTAT_ERROR_SEEK;
            goto cleanup;
        }
        if (io->read(page, ctx->page_size, io->io_ctx) < ctx->page_size) {
            retval = READSTAT_ERROR_READ;
            goto cleanup;
        }
        if (memcmp(&page[16], "XLSR", sizeof("XLSR")-1) == 0) {
            retval = sas7bcat_augment_index(&page[16], ctx->page_size - 16, ctx);
            if (retval != READSTAT_OK)
                goto cleanup;
        }
    }

    sas7bcat_sort_index(ctx);
    sas7bcat_uniq_index(ctx);

    // Pass 2 -- look up the individual block pointers
    for (i=0; i<ctx->block_pointers_used; i++) {
        int start_page = ctx->block_pointers[i] >> 32;
        int start_page_pos = (ctx->block_pointers[i]) & 0xFFFF;

        int buffer_len = sas7bcat_block_size(start_page, start_page_pos, ctx, &retval);
        if (buffer_len == -1) {
            goto cleanup;
        } else if (buffer_len == 0) {
            continue;
        }
        if ((buffer = readstat_realloc(buffer, buffer_len)) == NULL) {
            retval = READSTAT_ERROR_MALLOC;
            goto cleanup;
        }
        if ((retval = sas7bcat_read_block(buffer, buffer_len, start_page, start_page_pos, ctx)) != READSTAT_OK)
            goto cleanup;
        if ((retval = sas7bcat_parse_block(buffer, buffer_len, ctx)) != READSTAT_OK)
            goto cleanup;
    }

cleanup:
    io->close(io->io_ctx);
    if (page)
        free(page);
    if (buffer)
        free(buffer);
    if (ctx)
        sas7bcat_ctx_free(ctx);
    if (hinfo)
        free(hinfo);

    return retval;
}

Coverage Report

Created: 2026-08-14 06:26

Line	Count	Source
1		#include <stdio.h>
2		#include <stdlib.h>
3		#include <errno.h>
4		#include <string.h>
5		#include <math.h>
6		#include "readstat_sas.h"
7		#include "../readstat_iconv.h"
8		#include "../readstat_convert.h"
9		#include "../readstat_malloc.h"
10
11	1.16k	#define SAS_CATALOG_FIRST_INDEX_PAGE 1
12	1.06k	#define SAS_CATALOG_USELESS_PAGES 3
13
14		typedef struct sas7bcat_ctx_s {
15		readstat_metadata_handler metadata_handler;
16		readstat_value_label_handler value_label_handler;
17		void *user_ctx;
18		readstat_io_t *io;
19		int u64;
20		int pad1;
21		int bswap;
22		int64_t xlsr_size;
23		int64_t xlsr_offset;
24		int64_t xlsr_O_offset;
25		int64_t page_count;
26		int64_t page_size;
27		int64_t header_size;
28		uint64_t *block_pointers;
29		int block_pointers_used;
30		int block_pointers_capacity;
31		const char *input_encoding;
32		const char *output_encoding;
33		iconv_t converter;
34		} sas7bcat_ctx_t;
35
36	1.69k	static void sas7bcat_ctx_free(sas7bcat_ctx_t *ctx) {
37	1.69k	if (ctx->converter)
38	703	iconv_close(ctx->converter);
39	1.69k	if (ctx->block_pointers)
40	1.69k	free(ctx->block_pointers);
41
42	1.69k	free(ctx);
43	1.69k	}
44
45		static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, size_t value_labels_len,
46	1.63k	int label_count_used, int label_count_capacity, const char name, sas7bcat_ctx_t ctx) {
47	1.63k	readstat_error_t retval = READSTAT_OK;
48	1.63k	int i;
49	1.63k	const char *lbp1 = value_start;
50	1.63k	uint32_t *value_offset = readstat_calloc(label_count_used, sizeof(uint32_t));
51		/* Doubles appear to be stored as big-endian, always */
52	1.63k	int bswap_doubles = machine_is_little_endian();
53	1.63k	int is_string = (name[0] == '$');
54	1.63k	char *label = NULL;
55
56	1.63k	if (value_offset == NULL) {
57	150	retval = READSTAT_ERROR_MALLOC;
58	150	goto cleanup;
59	150	}
60
61		/* Pass 1 -- find out the offset of the labels */
62	4.85M	for (i=0; i<label_count_capacity; i++) {
63	4.85M	if (&lbp1[4] - value_start > value_labels_len \|\| sas_read2(&lbp1[2], ctx->bswap) < 0) {
64	49	retval = READSTAT_ERROR_PARSE;
65	49	goto cleanup;
66	49	}
67	4.85M	if (i<label_count_used) {
68	142k	if (&lbp1[10+ctx->pad1+4] - value_start > value_labels_len) {
69	13	retval = READSTAT_ERROR_PARSE;
70	13	goto cleanup;
71	13	}
72	142k	uint32_t label_pos = sas_read4(&lbp1[10+ctx->pad1], ctx->bswap);
73	142k	if (label_pos >= label_count_used) {
74	58	retval = READSTAT_ERROR_PARSE;
75	58	goto cleanup;
76	58	}
77	142k	value_offset[label_pos] = lbp1 - value_start;
78	142k	}
79	4.85M	lbp1 += 6 + sas_read2(&lbp1[2], ctx->bswap);
80	4.85M	}
81
82	1.36k	const char *lbp2 = lbp1;
83
84		/* Pass 2 -- parse pairs of values & labels */
85	45.1k	for (i=0; i<label_count_used && i<label_count_capacity; i++) {
86	43.9k	lbp1 = value_start + value_offset[i];
87
88	43.9k	if (&lbp1[30] - value_start > value_labels_len \|\|
89	43.9k	&lbp2[10] - value_start > value_labels_len) {
90	130	retval = READSTAT_ERROR_PARSE;
91	130	goto cleanup;
92	130	}
93	43.8k	readstat_value_t value = { .type = is_string ? READSTAT_TYPE_STRING : READSTAT_TYPE_DOUBLE };
94	43.8k	char string_val[4*16+1];
95	43.8k	if (is_string) {
96	529	size_t value_entry_len = 6 + sas_read2(&lbp1[2], ctx->bswap);
97	529	retval = readstat_convert(string_val, sizeof(string_val),
98	529	&lbp1[value_entry_len-16], 16, ctx->converter);
99	529	if (retval != READSTAT_OK)
100	1	goto cleanup;
101
102	528	value.v.string_value = string_val;
103	43.3k	} else {
104	43.3k	uint64_t val = sas_read8(&lbp1[22], bswap_doubles);
105	43.3k	double dval = NAN;
106	43.3k	if ((val \| 0xFF0000000000) == 0xFFFFFFFFFFFF) {
107	2.19k	sas_assign_tag(&value, (val >> 40));
108	41.1k	} else {
109	41.1k	memcpy(&dval, &val, 8);
110	41.1k	if (dval > 0.0) {
111	37.2k	val = ~val;
112	37.2k	memcpy(&dval, &val, 8);
113	37.2k	} else {
114	3.87k	dval *= -1.0;
115	3.87k	}
116	41.1k	}
117
118	43.3k	value.v.double_value = dval;
119	43.3k	}
120	43.8k	size_t label_len = sas_read2(&lbp2[8], ctx->bswap);
121	43.8k	if (&lbp2[10] > value_start + value_labels_len) {
122	0	retval = READSTAT_ERROR_PARSE;
123	0	goto cleanup;
124	0	}
125		/* Some labels seem to overflow the reported block length, truncate it */
126		/* (Observed with formats.sasbcat from GSS2021, produced with 9.0401M6X64_SR12R2 */
127	43.8k	if (label_len > value_start + value_labels_len - &lbp2[10]) {
128	368	label_len = value_start + value_labels_len - &lbp2[10];
129	368	}
130	43.8k	if (ctx->value_label_handler) {
131	43.8k	label = realloc(label, 4 * label_len + 1);
132	43.8k	retval = readstat_convert(label, 4 * label_len + 1,
133	43.8k	&lbp2[10], label_len, ctx->converter);
134	43.8k	if (retval != READSTAT_OK)
135	30	goto cleanup;
136
137	43.8k	if (ctx->value_label_handler(name, value, label, ctx->user_ctx) != READSTAT_HANDLER_OK) {
138	0	retval = READSTAT_ERROR_USER_ABORT;
139	0	goto cleanup;
140	0	}
141	43.8k	}
142
143	43.8k	lbp2 += 8 + 2 + label_len + 1;
144	43.8k	}
145
146	1.63k	cleanup:
147	1.63k	free(label);
148	1.63k	free(value_offset);
149	1.63k	return retval;
150	1.36k	}
151
152	3.62k	static readstat_error_t sas7bcat_parse_block(const char data, size_t data_size, sas7bcat_ctx_t ctx) {
153	3.62k	readstat_error_t retval = READSTAT_OK;
154
155	3.62k	size_t pad = 0;
156	3.62k	uint64_t label_count_capacity = 0;
157	3.62k	uint64_t label_count_used = 0;
158	3.62k	int payload_offset = 106;
159	3.62k	uint16_t flags = 0;
160	3.62k	char name[4*32+1];
161
162	3.62k	if (data_size < payload_offset)
163	763	goto cleanup;
164
165	2.86k	flags = sas_read2(&data[2], ctx->bswap);
166	2.86k	pad = (flags & 0x08) ? 4 : 0; // might be 0x10, not sure
167	2.86k	if (ctx->u64) {
168	494	label_count_capacity = sas_read8(&data[42+pad], ctx->bswap);
169	494	label_count_used = sas_read8(&data[50+pad], ctx->bswap);
170
171	494	payload_offset += 32;
172	2.37k	} else {
173	2.37k	label_count_capacity = sas_read4(&data[38+pad], ctx->bswap);
174	2.37k	label_count_used = sas_read4(&data[42+pad], ctx->bswap);
175	2.37k	}
176
177	2.86k	if ((retval = readstat_convert(name, sizeof(name), &data[8], 8, ctx->converter)) != READSTAT_OK)
178	5	goto cleanup;
179
180	2.86k	if (pad) {
181	1.19k	pad += 16;
182	1.19k	}
183
184	2.86k	if (((flags & 0x80) && !ctx->u64) \|\| ((flags & 0x20) && ctx->u64)) { // has long name
185	1.52k	if (data_size < payload_offset + pad + 32)
186	291	goto cleanup;
187
188	1.23k	retval = readstat_convert(name, sizeof(name), &data[payload_offset+pad], 32, ctx->converter);
189	1.23k	if (retval != READSTAT_OK)
190	2	goto cleanup;
191	1.23k	pad += 32;
192	1.23k	}
193
194	2.56k	if (data_size < payload_offset + pad)
195	213	goto cleanup;
196
197	2.35k	if (label_count_used == 0)
198	722	goto cleanup;
199
200	1.63k	if ((retval = sas7bcat_parse_value_labels(&data[payload_offset+pad], data_size - payload_offset - pad,
201	1.63k	label_count_used, label_count_capacity, name, ctx)) != READSTAT_OK)
202	431	goto cleanup;
203
204	3.62k	cleanup:
205	3.62k	return retval;
206	1.63k	}
207
208	9.00k	static readstat_error_t sas7bcat_augment_index(const char index, size_t len, sas7bcat_ctx_t ctx) {
209	9.00k	const char *xlsr = index;
210	9.00k	readstat_error_t retval = READSTAT_OK;
211	38.6k	while (xlsr + ctx->xlsr_size <= index + len) {
212	30.9k	if (memcmp(xlsr, "XLSR", 4) != 0) // some block pointers seem to have 8 bytes of extra padding
213	14.8k	xlsr += 8;
214	30.9k	if (memcmp(xlsr, "XLSR", 4) != 0)
215	1.35k	break;
216
217	29.6k	if (xlsr[ctx->xlsr_O_offset] == 'O') {
218	27.4k	uint64_t page = 0, pos = 0;
219	27.4k	if (ctx->u64) {
220	867	page = sas_read8(&xlsr[8], ctx->bswap);
221	867	pos = sas_read2(&xlsr[16], ctx->bswap);
222	26.5k	} else {
223	26.5k	page = sas_read4(&xlsr[4], ctx->bswap);
224	26.5k	pos = sas_read2(&xlsr[8], ctx->bswap);
225	26.5k	}
226	27.4k	ctx->block_pointers[ctx->block_pointers_used++] = (page << 32) + pos;
227	27.4k	}
228
229	29.6k	if (ctx->block_pointers_used == ctx->block_pointers_capacity) {
230	68	ctx->block_pointers = readstat_realloc(ctx->block_pointers, (ctx->block_pointers_capacity = 2) sizeof(uint64_t));
231	68	if (ctx->block_pointers == NULL) {
232	0	retval = READSTAT_ERROR_MALLOC;
233	0	goto cleanup;
234	0	}
235	68	}
236
237	29.6k	xlsr += ctx->xlsr_size;
238	29.6k	}
239	9.00k	cleanup:
240	9.00k	return retval;
241	9.00k	}
242
243	107k	static int compare_block_pointers(const void elem1, const void elem2) {
244	107k	uint64_t v1 = (const uint64_t )elem1;
245	107k	uint64_t v2 = (const uint64_t )elem2;
246	107k	return v1 - v2;
247	107k	}
248
249	976	static void sas7bcat_sort_index(sas7bcat_ctx_t *ctx) {
250	976	if (ctx->block_pointers_used == 0)
251	110	return;
252
253	866	int i;
254	1.24k	for (i=1; i<ctx->block_pointers_used; i++) {
255	592	if (ctx->block_pointers[i] < ctx->block_pointers[i-1]) {
256	210	qsort(ctx->block_pointers, ctx->block_pointers_used, sizeof(uint64_t), &compare_block_pointers);
257	210	break;
258	210	}
259	592	}
260	866	}
261
262	976	static void sas7bcat_uniq_index(sas7bcat_ctx_t *ctx) {
263	976	if (ctx->block_pointers_used == 0)
264	110	return;
265
266	866	int i;
267	866	int out_i = 1;
268	19.1k	for (i=1; i<ctx->block_pointers_used; i++) {
269	18.2k	if (ctx->block_pointers[i] != ctx->block_pointers[i-1]) {
270	13.7k	if (out_i != i) {
271	12.2k	ctx->block_pointers[out_i] = ctx->block_pointers[i];
272	12.2k	}
273	13.7k	out_i++;
274	13.7k	}
275	18.2k	}
276	866	ctx->block_pointers_used = out_i;
277	866	}
278
279	14.5k	static int sas7bcat_block_size(int start_page, int start_page_pos, sas7bcat_ctx_t ctx, readstat_error_t outError) {
280	14.5k	readstat_error_t retval = READSTAT_OK;
281	14.5k	readstat_io_t *io = ctx->io;
282	14.5k	int next_page = start_page;
283	14.5k	int next_page_pos = start_page_pos;
284	14.5k	int link_count = 0;
285
286	14.5k	int buffer_len = 0;
287	14.5k	int chain_link_len = 0;
288
289	14.5k	char chain_link[32];
290	14.5k	int chain_link_header_len = 16;
291	14.5k	if (ctx->u64) {
292	646	chain_link_header_len = 32;
293	646	}
294
295		// calculate buffer size needed
296	51.6k	while (next_page > 0 && next_page_pos > 0 && next_page <= ctx->page_count && link_count++ < ctx->page_count) {
297	37.1k	if (io->seek(ctx->header_size+(next_page-1)*ctx->page_size+next_page_pos, READSTAT_SEEK_SET, io->io_ctx) == -1) {
298	14	retval = READSTAT_ERROR_SEEK;
299	14	goto cleanup;
300	14	}
301	37.1k	if (io->read(chain_link, chain_link_header_len, io->io_ctx) < chain_link_header_len) {
302	19	retval = READSTAT_ERROR_READ;
303	19	goto cleanup;
304	19	}
305
306	37.1k	if (ctx->u64) {
307	1.39k	next_page = sas_read4(&chain_link[0], ctx->bswap);
308	1.39k	next_page_pos = sas_read2(&chain_link[8], ctx->bswap);
309	1.39k	chain_link_len = sas_read2(&chain_link[10], ctx->bswap);
310	35.7k	} else {
311	35.7k	next_page = sas_read4(&chain_link[0], ctx->bswap);
312	35.7k	next_page_pos = sas_read2(&chain_link[4], ctx->bswap);
313	35.7k	chain_link_len = sas_read2(&chain_link[6], ctx->bswap);
314	35.7k	}
315
316	37.1k	buffer_len += chain_link_len;
317	37.1k	}
318
319	14.5k	cleanup:
320	14.5k	if (outError)
321	14.5k	*outError = retval;
322
323	14.5k	return retval == READSTAT_OK ? buffer_len : -1;
324	14.5k	}
325
326		static readstat_error_t sas7bcat_read_block(char *buffer, size_t buffer_len,
327	3.67k	int start_page, int start_page_pos, sas7bcat_ctx_t *ctx) {
328	3.67k	readstat_error_t retval = READSTAT_OK;
329	3.67k	readstat_io_t *io = ctx->io;
330	3.67k	int next_page = start_page;
331	3.67k	int next_page_pos = start_page_pos;
332	3.67k	int link_count = 0;
333
334	3.67k	int chain_link_len = 0;
335	3.67k	int buffer_offset = 0;
336
337	3.67k	char chain_link[32];
338	3.67k	int chain_link_header_len = 16;
339	3.67k	if (ctx->u64) {
340	573	chain_link_header_len = 32;
341	573	}
342
343	37.2k	while (next_page > 0 && next_page_pos > 0 && next_page <= ctx->page_count && link_count++ < ctx->page_count) {
344	33.6k	if (io->seek(ctx->header_size+(next_page-1)*ctx->page_size+next_page_pos, READSTAT_SEEK_SET, io->io_ctx) == -1) {
345	0	retval = READSTAT_ERROR_SEEK;
346	0	goto cleanup;
347	0	}
348	33.6k	if (io->read(chain_link, chain_link_header_len, io->io_ctx) < chain_link_header_len) {
349	0	retval = READSTAT_ERROR_READ;
350	0	goto cleanup;
351	0	}
352	33.6k	if (ctx->u64) {
353	1.38k	next_page = sas_read4(&chain_link[0], ctx->bswap);
354	1.38k	next_page_pos = sas_read2(&chain_link[8], ctx->bswap);
355	1.38k	chain_link_len = sas_read2(&chain_link[10], ctx->bswap);
356	32.2k	} else {
357	32.2k	next_page = sas_read4(&chain_link[0], ctx->bswap);
358	32.2k	next_page_pos = sas_read2(&chain_link[4], ctx->bswap);
359	32.2k	chain_link_len = sas_read2(&chain_link[6], ctx->bswap);
360	32.2k	}
361	33.6k	if (buffer_offset + chain_link_len > buffer_len) {
362	0	retval = READSTAT_ERROR_PARSE;
363	0	goto cleanup;
364	0	}
365	33.6k	if (io->read(buffer + buffer_offset, chain_link_len, io->io_ctx) < chain_link_len) {
366	51	retval = READSTAT_ERROR_READ;
367	51	goto cleanup;
368	51	}
369	33.6k	buffer_offset += chain_link_len;
370	33.6k	}
371	3.67k	cleanup:
372
373	3.67k	return retval;
374	3.67k	}
375
376	1.69k	readstat_error_t readstat_parse_sas7bcat(readstat_parser_t parser, const char path, void *user_ctx) {
377	1.69k	readstat_error_t retval = READSTAT_OK;
378	1.69k	readstat_io_t *io = parser->io;
379	1.69k	int64_t i;
380	1.69k	char *page = NULL;
381	1.69k	char *buffer = NULL;
382
383	1.69k	sas7bcat_ctx_t *ctx = calloc(1, sizeof(sas7bcat_ctx_t));
384	1.69k	sas_header_info_t *hinfo = calloc(1, sizeof(sas_header_info_t));
385
386	1.69k	ctx->block_pointers = malloc((ctx->block_pointers_capacity = 200) * sizeof(uint64_t));
387
388	1.69k	ctx->value_label_handler = parser->handlers.value_label;
389	1.69k	ctx->metadata_handler = parser->handlers.metadata;
390	1.69k	ctx->input_encoding = parser->input_encoding;
391	1.69k	ctx->output_encoding = parser->output_encoding;
392	1.69k	ctx->user_ctx = user_ctx;
393	1.69k	ctx->io = io;
394
395	1.69k	if (io->open(path, io->io_ctx) == -1) {
396	0	retval = READSTAT_ERROR_OPEN;
397	0	goto cleanup;
398	0	}
399
400	1.69k	if ((retval = sas_read_header(io, hinfo, parser->handlers.error, user_ctx)) != READSTAT_OK) {
401	517	goto cleanup;
402	517	}
403
404	1.17k	ctx->u64 = hinfo->u64;
405	1.17k	ctx->pad1 = hinfo->pad1;
406	1.17k	ctx->bswap = machine_is_little_endian() ^ hinfo->little_endian;
407	1.17k	ctx->header_size = hinfo->header_size;
408	1.17k	ctx->page_count = hinfo->page_count;
409	1.17k	ctx->page_size = hinfo->page_size;
410	1.17k	if (ctx->input_encoding == NULL) {
411	1.17k	ctx->input_encoding = hinfo->encoding;
412	1.17k	}
413
414	1.17k	ctx->xlsr_size = 212 + ctx->pad1;
415	1.17k	ctx->xlsr_offset = 856 + 2 * ctx->pad1;
416	1.17k	ctx->xlsr_O_offset = 50 + ctx->pad1;
417	1.17k	if (ctx->u64) {
418	275	ctx->xlsr_offset += 144;
419	275	ctx->xlsr_size += 72;
420	275	ctx->xlsr_O_offset += 24;
421	275	}
422
423	1.17k	if (ctx->input_encoding && ctx->output_encoding && strcmp(ctx->input_encoding, ctx->output_encoding) != 0) {
424	705	iconv_t converter = iconv_open(ctx->output_encoding, ctx->input_encoding);
425	705	if (converter == (iconv_t)-1) {
426	2	retval = READSTAT_ERROR_UNSUPPORTED_CHARSET;
427	2	goto cleanup;
428	2	}
429	703	ctx->converter = converter;
430	703	}
431
432	1.17k	if (ctx->metadata_handler) {
433	1.17k	char table_name[4*32+1];
434	1.17k	readstat_metadata_t metadata = {
435	1.17k	.file_encoding = ctx->input_encoding, /* orig encoding? */
436	1.17k	.modified_time = hinfo->modification_time,
437	1.17k	.creation_time = hinfo->creation_time,
438	1.17k	.file_format_version = hinfo->major_version,
439	1.17k	.endianness = hinfo->little_endian ? READSTAT_ENDIAN_LITTLE : READSTAT_ENDIAN_BIG,
440	1.17k	.is64bit = ctx->u64
441	1.17k	};
442	1.17k	retval = readstat_convert(table_name, sizeof(table_name),
443	1.17k	hinfo->table_name, sizeof(hinfo->table_name), ctx->converter);
444	1.17k	if (retval != READSTAT_OK)
445	3	goto cleanup;
446
447	1.16k	metadata.table_name = table_name;
448
449	1.16k	if (ctx->metadata_handler(&metadata, ctx->user_ctx) != READSTAT_HANDLER_OK) {
450	0	retval = READSTAT_ERROR_USER_ABORT;
451	0	goto cleanup;
452	0	}
453	1.16k	}
454
455	1.16k	if ((page = readstat_malloc(ctx->page_size)) == NULL) {
456	0	retval = READSTAT_ERROR_MALLOC;
457	0	goto cleanup;
458	0	}
459	1.16k	if (io->seek(ctx->header_size+SAS_CATALOG_FIRST_INDEX_PAGE*ctx->page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) {
460	63	retval = READSTAT_ERROR_SEEK;
461	63	goto cleanup;
462	63	}
463	1.10k	if (io->read(page, ctx->page_size, io->io_ctx) < ctx->page_size) {
464	41	retval = READSTAT_ERROR_READ;
465	41	goto cleanup;
466	41	}
467
468	1.06k	retval = sas7bcat_augment_index(&page[ctx->xlsr_offset], ctx->page_size - ctx->xlsr_offset, ctx);
469	1.06k	if (retval != READSTAT_OK)
470	0	goto cleanup;
471
472		// Pass 1 -- find the XLSR entries
473	22.5k	for (i=SAS_CATALOG_USELESS_PAGES; i<ctx->page_count; i++) {
474	21.5k	if (io->seek(ctx->header_size+i*ctx->page_size, READSTAT_SEEK_SET, io->io_ctx) == -1) {
475	15	retval = READSTAT_ERROR_SEEK;
476	15	goto cleanup;
477	15	}
478	21.5k	if (io->read(page, ctx->page_size, io->io_ctx) < ctx->page_size) {
479	74	retval = READSTAT_ERROR_READ;
480	74	goto cleanup;
481	74	}
482	21.5k	if (memcmp(&page[16], "XLSR", sizeof("XLSR")-1) == 0) {
483	7.94k	retval = sas7bcat_augment_index(&page[16], ctx->page_size - 16, ctx);
484	7.94k	if (retval != READSTAT_OK)
485	0	goto cleanup;
486	7.94k	}
487	21.5k	}
488
489	976	sas7bcat_sort_index(ctx);
490	976	sas7bcat_uniq_index(ctx);
491
492		// Pass 2 -- look up the individual block pointers
493	14.9k	for (i=0; i<ctx->block_pointers_used; i++) {
494	14.5k	int start_page = ctx->block_pointers[i] >> 32;
495	14.5k	int start_page_pos = (ctx->block_pointers[i]) & 0xFFFF;
496
497	14.5k	int buffer_len = sas7bcat_block_size(start_page, start_page_pos, ctx, &retval);
498	14.5k	if (buffer_len == -1) {
499	33	goto cleanup;
500	14.4k	} else if (buffer_len == 0) {
501	10.8k	continue;
502	10.8k	}
503	3.68k	if ((buffer = readstat_realloc(buffer, buffer_len)) == NULL) {
504	4	retval = READSTAT_ERROR_MALLOC;
505	4	goto cleanup;
506	4	}
507	3.67k	if ((retval = sas7bcat_read_block(buffer, buffer_len, start_page, start_page_pos, ctx)) != READSTAT_OK)
508	51	goto cleanup;
509	3.62k	if ((retval = sas7bcat_parse_block(buffer, buffer_len, ctx)) != READSTAT_OK)
510	438	goto cleanup;
511	3.62k	}
512
513	1.69k	cleanup:
514	1.69k	io->close(io->io_ctx);
515	1.69k	if (page)
516	1.16k	free(page);
517	1.69k	if (buffer)
518	706	free(buffer);
519	1.69k	if (ctx)
520	1.69k	sas7bcat_ctx_free(ctx);
521	1.69k	if (hinfo)
522	1.69k	free(hinfo);
523
524	1.69k	return retval;
525	976	}