Basic C library changes for 64 bit tsk_size_t

Partial

Update offset column accessors

Fix uint32 in sample_set_sizes

Tests passing

Partial

Tests mostly passing here

Author: jeromekelleher

c/tests/test_file_format.c

@@ -359,10 +359,10 @@ verify_bad_offset_columns(tsk_treeseq_t *ts, const char *offset_col)
     int ret = 0;
     kastore_t store;
     tsk_table_collection_t tables;
-    tsk_size_t *offset_array, *offset_copy;
+    uint32_t *offset_array, *offset_copy;
     size_t offset_len;
     int type;
-    tsk_size_t data_len;
+    uint32_t data_len;
 
     ret = tsk_treeseq_dump(ts, _tmp_file_name, 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
@@ -371,7 +371,7 @@ verify_bad_offset_columns(tsk_treeseq_t *ts, const char *offset_col)
 
     ret = kastore_gets(&store, offset_col, (void **) &offset_array, &offset_len, &type);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
-    CU_ASSERT_EQUAL_FATAL(type, TSK_SIZE_STORAGE_TYPE);
+    CU_ASSERT_EQUAL_FATAL(type, KAS_UINT32);
     offset_copy = malloc(offset_len * sizeof(*offset_array));
     CU_ASSERT_FATAL(offset_copy != NULL);
     memcpy(offset_copy, offset_array, offset_len * sizeof(*offset_array));
@@ -383,8 +383,7 @@ verify_bad_offset_columns(tsk_treeseq_t *ts, const char *offset_col)
     copy_store_drop_columns(ts, 1, &offset_col, _tmp_file_name);
     ret = kastore_open(&store, _tmp_file_name, "a", 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
-    ret = kastore_puts(
-        &store, offset_col, offset_copy, offset_len, TSK_SIZE_STORAGE_TYPE, 0);
+    ret = kastore_puts(&store, offset_col, offset_copy, offset_len, KAS_UINT32, 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
     ret = kastore_close(&store);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
@@ -397,8 +396,7 @@ verify_bad_offset_columns(tsk_treeseq_t *ts, const char *offset_col)
     copy_store_drop_columns(ts, 1, &offset_col, _tmp_file_name);
     ret = kastore_open(&store, _tmp_file_name, "a", 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
-    ret = kastore_puts(
-        &store, offset_col, offset_copy, offset_len, TSK_SIZE_STORAGE_TYPE, 0);
+    ret = kastore_puts(&store, offset_col, offset_copy, offset_len, KAS_UINT32, 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
     ret = kastore_close(&store);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
@@ -410,8 +408,7 @@ verify_bad_offset_columns(tsk_treeseq_t *ts, const char *offset_col)
     copy_store_drop_columns(ts, 1, &offset_col, _tmp_file_name);
     ret = kastore_open(&store, _tmp_file_name, "a", 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
-    ret = kastore_puts(
-        &store, offset_col, offset_copy, offset_len, TSK_SIZE_STORAGE_TYPE, 0);
+    ret = kastore_puts(&store, offset_col, offset_copy, offset_len, KAS_UINT32, 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
     ret = kastore_close(&store);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
@@ -422,7 +419,7 @@ verify_bad_offset_columns(tsk_treeseq_t *ts, const char *offset_col)
     copy_store_drop_columns(ts, 1, &offset_col, _tmp_file_name);
     ret = kastore_open(&store, _tmp_file_name, "a", 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
-    ret = kastore_puts(&store, offset_col, NULL, 0, TSK_SIZE_STORAGE_TYPE, 0);
+    ret = kastore_puts(&store, offset_col, NULL, 0, KAS_UINT32, 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
     ret = kastore_close(&store);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
@@ -977,24 +974,24 @@ static void
 test_load_node_table_errors(void)
 {
     char format_name[TSK_FILE_FORMAT_NAME_LENGTH];
-    tsk_size_t uuid_size = 36;
+    size_t uuid_size = 36;
     char uuid[uuid_size];
     double L = 1;
     double time = 0;
     double flags = 0;
     tsk_id_t population = 0;
     tsk_id_t individual = 0;
     int8_t metadata = 0;
-    tsk_size_t metadata_offset[] = { 0, 1 };
-    tsk_size_t version[2]
+    uint32_t metadata_offset[] = { 0, 1 };
+    uint32_t version[2]
         = { TSK_FILE_FORMAT_VERSION_MAJOR, TSK_FILE_FORMAT_VERSION_MINOR };
     write_table_col_t write_cols[] = {
         { "nodes/time", (void *) &time, 1, KAS_FLOAT64 },
         { "nodes/flags", (void *) &flags, 1, TSK_FLAGS_STORAGE_TYPE },
         { "nodes/population", (void *) &population, 1, TSK_ID_STORAGE_TYPE },
         { "nodes/individual", (void *) &individual, 1, TSK_ID_STORAGE_TYPE },
         { "nodes/metadata", (void *) &metadata, 1, KAS_UINT8 },
-        { "nodes/metadata_offset", (void *) metadata_offset, 2, TSK_SIZE_STORAGE_TYPE },
+        { "nodes/metadata_offset", (void *) metadata_offset, 2, KAS_UINT32 },
         { "format/name", (void *) format_name, sizeof(format_name), KAS_INT8 },
         { "format/version", (void *) version, 2, KAS_UINT32 },
         { "uuid", (void *) uuid, uuid_size, KAS_INT8 },

c/tskit/core.h

@@ -147,16 +147,9 @@ typedef int32_t tsk_id_t;
 The ``tsk_size_t`` type is an unsigned integer used for any size or count value.
 @endrst
 */
-#ifdef _TSK_BIG_TABLES
-/* TODO get rid of this typdef once we move to 64 bit sizes */
 typedef uint64_t tsk_size_t;
 #define TSK_MAX_SIZE UINT64_MAX
 #define TSK_SIZE_STORAGE_TYPE KAS_UINT64
-#else
-typedef uint32_t tsk_size_t;
-#define TSK_MAX_SIZE UINT32_MAX
-#define TSK_SIZE_STORAGE_TYPE KAS_UINT32
-#endif
 
 /**
 @brief Container for bitwise flags.

c/tskit/tables.c

@@ -146,12 +146,12 @@ read_table_cols(kastore_t *store, tsk_size_t *num_rows, read_table_col_t *cols,
 }
 
 static int
-cast_offset_array(read_table_ragged_col_t *col, uint64_t *source, tsk_size_t num_rows)
+cast_offset_array(read_table_ragged_col_t *col, uint32_t *source, tsk_size_t num_rows)
 {
     int ret = 0;
     tsk_size_t len = num_rows + 1;
     tsk_size_t j;
-    uint32_t *dest = tsk_malloc(len * sizeof(*dest));
+    uint64_t *dest = tsk_malloc(len * sizeof(*dest));
 
     if (dest == NULL) {
         ret = TSK_ERR_NO_MEMORY;
@@ -160,9 +160,7 @@ cast_offset_array(read_table_ragged_col_t *col, uint64_t *source, tsk_size_t num
     col->offset_array_mem = dest;
     *col->offset_array_dest = dest;
     for (j = 0; j < len; j++) {
-        /* We don't bother catching errors here because we'll be switching this
-         * to casting up to 64 bit soon; current version is temporary. */
-        dest[j] = (uint32_t) source[j];
+        dest[j] = source[j];
     }
 out:
     return ret;
@@ -243,10 +241,10 @@ read_table_ragged_cols(kastore_t *store, tsk_size_t *num_rows,
                     goto out;
                 }
             }
-            if (type == KAS_UINT32) {
-                *col->offset_array_dest = (uint32_t *) store_offset_array;
-            } else if (type == KAS_UINT64) {
-                ret = cast_offset_array(col, (uint64_t *) store_offset_array, *num_rows);
+            if (type == KAS_UINT64) {
+                *col->offset_array_dest = (uint64_t *) store_offset_array;
+            } else if (type == KAS_UINT32) {
+                ret = cast_offset_array(col, (uint32_t *) store_offset_array, *num_rows);
                 if (ret != 0) {
                     goto out;
                 }
@@ -351,44 +349,39 @@ write_offset_col(
 {
     int ret = 0;
     char offset_col_name[TSK_MAX_COL_NAME_LEN];
-    int64_t *offset64 = NULL;
+    uint32_t *offset32 = NULL;
     tsk_size_t len = col->num_rows + 1;
     tsk_size_t j;
     int type;
     const void *data;
+    bool needs_64 = col->offset_array[col->num_rows] > UINT32_MAX;
 
     assert(strlen(col->name) + strlen("_offset") + 2 < sizeof(offset_col_name));
     strcpy(offset_col_name, col->name);
     strcat(offset_col_name, "_offset");
 
-    /* Note: this is a temporary implementation while we're getting some infrastructure
-     * in place for the change to 64 bit offsets. Ultimately we'll be doing the cast
-     * in the other direction, if the size of small enough. The TSK_DUMP_FORCE_OFFSET_64
-     * option will still be useful for testing though, because that means we don't have
-     * to force huge arrays to test all the code paths.
-     */
-    if (options & TSK_DUMP_FORCE_OFFSET_64) {
-        offset64 = tsk_malloc(len * sizeof(*offset64));
-        if (offset64 == NULL) {
+    if (options & TSK_DUMP_FORCE_OFFSET_64 || needs_64) {
+        type = KAS_UINT64;
+        data = col->offset_array;
+    } else {
+        offset32 = tsk_malloc(len * sizeof(*offset32));
+        if (offset32 == NULL) {
             ret = TSK_ERR_NO_MEMORY;
             goto out;
         }
         for (j = 0; j < len; j++) {
-            offset64[j] = col->offset_array[j];
+            offset32[j] = (uint32_t) col->offset_array[j];
         }
-        type = KAS_UINT64;
-        data = offset64;
-    } else {
         type = KAS_UINT32;
-        data = col->offset_array;
+        data = offset32;
     }
     ret = kastore_puts(store, offset_col_name, data, len, type, 0);
     if (ret != 0) {
         ret = tsk_set_kas_error(ret);
         goto out;
     }
 out:
-    tsk_safe_free(offset64);
+    tsk_safe_free(offset32);
     return ret;
 }
 

docs/conf.py

@@ -287,9 +287,11 @@ def handle_item(fieldarg, content):
 nitpick_ignore = [
     ("c:identifier", "int32_t"),
     ("c:identifier", "uint32_t"),
+    ("c:identifier", "uint64_t"),
     ("c:identifier", "FILE"),
     ("c:type", "int32_t"),
     ("c:type", "uint32_t"),
+    ("c:type", "uint64_t"),
     ("c:type", "bool"),
     ("py:class", "tskit.metadata.AbstractMetadataCodec"),
     ("py:class", "tskit.trees.Site"),