Add TSK_DUMP_FORCE_OFFSET64 flag and test

This adds an intermediate step towards using 64 bit sizes internally.

Author: jeromekelleher

c/tests/test_file_format.c

@@ -480,6 +480,7 @@ test_force_offset_64(void)
     int ret;
     tsk_treeseq_t *ts = caterpillar_tree(5, 3, 3);
     tsk_table_collection_t t1;
+    tsk_table_collection_t t2;
     kastore_t store;
     kaitem_t *item;
     const char *suffix;
@@ -509,7 +510,15 @@ test_force_offset_64(void)
 
     ret = tsk_table_collection_load(&t1, _tmp_file_name, 0);
     CU_ASSERT_EQUAL_FATAL(ret, 0);
+
+    ret = tsk_treeseq_copy_tables(ts, &t2, 0);
+    CU_ASSERT_EQUAL_FATAL(ret, 0);
+    CU_ASSERT_TRUE(tsk_table_collection_equals(&t1, &t2, 0));
+
     tsk_table_collection_free(&t1);
+    tsk_table_collection_free(&t2);
+    tsk_treeseq_free(ts);
+    free(ts);
 }
 
 static void

c/tskit/tables.c

@@ -146,23 +146,24 @@ read_table_cols(kastore_t *store, tsk_size_t *num_rows, read_table_col_t *cols,
 }
 
 static int
-cast_offset_array(read_table_ragged_col_t *col, tsk_size_t num_rows)
+cast_offset_array(read_table_ragged_col_t *col, uint64_t *source, tsk_size_t num_rows)
 {
     int ret = 0;
     tsk_size_t len = num_rows + 1;
     tsk_size_t j;
-    uint64_t *source = *col->offset_array_dest;
     uint32_t *dest = malloc(len * sizeof(*dest));
 
     if (dest == NULL) {
         ret = TSK_ERR_NO_MEMORY;
         goto out;
     }
     col->offset_array_mem = dest;
+    *col->offset_array_dest = dest;
     for (j = 0; j < len; j++) {
-        dest[j] = source[j]
+        /* We don't bother catching errors here because we'll be switching this
+         * to casting up to 64 bit soon; current version is temporary. */
+        dest[j] = (uint32_t) source[j];
     }
-    col->offset_col_present
 out:
     return ret;
 }
@@ -177,6 +178,7 @@ read_table_ragged_cols(kastore_t *store, tsk_size_t *num_rows,
     read_table_ragged_col_t *col;
     char offset_col_name[TSK_MAX_COL_NAME_LEN];
     bool data_col_present, offset_col_present;
+    void *store_offset_array;
     tsk_size_t *offset_array;
 
     for (col = cols; col->name != NULL; col++) {
@@ -220,8 +222,8 @@ read_table_ragged_cols(kastore_t *store, tsk_size_t *num_rows,
             goto out;
         }
         if (offset_col_present) {
-            ret = kastore_gets(store, offset_col_name, (void **) col->offset_array_dest,
-                &offset_len, &type);
+            ret = kastore_gets(
+                store, offset_col_name, &store_offset_array, &offset_len, &type);
             if (ret != 0) {
                 ret = tsk_set_kas_error(ret);
                 goto out;
@@ -241,12 +243,14 @@ read_table_ragged_cols(kastore_t *store, tsk_size_t *num_rows,
                     goto out;
                 }
             }
-            if (type == KAS_UINT64) {
-                ret = cast_offset_array(col, *num_rows);
+            if (type == KAS_UINT32) {
+                *col->offset_array_dest = (uint32_t *) store_offset_array;
+            } else if (type == KAS_UINT64) {
+                ret = cast_offset_array(col, (uint64_t *) store_offset_array, *num_rows);
                 if (ret != 0) {
                     goto out;
                 }
-            } else if (type == KAS_UINT32) {
+            } else {
                 ret = TSK_ERR_BAD_COLUMN_TYPE;
                 goto out;
             }
@@ -328,8 +332,22 @@ read_table(kastore_t *store, tsk_size_t *num_rows, read_table_col_t *cols,
     return ret;
 }
 
+static void
+free_read_table_mem(read_table_col_t *TSK_UNUSED(cols),
+    read_table_ragged_col_t *ragged_cols, read_table_property_t *TSK_UNUSED(properties))
+{
+    read_table_ragged_col_t *ragged_col;
+
+    if (ragged_cols != NULL) {
+        for (ragged_col = ragged_cols; ragged_col->name != NULL; ragged_col++) {
+            tsk_safe_free(ragged_col->offset_array_mem);
+        }
+    }
+}
+
 static int
-write_offset_col(kastore_t *store, const write_table_ragged_col_t *col, tsk_flags_t options)
+write_offset_col(
+    kastore_t *store, const write_table_ragged_col_t *col, tsk_flags_t options)
 {
     int ret = 0;
     char offset_col_name[TSK_MAX_COL_NAME_LEN];
@@ -350,7 +368,6 @@ write_offset_col(kastore_t *store, const write_table_ragged_col_t *col, tsk_flag
      * to force huge arrays to test all the code paths.
      */
     if (options & TSK_DUMP_FORCE_OFFSET_64) {
-        printf("write 64\n");
         offset64 = malloc(len * sizeof(*offset64));
         if (offset64 == NULL) {
             ret = TSK_ERR_NO_MEMORY;
@@ -376,8 +393,8 @@ write_offset_col(kastore_t *store, const write_table_ragged_col_t *col, tsk_flag
 }
 
 static int
-write_table_ragged_cols(kastore_t *store, const write_table_ragged_col_t *write_cols,
-    tsk_flags_t options)
+write_table_ragged_cols(
+    kastore_t *store, const write_table_ragged_col_t *write_cols, tsk_flags_t options)
 {
     int ret = 0;
     const write_table_ragged_col_t *col;
@@ -1234,7 +1251,8 @@ tsk_individual_table_equals(const tsk_individual_table_t *self,
 }
 
 static int
-tsk_individual_table_dump(const tsk_individual_table_t *self, kastore_t *store, tsk_flags_t options)
+tsk_individual_table_dump(
+    const tsk_individual_table_t *self, kastore_t *store, tsk_flags_t options)
 {
     const write_table_col_t write_cols[] = {
         { "individuals/flags", (void *) self->flags, self->num_rows,
@@ -1277,11 +1295,11 @@ tsk_individual_table_load(tsk_individual_table_t *self, kastore_t *store)
     };
     read_table_ragged_col_t ragged_cols[] = {
         { "individuals/location", (void **) &location, &location_length, KAS_FLOAT64,
-            &location_offset, 0, NULL},
+            &location_offset, 0, NULL },
         { "individuals/parents", (void **) &parents, &parents_length,
-            TSK_ID_STORAGE_TYPE, &parents_offset, TSK_COL_OPTIONAL, NULL},
+            TSK_ID_STORAGE_TYPE, &parents_offset, TSK_COL_OPTIONAL, NULL },
         { "individuals/metadata", (void **) &metadata, &metadata_length, KAS_UINT8,
-            &metadata_offset, 0, NULL},
+            &metadata_offset, 0, NULL },
         { .name = NULL },
     };
     read_table_property_t properties[] = {
@@ -1307,6 +1325,7 @@ tsk_individual_table_load(tsk_individual_table_t *self, kastore_t *store)
         }
     }
 out:
+    free_read_table_mem(cols, ragged_cols, properties);
     return ret;
 }
 
@@ -1873,7 +1892,7 @@ tsk_node_table_load(tsk_node_table_t *self, kastore_t *store)
     };
     read_table_ragged_col_t ragged_cols[] = {
         { "nodes/metadata", (void **) &metadata, &metadata_length, KAS_UINT8,
-            &metadata_offset, 0, NULL},
+            &metadata_offset, 0, NULL },
         { .name = NULL },
     };
     read_table_property_t properties[] = {
@@ -1899,6 +1918,7 @@ tsk_node_table_load(tsk_node_table_t *self, kastore_t *store)
         }
     }
 out:
+    free_read_table_mem(cols, ragged_cols, properties);
     return ret;
 }
 
@@ -2509,7 +2529,7 @@ tsk_edge_table_load(tsk_edge_table_t *self, kastore_t *store)
     };
     read_table_ragged_col_t ragged_cols[] = {
         { "edges/metadata", (void **) &metadata, &metadata_length, KAS_UINT8,
-            &metadata_offset, TSK_COL_OPTIONAL, NULL},
+            &metadata_offset, TSK_COL_OPTIONAL, NULL },
         { .name = NULL },
     };
     read_table_property_t properties[] = {
@@ -2535,6 +2555,7 @@ tsk_edge_table_load(tsk_edge_table_t *self, kastore_t *store)
         }
     }
 out:
+    free_read_table_mem(cols, ragged_cols, properties);
     return ret;
 }
 
@@ -3180,9 +3201,9 @@ tsk_site_table_load(tsk_site_table_t *self, kastore_t *store)
     };
     read_table_ragged_col_t ragged_cols[] = {
         { "sites/ancestral_state", (void **) &ancestral_state, &ancestral_state_length,
-            KAS_UINT8, &ancestral_state_offset, 0, NULL},
+            KAS_UINT8, &ancestral_state_offset, 0, NULL },
         { "sites/metadata", (void **) &metadata, &metadata_length, KAS_UINT8,
-            &metadata_offset, 0, NULL},
+            &metadata_offset, 0, NULL },
         { .name = NULL },
     };
     read_table_property_t properties[] = {
@@ -3208,6 +3229,7 @@ tsk_site_table_load(tsk_site_table_t *self, kastore_t *store)
         }
     }
 out:
+    free_read_table_mem(cols, ragged_cols, properties);
     return ret;
 }
 
@@ -3817,7 +3839,8 @@ tsk_mutation_table_dump_text(const tsk_mutation_table_t *self, FILE *out)
 }
 
 static int
-tsk_mutation_table_dump(const tsk_mutation_table_t *self, kastore_t *store, tsk_flags_t options)
+tsk_mutation_table_dump(
+    const tsk_mutation_table_t *self, kastore_t *store, tsk_flags_t options)
 {
     const write_table_col_t cols[] = {
         { "mutations/site", (void *) self->site, self->num_rows, TSK_ID_STORAGE_TYPE },
@@ -3865,9 +3888,9 @@ tsk_mutation_table_load(tsk_mutation_table_t *self, kastore_t *store)
     };
     read_table_ragged_col_t ragged_cols[] = {
         { "mutations/derived_state", (void **) &derived_state, &derived_state_length,
-            KAS_UINT8, &derived_state_offset, 0, NULL},
+            KAS_UINT8, &derived_state_offset, 0, NULL },
         { "mutations/metadata", (void **) &metadata, &metadata_length, KAS_UINT8,
-            &metadata_offset, 0, NULL},
+            &metadata_offset, 0, NULL },
         { .name = NULL },
     };
     read_table_property_t properties[] = {
@@ -3893,6 +3916,7 @@ tsk_mutation_table_load(tsk_mutation_table_t *self, kastore_t *store)
         }
     }
 out:
+    free_read_table_mem(cols, ragged_cols, properties);
     return ret;
 }
 
@@ -4413,7 +4437,8 @@ tsk_migration_table_equals(const tsk_migration_table_t *self,
 }
 
 static int
-tsk_migration_table_dump(const tsk_migration_table_t *self, kastore_t *store, tsk_flags_t options)
+tsk_migration_table_dump(
+    const tsk_migration_table_t *self, kastore_t *store, tsk_flags_t options)
 {
     const write_table_col_t cols[] = {
         { "migrations/left", (void *) self->left, self->num_rows, KAS_FLOAT64 },
@@ -4462,7 +4487,7 @@ tsk_migration_table_load(tsk_migration_table_t *self, kastore_t *store)
     };
     read_table_ragged_col_t ragged_cols[] = {
         { "migrations/metadata", (void **) &metadata, &metadata_length, KAS_UINT8,
-            &metadata_offset, TSK_COL_OPTIONAL, NULL},
+            &metadata_offset, TSK_COL_OPTIONAL, NULL },
         { .name = NULL },
     };
     read_table_property_t properties[] = {
@@ -4488,6 +4513,7 @@ tsk_migration_table_load(tsk_migration_table_t *self, kastore_t *store)
         }
     }
 out:
+    free_read_table_mem(cols, ragged_cols, properties);
     return ret;
 }
 
@@ -4942,7 +4968,8 @@ tsk_population_table_equals(const tsk_population_table_t *self,
 }
 
 static int
-tsk_population_table_dump(const tsk_population_table_t *self, kastore_t *store, tsk_flags_t options)
+tsk_population_table_dump(
+    const tsk_population_table_t *self, kastore_t *store, tsk_flags_t options)
 {
     const write_table_col_t cols[] = {
         { "populations/metadata_schema", (void *) self->metadata_schema,
@@ -4969,7 +4996,7 @@ tsk_population_table_load(tsk_population_table_t *self, kastore_t *store)
 
     read_table_ragged_col_t ragged_cols[] = {
         { "populations/metadata", (void **) &metadata, &metadata_length, KAS_UINT8,
-            &metadata_offset, 0, NULL},
+            &metadata_offset, 0, NULL },
         { .name = NULL },
     };
     read_table_property_t properties[] = {
@@ -4994,6 +5021,7 @@ tsk_population_table_load(tsk_population_table_t *self, kastore_t *store)
         }
     }
 out:
+    free_read_table_mem(NULL, ragged_cols, properties);
     return ret;
 }
 
@@ -5510,7 +5538,8 @@ tsk_provenance_table_equals(const tsk_provenance_table_t *self,
 }
 
 static int
-tsk_provenance_table_dump(const tsk_provenance_table_t *self, kastore_t *store, tsk_flags_t options)
+tsk_provenance_table_dump(
+    const tsk_provenance_table_t *self, kastore_t *store, tsk_flags_t options)
 {
     write_table_ragged_col_t ragged_cols[] = {
         { "provenances/timestamp", (void *) self->timestamp, self->timestamp_length,
@@ -5535,9 +5564,9 @@ tsk_provenance_table_load(tsk_provenance_table_t *self, kastore_t *store)
 
     read_table_ragged_col_t ragged_cols[] = {
         { "provenances/timestamp", (void **) &timestamp, &timestamp_length, KAS_UINT8,
-            &timestamp_offset, 0, NULL},
+            &timestamp_offset, 0, NULL },
         { "provenances/record", (void **) &record, &record_length, KAS_UINT8,
-            &record_offset, 0, NULL},
+            &record_offset, 0, NULL },
         { .name = NULL },
     };
 
@@ -5551,6 +5580,7 @@ tsk_provenance_table_load(tsk_provenance_table_t *self, kastore_t *store)
         goto out;
     }
 out:
+    free_read_table_mem(NULL, ragged_cols, NULL);
     return ret;
 }
 
@@ -9949,7 +9979,8 @@ tsk_table_collection_read_format_data(tsk_table_collection_t *self, kastore_t *s
 }
 
 static int TSK_WARN_UNUSED
-tsk_table_collection_dump_indexes(const tsk_table_collection_t *self, kastore_t *store, tsk_flags_t TSK_UNUSED(options))
+tsk_table_collection_dump_indexes(const tsk_table_collection_t *self, kastore_t *store,
+    tsk_flags_t TSK_UNUSED(options))
 {
     int ret = 0;
     write_table_col_t cols[] = {
@@ -10136,8 +10167,8 @@ tsk_table_collection_load(
 }
 
 static int TSK_WARN_UNUSED
-tsk_table_collection_write_format_data(
-    const tsk_table_collection_t *self, kastore_t *store, tsk_flags_t TSK_UNUSED(options))
+tsk_table_collection_write_format_data(const tsk_table_collection_t *self,
+    kastore_t *store, tsk_flags_t TSK_UNUSED(options))
 {
     int ret = 0;
     char format_name[TSK_FILE_FORMAT_NAME_LENGTH];
@@ -10199,7 +10230,8 @@ tsk_table_collection_dump(
 }
 
 int TSK_WARN_UNUSED
-tsk_table_collection_dumpf(const tsk_table_collection_t *self, FILE *file, tsk_flags_t options)
+tsk_table_collection_dumpf(
+    const tsk_table_collection_t *self, FILE *file, tsk_flags_t options)
 {
     int ret = 0;
     kastore_t store;