SV - Save Version - Saving versioned binary data

serialize.h

//--------------------------------------------------------------------------------------------------
// Hacky compat layer (fill in with your own definitions if you want).

#define _CRT_SECURE_NO_WARNINGS
#define _CRT_NONSTDC_NO_DEPRECATE

typedef struct v2 { float x, y; } v2;
#define V2(x,y) ((v2){ x, y })
#define dyna
#define sdyna
#define UNUSED(x) (void)x
#define sintern strdup // omg so hacky

#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>

#if defined(_MSC_VER)
#include <malloc.h>
#	define alloca _alloca
#else
#	include <alloca.h>
#endif

#define FREE free
#define MALLOC malloc
#define CALLOC(T) calloc(sizeof(T), 1)

// File IO.
#define CF_File FILE
#define read(file, ptr, size) fread(ptr, size, 1, file)
#define write(file, ptr, size) fwrite(ptr, size, 1, file)
#define close fclose
#define open_for_read(path) fopen(path, "rb")
#define open_for_write(path) fopen(path, "wb")

// Dynamic array.
#define asize(a)      ((a) ? ARRAY_HEADER(a)->len : 0)
#define acap(a)       ((a) ? ARRAY_HEADER(a)->cap : 0)
#define afree(a)      ((a) ? (free(ARRAY_HEADER(a)), (a)=NULL, 0) : 0)
#define apush(a, val) (array__maybegrow(a, 1), (a)[ARRAY_HEADER(a)->len++] = (val))
#define afit(a, n)    (array__maybegrow((a), (n) - asize(a)))
#define alen(a)       (ARRAY_HEADER(a)->len)

#define ARRAY_HEADER(a)    ((ArrayHeader*)a - 1)

typedef struct ArrayHeader
{
	int len;
	int cap;
} ArrayHeader;

static void* array_grow_impl(void* buf, size_t element_size, size_t add)
{
	size_t new_len = buf ? ARRAY_HEADER(buf)->len + add : add;
	size_t new_cap = buf ? ARRAY_HEADER(buf)->cap * 2 : 16;
	if (new_cap < new_len) new_cap = new_len;
	size_t size = sizeof(ArrayHeader) + new_cap * element_size;
	ArrayHeader* new_hdr;
	if (buf) {
		new_hdr = (ArrayHeader*)realloc(ARRAY_HEADER(buf), size);
	} else {
		new_hdr = (ArrayHeader*)malloc(size);
		new_hdr->len = 0;
	}
	new_hdr->cap = (int)new_cap;
	return (void*)(new_hdr + 1);
}
#define array__needgrow(a, n)  ((a)==NULL || ARRAY_HEADER(a)->len + (n) > ARRAY_HEADER(a)->cap)
#define array__maybegrow(a, n) (array__needgrow(a, (n)) ? (*(void**)&(a)=array_grow_impl(a, sizeof(*(a)), n)) : 0)

//--------------------------------------------------------------------------------------------------
// SV - Save Version
// This file has an API for saving binary data with versioning support for backwards compatibility.
// Original API design by Media Molecule.
// See: https://gist.githubusercontent.com/OswaldHurlem/4810ad510669097db872c6de305c9df0/raw/2fdf47eead527e954d29950aa41debf34547e5bd/mmalex_serialization_and_formats.log
// 
// Design specs:
// + Very fast reads/writes
// + Backwards compat
// - Not self-describing (serializes opaque data)
// + The code itself describes the data, and versioning, all in one place
// - Not forward tolerant (cannot open newer data version with older application)
// - Uncompressed in serialized form (can do this with an external tool trivially)
// - Doesn't scale well to double digit team size (basically: merge conflicts)
// + Extremely simple (just ~500 loc here, including comments)
// - Slightly error prone with versioning (but simple to notice + fix)
// 
// Steps to use:
// 1. Add the type you want to serialize to SV_TYPES table, or to SV_MEMCPY_SAFE_TYPES.
//    - Note: SV_MEMCPY_SAFE_TYPES can only be used on types you *will not change*. There is *no*
//            backwards compatibility for these types, but, as an optimization they will get memcpy'd
//            to disk in a single call, even for arrays of this type. Recommended for vector, transform,
//            or other fundamental types that will never change.
// 
// 2. Increment the version enum. You will increment this enum every single time you modify a serialization
//    routine. This is how the versioning system works -- with a global monotincally incrementing version id.
// 
// 3. Create the serialization routine for your type.
//    Example:
//    
//        typedef struct ExampleData
//        {
//            int a;
//            float b;
//            const char* c;
//        };
//        
//        SV_SERIALIZE(ExampleData)
//        {
//            SV_ADD(SV_ADDED_EXAMPLE_DATA, a);
//            SV_ADD(SV_ADDED_EXAMPLE_DATA, b);
//            SV_ADD(SV_ADDED_EXAMPLE_DATA, c);
//        }
// 
//    Where `SV_ADDED_EXAMPLE_DATA` is the new enum version when `ExampleData` was created. Increment it:
// 
//        enum
//        {
//            SV_INITIAL,
//            SV_ADDED_EXAMPLE_DATA, // <-- Newly added.
//            // --
//            SV_LATEST_PLUS_ONE
//        };
// 
//    Then, if we want to remove a member, we use `SV_REM`:
// 
//        typedef struct ExampleData
//        {
//            int a;
//            float b;
//            // const char* c; <-- Removed in SV_REMOVE_C_FROM_EXAMPLE_DATA.
//        };
//        
//        SV_SERIALIZE(ExampleData)
//        {
//            SV_ADD(SV_ADDED_EXAMPLE_DATA, a);
//            SV_ADD(SV_ADDED_EXAMPLE_DATA, b);
//            //SV_ADD(SV_ADDED_EXAMPLE_DATA, c);
//            SV_REM(SV_ADDED_EXAMPLE_DATA, SV_REMOVE_C_FROM_EXAMPLE_DATA, const char*, c, NULL);
//            // If-needed you can use local variable c for migrating to new fields later (not needed here).
//        }
// 
//    If you want to migrate an old value to something new, you can write whatever custom code you want to
//    do so (you have access to the object `o` being serialized):
// 
//        typedef struct ExampleData
//        {
//            int a;
//            float b;
//            // const char* c; <-- Removed.
//            const char* d;
//        };
//        
//        SV_SERIALIZE(ExampleData)
//        {
//            SV_ADD(SV_ADDED_EXAMPLE_DATA, a);
//            SV_ADD(SV_ADDED_EXAMPLE_DATA, b);
//            //SV_ADD(SV_ADDED_EXAMPLE_DATA, c);
//            SV_REM(SV_ADDED_EXAMPLE_DATA, SV_REMOVE_C_FROM_EXAMPLE_DATA, const char*, c, NULL);
//            o->d = c; // Copy over the missing field (essentially just renaming c to d).
//            SV_ADD(SV_ADDED_EXAMPLE_DATA_D, d);
//        }
// 
// Usually that's it! But, if you want to create a new kind of file:
// 
// 4. Open a file with either `SV_SAVE_BEGIN` or `SV_LOAD_BEGIN`. Call `SV_ADD_LOCAL` to recursively serialize.
//    When done, `call `SV_SAVE_END` or `SV_LOAD_END`.
//    Example:
// 
//        void save(const char* path, ExampleData data)
//        {
//            SV_SAVE_BEGIN(path);
//            
//            SV_ADD_LOCAL(SV_ADDED_EXAMPLE_DATA, data);
//            
//            SV_SAVE_END();
//        }
// 
// Other rules:
// - All strings coming out of the serialization layer (when reading) are unique stable strings from
//   `sintern` string interning.
// - If you use `SV_ADD_LIST` objects will be allocated with calloc (cleared to zero) and handed back
//   to you. You should design your object for valid zero-initialization to play nicely here, or, run
//   initialization after serialization. Otherwise, redesign your data layout to flattened arrays and
//   simply call `SV_ADD_ARRAY`.

//--------------------------------------------------------------------------------------------------
// Some temporary test types (delete these).

typedef struct Inner
{
	int a;
	float b;
} Inner;

typedef struct Outer
{
	int x;
	Inner inner;
	v2 point;
	dyna v2* points;
	sdyna const char* s;
	dyna const char** s_array;
} Outer;

typedef struct Object
{
	int data;
} Object;

typedef struct ObjectList
{
	dyna Object** objects;
} ObjectList;

//--------------------------------------------------------------------------------------------------
// Public API.

// Global version id. Increment this each time you alter any serialiation routines.
// ...Add to the end, but make sure `SV_LATEST_PLUS_ONE` is last.
enum
{
	SV_INITIAL,
	// --
	SV_LATEST_PLUS_ONE
};

#define SV_LATEST (SV_LATEST_PLUS_ONE - 1)

// Extend this table whenever you add a new type to serialize.
// ...For any type that's memcpy safe add it below to the `SV_MEMCPY_SAFE_TYPES` table.
#define SV_TYPES(X) \
	X(Inner) \
	X(Outer) \
	X(Object) \
	X(ObjectList) \

// As an optimization we place memcpy-safe types here.
#define SV_MEMCPY_SAFE_TYPES(X) \
	X(v2) \

// Intended use pattern:
//    void save(const char* path, ExampleData data)
//    {
//        SV_SAVE_BEGIN(path);
//        
//        SV_ADD_LOCAL(SV_ADDED_EXAMPLE_DATA, data);
//        
//        SV_SAVE_END();
//    }
#define SV_SAVE_BEGIN(path) SV_Context ctx = sv_make(path, true), *S = &ctx;
#define SV_SAVE_END() sv_destroy(S)
#define SV_LOAD_BEGIN(path) SV_Context ctx = sv_make(path, false), *S = &ctx;
#define SV_LOAD_END() sv_destroy(S)

// Add a struct member.
#define SV_ADD(VERSION, MEMBER)                           \
	do {                                                  \
		if (S->saving) {                                  \
			SV_WRITE(o->MEMBER);                          \
		} else if (S->loading && S->version >= VERSION) { \
			SV_READ(o->MEMBER);                           \
		}                                                 \
	} while (0)

// Add a local variable.
// ...Does not support arrays.
#define SV_ADD_LOCAL(VERSION, LOCAL_VAR)                  \
	do {                                                  \
		if (S->saving) {                                  \
			SV_WRITE(LOCAL_VAR);                          \
		} else if (S->loading && S->version >= VERSION) { \
			SV_READ(LOCAL_VAR);                           \
		}                                                 \
	} while (0)

// Add an array (as a struct member).
#define SV_ADD_ARRAY(VERSION, ARRAY)                              \
	do {                                                          \
		if (S->saving || S->loading && S->version >= VERSION) {   \
			if (SV_IS_MEMCPY_SAFE(o->ARRAY)) {                    \
				if (S->saving) {                                  \
					SV_WRITE(o->ARRAY);                           \
				} else if (S->loading && S->version >= VERSION) { \
					SV_READ(o->ARRAY);                            \
				}                                                 \
			} else {                                              \
				int n = asize(o->ARRAY);                          \
				SV_ADD_LOCAL(VERSION, n);                         \
				afit(o->ARRAY, n);                                \
				alen(o->ARRAY) = n;                               \
				for (int i = 0; i < n; ++i) {                     \
					if (S->saving) {                              \
						SV_WRITE(o->ARRAY[i]);                    \
					} else {                                      \
						SV_READ(o->ARRAY[i]);                     \
					}                                             \
				}                                                 \
			}                                                     \
		}                                                         \
	} while (0)

// Same as `SV_ADD_ARRAY` but works for arrays of pointers, such as: struct ObjectList { dyna Object** objects; };
#define SV_ADD_LIST(VERSION, ARRAY_OF_PTRS)                      \
	do {                                                         \
		int n = asize(o->ARRAY_OF_PTRS);                         \
		SV_ADD_LOCAL(VERSION, n);                                \
		afit(o->ARRAY_OF_PTRS, n);                               \
		alen(o->ARRAY_OF_PTRS) = n;                              \
		for (int i = 0; i < n; ++i) {                            \
			if (S->loading) {                                    \
				void* v = CALLOC(o->ARRAY_OF_PTRS[0][0]);        \
				memcpy(o->ARRAY_OF_PTRS + i, &v, sizeof(void*)); \
			}                                                    \
			SV_ADD(VERSION, ARRAY_OF_PTRS[i][0]);                \
		}                                                        \
	} while (0)

// Remove something from the serialization. This needs to increment the global version and passed in as `VERSION_REMOVED`.
// ...T is the type of the value removed.
// ...A local variable called `NAME` is created and data is read into it. You can then freely
//    use this local variable to handle conversion to your newer format (if-needed).
// ...DEFAULT is a default value to use for the removed value in case it isn't present in the file version.
// ...Remove the prior `SV_ADD` for the removed data.
#define SV_REM(VERSION_ADDED, VERSION_REMOVED, T, NAME, DEFAULT)                     \
	T NAME = (DEFAULT);                                                              \
	if (S->loading && S->version >= VERSION_ADDED && S->version < VERSION_REMOVED) { \
		SV_READ(NAME);                                                               \
	}

// Define a serialization routine for a user struct.
#define SV_SERIALIZABLE(T) void serialize_##T(SV_Context* S, T* o)

// Optional sync check. Asserts if serialization is out of sync.
// ...Place this at the end of a `SV_SERIALIZABLE` routine to narrow down issues.
// ...Will assert when loading to catch (likely) bugs when saving.
// ...This adds an int counter to your struct, so you *must* bump global version enum.
// Common issues:
// - Missing ADD/REM
// - Wrong version specified
// - Field reordered/removed
// - Wrong array count/loop
// - Struct changed w/o bumping version
#define SV_SYNC()                                   \
	do {                                            \
		int e = S->sync_counter;                    \
		if (S->saving) { SV_WRITE(e); }             \
		else { int g; SV_READ(g); assert(g == e); } \
		S->sync_counter++;                          \
	} while (0)

//--------------------------------------------------------------------------------------------------
// Private implementation details.

// Tells whether a type can be safe memcpy'd. Used for optimizing serialization for certain types.
// Arrays of these types can be dumped all at once.
#define SV_MEMCPY_TYPE(T) T: true,
#define SV_IS_MEMCPY_SAFE(T)                 \
	_Generic(T,                              \
		SV_MEMCPY_SAFE_TYPES(SV_MEMCPY_TYPE) \
		default: false                       \
	)

// Create declarations for all serializable types for compilation simplicity.
#define SV_SERIALIZE_DECL(T) void serialize_##T(struct SV_Context* S, T* o);
SV_TYPES(SV_SERIALIZE_DECL)

// Context struct passed through all serialization routines.
typedef struct SV_Context
{
	int version;
	bool saving;
	bool loading;
	const char* path;
	CF_File* file;
	int sync_counter;
} SV_Context;

// Create definitions for functions to dump full arrays for the types in SV_MEMCPY_SAFE_TYPES.
#define SV_SERIALIZE_BY_MEMCPY(T)                \
void serialize_##T(SV_Context* S, T* o)          \
{                                                \
	if (S->saving) write(S->file, o, sizeof(T)); \
	else read(S->file, o, sizeof(T));            \
}                                                \
void serialize_##T##_array(SV_Context* S, T** a) \
{                                                \
	T* o = *a;                                   \
	if (S->saving) {                             \
		int sz = asize(o) * sizeof(T);           \
		write(S->file, &sz, sizeof(sz));         \
		write(S->file, o, sz);                   \
	} else {                                     \
		int sz;                                  \
		read(S->file, &sz, sizeof(sz));          \
		int n = sz / sizeof(T);                  \
		afit(o, n);                              \
		alen(o) = n;                             \
		read(S->file, o, sz);                    \
	}                                            \
	*a = o;                                      \
}
SV_MEMCPY_SAFE_TYPES(SV_SERIALIZE_BY_MEMCPY)

// Internally used read/write function overloads for all types.
#define SV_TYPE(T) T: serialize_##T,
#define SV_MTYPE(T) T: serialize_##T, T*: serialize_##T##_array,
#define SV_READ(V)                      \
	_Generic(V,                         \
		SV_TYPES(SV_TYPE)               \
		SV_MEMCPY_SAFE_TYPES(SV_MTYPE)  \
		uint8_t:      read_uint8,       \
		uint16_t:     read_uint16,      \
		uint32_t:     read_uint32,      \
		uint64_t:     read_uint64,      \
		int8_t:       read_int8,        \
		int16_t:      read_int16,       \
		int32_t:      read_int32,       \
		int64_t:      read_int64,       \
		bool:         read_bool,        \
		float:        read_float,       \
		double:       read_double,      \
		const char*:  read_string,      \
		char*:        read_string,      \
		const char**: read_string_noop, \
		char**:       read_string_noop  \
	)(S, &V)
#define SV_WRITE(V)                      \
	_Generic(V,                          \
		SV_TYPES(SV_TYPE)                \
		SV_MEMCPY_SAFE_TYPES(SV_MTYPE)   \
		uint8_t:      write_uint8,       \
		uint16_t:     write_uint16,      \
		uint32_t:     write_uint32,      \
		uint64_t:     write_uint64,      \
		int8_t:       write_int8,        \
		int16_t:      write_int16,       \
		int32_t:      write_int32,       \
		int64_t:      write_int64,       \
		bool:         write_bool,        \
		float:        write_float,       \
		double:       write_double,      \
		const char*:  write_string,      \
		char*:        write_string,      \
		const char**: write_string_noop, \
		char**:       write_string_noop  \
	)(S, &V)

void read_uint8(SV_Context* S, uint8_t* v) { read(S->file, v, sizeof(*v)); }
void read_uint16(SV_Context* S, uint16_t* v) { read(S->file, v, sizeof(*v)); }
void read_uint32(SV_Context* S, uint32_t* v) { read(S->file, v, sizeof(*v)); }
void read_uint64(SV_Context* S, uint64_t* v) { read(S->file, v, sizeof(*v)); }
void read_int8(SV_Context* S, int8_t* v) { read(S->file, v, sizeof(*v)); }
void read_int16(SV_Context* S, int16_t* v) { read(S->file, v, sizeof(*v)); }
void read_int32(SV_Context* S, int32_t* v) { read(S->file, v, sizeof(*v)); }
void read_int64(SV_Context* S, int64_t* v) { read(S->file, v, sizeof(*v)); }
void read_bool(SV_Context* S, bool* v) { uint8_t t; read(S->file, &t, sizeof(t)); *v = t != 0; }
void read_float(SV_Context* S, float*  v) { read(S->file, v, sizeof(*v)); }
void read_double(SV_Context* S, double* v) { read(S->file, v, sizeof(*v)); }
void read_string(SV_Context* S, const char** out) { uint32_t len; read_uint32(S, &len); char* buf = (char*)alloca(len+1); read(S->file, buf, len); buf[len] = 0; *out = sintern(buf); }
void read_string_noop(SV_Context* S, const char*** out) { UNUSED(S); UNUSED(out); assert(!"SV_ADD_ARRAY performs array loop itself, this is just here to compile."); }

void write_uint8(SV_Context* S, const uint8_t* v) { write(S->file, v, sizeof(*v)); }
void write_uint16(SV_Context* S, const uint16_t* v) { write(S->file, v, sizeof(*v)); }
void write_uint32(SV_Context* S, const uint32_t* v) { write(S->file, v, sizeof(*v)); }
void write_uint64(SV_Context* S, const uint64_t* v) { write(S->file, v, sizeof(*v)); }
void write_int8(SV_Context* S, const int8_t* v) { write(S->file, v, sizeof(*v)); }
void write_int16(SV_Context* S, const int16_t* v) { write(S->file, v, sizeof(*v)); }
void write_int32(SV_Context* S, const int32_t* v) { write(S->file, v, sizeof(*v)); }
void write_int64(SV_Context* S, const int64_t* v) { write(S->file, v, sizeof(*v)); }
void write_bool(SV_Context* S, const bool* v) { uint8_t b = *v ? 1 : 0; write(S->file, &b, sizeof(b)); }
void write_float(SV_Context* S, const float* v) { write(S->file, v, sizeof(*v)); }
void write_double(SV_Context* S, const double* v) { write(S->file, v, sizeof(*v)); }
void write_string(SV_Context* S, const char** v) { uint32_t len = (uint32_t)strlen(*v); write_uint32(S, &len); write(S->file, *v, len); }
void write_string_noop(SV_Context* S, const char*** out) { UNUSED(S); UNUSED(out); assert(!"SV_ADD_ARRAY performs array loop itself, this is just here to compile."); }

SV_Context sv_make(const char* path, bool saving)
{
	SV_Context ctx = { 0 };
	SV_Context* S = &ctx;
	ctx.saving  = saving;
	ctx.loading = !saving;
	ctx.path = sintern(path);
	if (saving) {
		ctx.file = open_for_write(path);
		assert(ctx.file);
		ctx.version = SV_LATEST;
		SV_WRITE(ctx.version);
	} else {
		ctx.file = open_for_read(path);
		assert(ctx.file);
		SV_READ(ctx.version);
	}
	return ctx;
}

void sv_destroy(SV_Context* S)
{
	close(S->file);
}

//--------------------------------------------------------------------------------------------------
// Temporary test cases (delete these).

SV_SERIALIZABLE(Inner)
{
	SV_ADD(SV_INITIAL, a);
	SV_ADD(SV_INITIAL, b);
}

SV_SERIALIZABLE(Outer)
{
	SV_ADD(SV_INITIAL, x);
	SV_ADD(SV_INITIAL, inner);
	SV_ADD(SV_INITIAL, point);
	SV_ADD_ARRAY(SV_INITIAL, points);
	SV_ADD(SV_INITIAL, s);
	SV_ADD_ARRAY(SV_INITIAL, s_array);
	SV_SYNC();
}

SV_SERIALIZABLE(Object)
{
	SV_ADD(SV_INITIAL, data);
}

SV_SERIALIZABLE(ObjectList)
{
	SV_ADD_LIST(SV_INITIAL, objects);
}

void serialize_test()
{
	{
		SV_SAVE_BEGIN("test.bin");

		Outer a = { 0 };
		a.x = 5;
		a.inner.a = 2;
		a.inner.b = 3;
		a.point = V2(-1,-1);
		a.points = NULL;
		a.s = "test";
		apush(a.s_array, "element 0");
		apush(a.s_array, "element 1");
		apush(a.s_array, "element 2");
		apush(a.points, V2(1,2));
		apush(a.points, V2(3,4));
		SV_ADD_LOCAL(SV_INITIAL, a);
		afree(a.s_array);
		afree(a.points);

		SV_SAVE_END();
	}
	{
		SV_LOAD_BEGIN("test.bin");

		Outer a = { 0 };
		SV_ADD_LOCAL(SV_INITIAL, a);
		v2 pts[2] = { a.points[0], a.points[1] };
		const char* strings[3] = { a.s_array[0], a.s_array[1], a.s_array[2] };
		afree(a.s_array);
		afree(a.points);

		SV_LOAD_END();
	}
	{
		SV_SAVE_BEGIN("object.bin");

		dyna Object** objects = NULL;
		apush(objects, (Object*)MALLOC(sizeof(Object)));
		apush(objects, (Object*)MALLOC(sizeof(Object)));
		objects[0]->data = 1;
		objects[1]->data = 2;
		ObjectList list = { objects };
		SV_ADD_LOCAL(SV_INITIAL, list);
		FREE(objects[0]);
		FREE(objects[1]);
		afree(objects);

		SV_SAVE_END();
	}
	{
		SV_LOAD_BEGIN("object.bin");

		ObjectList list = { 0 };
		SV_ADD_LOCAL(SV_INITIAL, list);
		Object* o0 = list.objects[0];
		Object* o1 = list.objects[1];
		FREE(list.objects[0]);
		FREE(list.objects[1]);
		afree(list.objects);

		SV_SAVE_END();
	}
}

//--------------------------------------------------------------------------------------------------
// Demo program.

int main()
{
	serialize_test();

	return 0;
}