Make progress towards the new builtin asset compiler

Attempts to compile HLSL shaders (with includes)

assets/shader/cell.as

@@ -1 +0,0 @@
-package pipelines.pkg;

assets/shader/cell.pipeline

@@ -1,13 +0,0 @@
-optimization speed;
-
-vertex {
-    vk assets/shader/cell_vert.glsl;
-}
-fragment {
-    vk assets/shader/cell_frag.glsl;
-}
-
-texture_bindings {
-    0 MATERIAL_ALBEDO;
-    1 MATERIAL_NORMAL;
-}

assets/shader/cell_frag.as

@@ -1,2 +0,0 @@
-package shaders.pkg;
-processing_flags 0002;

assets/shader/cell_frag.glsl

@@ -1,8 +0,0 @@
-#version 450
-#pragma shader_stage(fragment)
-
-layout (location = 0) out vec3 color;
-
-void main() {
-    color = vec3(1, 1, 1);
-}

assets/shader/cell_vert.as

@@ -1,2 +0,0 @@
-package shaders.pkg;
-processing_flags 0001;

assets/shader/cell_vert.glsl

@@ -1,6 +0,0 @@
-#version 450
-#pragma shader_stage(vertex)
-
-void main() {
-    gl_Position = vec4(0, 0, 0, 1);
-}

assets/shader/test.as

@@ -1 +0,0 @@
-package pipelines.pkg;

assets/shader/test.hlsl

@@ -0,0 +1 @@
+void TestFn() {}

assets/shader/test.pipeline

@@ -1,7 +1,10 @@
 optimization speed;
 
-vertex BEGIN
+vertex {
-layout (location = 0) uniform vec3 foo;
+	vk BEGIN
-void main() {
+#include "test.hlsl"
+
+void VsMain() {
+}
+	END
 }
-END

meson.build

@@ -1,5 +1,10 @@
 project('voyage', 'c',
-  default_options: ['buildtype=debug', 'b_sanitize=address', 'c_std=c17', 'warning_level=3'])
+  default_options: ['buildtype=debug', 
+    'b_sanitize=address', 
+    'c_std=c17', 
+    'warning_level=3',
+    'werror=true',
+    'b_vscrt=static_from_buildtype'])
 
 compiler = meson.get_compiler('c')
 buildtype = get_option('buildtype')
@@ -51,63 +56,90 @@ runtime_incdirs = common_incdirs
 runtime_linkargs = []
 runtime_additional_sources = []
 runtime_cargs = []
+runtime_deps = [thread_dep, m_dep, windowing_dep]
+
 if get_option('build_asset_compiler')
   runtime_cargs += ['-DRT_BUILD_ASSET_COMPILER']
 
   # Shaderc for shaders
-  shaderc_include = include_directories('contrib/shaderc/libshaderc/include')
+  if get_option('enable_vulkan_shader_compiler')
-  shaderc_libdir = 'NONE'
+    shaderc_include = include_directories('contrib\\shaderc\\libshaderc\\include')
-  if host_machine.system() == 'windows'
+    shaderc_libdir = 'NONE'
-    shaderc_libdir = meson.project_source_root() / 'contrib/shaderc/build-win/libshaderc/Release'
+    if host_machine.system() == 'windows'
+      if buildtype == 'debug' or buildtype == 'debugoptimized'
+        shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Debug'
+      else
+        shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Release'
+      endif
+    endif
+    shaderc_dep = declare_dependency(link_args : ['-L'+shaderc_libdir, '-lshaderc_combined'],
+      include_directories : shaderc_include)
+    runtime_deps += shaderc_dep
+    runtime_additional_sources += [
+      'src/runtime/vulkan_shader_compiler.c'
+    ]
+    runtime_cargs += ['-DRT_BUILD_VULKAN_SHADER_COMPILER']
   endif
-  runtime_incdirs += shaderc_include
+
-  runtime_linkargs += ['-L'+shaderc_libdir, '-lshaderc_combined']
+  # Asset compiler sources
+  runtime_additional_sources += [
+    'src/runtime/asset_compiler.h',
+    'src/runtime/description_parser.h',
+    'src/runtime/shader_compiler.h',
+
+    'src/runtime/asset_compiler.c',
+    'src/runtime/description_parser.c',
+    'src/runtime/pipeline_processor.c',
+    'src/runtime/shader_compiler.c',
+  ]
 endif
 
 runtime_lib = library('rt',
   # Project Sources
   'src/runtime/aio.h',
   'src/runtime/app.h',
-  'src/runtime/asset_dependencies.h',
   'src/runtime/assets.h',
   'src/runtime/buffer_manager.h',
   'src/runtime/config.h',
   'src/runtime/dynamic_libs.h',
   'src/runtime/file_tab.h',
+  'src/runtime/fsutils.h',
   'src/runtime/gfx.h',
   'src/runtime/handles.h',
+  'src/runtime/hashing.h',
   'src/runtime/jobs.h',
+  'src/runtime/mem_arena.h',
   'src/runtime/packages.h',
   'src/runtime/renderer_api.h',
   'src/runtime/runtime.h',
   'src/runtime/threading.h',
-  'src/runtime/uidtab.h',
   
   'src/runtime/aio.c',
   'src/runtime/app.c',
-  'src/runtime/asset_cache.c',
+  'src/runtime/asset_manager.c',
-  'src/runtime/asset_dependencies.c',
-  'src/runtime/asset_loading.c',
   'src/runtime/buffer_manager.c',
   'src/runtime/config.c',
   'src/runtime/dynamic_libs.c',
   'src/runtime/error_report.c',
   'src/runtime/file_tab.c',
+  'src/runtime/fsutils.c',
   'src/runtime/gfx_main.c',
+  'src/runtime/hashing.c',
   'src/runtime/init.c',
   'src/runtime/jobs.c',
+  'src/runtime/mem_arena.c',
   'src/runtime/packages.c',
   'src/runtime/text.c',
   'src/runtime/threading_cond.c',
   'src/runtime/threading_mutex.c',
   'src/runtime/threading_rwlock.c',
   'src/runtime/threading_thread.c',
-  'src/runtime/uidtab.c',
 
   # Contrib Sources
   'contrib/xxhash/xxhash.c',
   'contrib/lz4/lz4.c',
-  dependencies : [thread_dep, m_dep, windowing_dep],
+  sources : runtime_additional_sources,
+  dependencies : runtime_deps,
   include_directories : runtime_incdirs,
   link_args : runtime_linkargs,
   c_args : runtime_cargs,

meson_options.txt

@@ -1,3 +1,4 @@
 option('use_xlib', type : 'boolean', value : false, description : 'Use Xlib for window creation under linux')
 option('error_report_debugbreak', type : 'boolean', value : true, description : 'Debugbreak in ReportError')
 option('build_asset_compiler', type : 'boolean', value : true, description : 'Enables or disables the asset compiler inside runtime.')
+option('enable_vulkan_shader_compiler', type : 'boolean', value : true, description : 'Enables building the vulkan shader compiler (Requires shaderc).')

scripts/download_shaderc.bat

@@ -28,6 +28,7 @@ MKDIR build-win
 PUSHD build-win
 cmake ..
 cmake --build . --config Release
+cmake --build . --config Debug
 POPD 
 
 

src/runtime/aio.c

@@ -6,6 +6,8 @@
 #define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 
+void Win32ErrorToString(DWORD last_error, char *out, int bufsize);
+
 #elif defined(__linux__)
 #include <sched.h>
 
@@ -183,10 +185,14 @@ RT_DLLEXPORT rt_result rtSubmitLoadBatch(const rt_load_batch *batch, rt_aio_hand
                                          FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED,
                                          NULL);
         if (file_handle == INVALID_HANDLE_VALUE) {
+            DWORD err = GetLastError();
+            char error_msg[256];
+            Win32ErrorToString(err, error_msg, 256);
             rtReportError("aio",
-                          "CreateFileW failed for file: %s with error code: %u",
+                          "CreateFileW failed for file: %s with error code: %u  (%s)",
                           file_path,
-                          GetLastError());
+                          err,
+                          error_msg);
             op->state  = RT_AIO_STATE_INVALID;
             handles[i] = RT_AIO_INVALID_HANDLE;
             continue;
@@ -199,7 +205,9 @@ RT_DLLEXPORT rt_result rtSubmitLoadBatch(const rt_load_batch *batch, rt_aio_hand
                                  win32CompletionRoutine);
         DWORD err       = GetLastError();
         if (!result || err != ERROR_SUCCESS) {
-            rtReportError("aio", "ReadFileEx failed with error code: %u", err);
+            char error_msg[256];
+            Win32ErrorToString(err, error_msg, 256);
+            rtReportError("aio", "ReadFileEx failed with error code: %u  (%s)", err, error_msg);
             op->state  = RT_AIO_STATE_FINISHED;
             handles[i] = RT_AIO_INVALID_HANDLE;
             CloseHandle(file_handle);

src/runtime/asset_cache.c

@@ -1,379 +0,0 @@
-#include "assets.h"
-#include "asset_dependencies.h"
-#include "aio.h"
-#include "buffer_manager.h"
-#include "config.h"
-#include "threading.h"
-#include "uidtab.h"
-
-#include <assert.h>
-#include <stdlib.h>
-
-RT_CVAR_I(rt_AssetCacheSize, "Number of asset cache entries. Default: 1024.", 1024);
-
-/* asset_loading.c */
-extern rt_result DecompressAsset(void *compressed_buffer,
-                                 size_t compressed_buffer_size,
-                                 void **p_decompressed,
-                                 size_t *p_decompressed_size);
-
-typedef enum {
-    CACHE_ENTRY_STATE_FREE,
-    CACHE_ENTRY_STATE_LOADING,
-    CACHE_ENTRY_STATE_LOADED,
-} rt_asset_cache_entry_state;
-
-typedef struct rt_asset_cache_entry_s {
-    rt_asset_cache_entry_state state;
-    rt_aio_handle load;
-    void *buffer;
-    size_t size;
-    int refcount;
-
-    /* Reclaim list */
-    struct rt_asset_cache_entry_s *prev_reclaim;
-    struct rt_asset_cache_entry_s *next_reclaim;
-} rt_asset_cache_entry;
-
-static rt_uid *_uids;
-static rt_asset_cache_entry *_entries;
-static rt_asset_cache_entry *_first_reclaim;
-static rt_asset_cache_entry *_last_reclaim;
-
-/* Locked as writer when modifiying entries, as reader when searching */
-static rt_rwlock _lock;
-
-rt_result InitAssetCache(void) {
-    _entries = calloc((size_t)rt_AssetCacheSize.i, sizeof(rt_asset_cache_entry));
-    if (!_entries) {
-        return RT_BUFFER_ALLOC_FAILED;
-    }
-    _uids = calloc((size_t)rt_AssetCacheSize.i, sizeof(rt_uid));
-    if (!_uids) {
-        free(_entries);
-        return RT_BUFFER_ALLOC_FAILED;
-    }
-    rt_create_rwlock_result lock_res = rtCreateRWLock();
-    if (!lock_res.ok) {
-        free(_entries);
-        free(_uids);
-        return RT_UNKNOWN_ERROR;
-    }
-    _lock = lock_res.lock;
-    return RT_SUCCESS;
-}
-
-void ShutdownAssetCache(void) {
-    free(_entries);
-    free(_uids);
-    rtDestroyRWLock(&_lock);
-    _first_reclaim = NULL;
-    _last_reclaim  = NULL;
-}
-
-static void ReleaseEntry(rt_asset_cache_entry *entry) {
-    if (entry->load != RT_AIO_INVALID_HANDLE) {
-        rtWaitForAIOCompletion(entry->load);
-        rtReleaseAIO(entry->load);
-        entry->load = RT_AIO_INVALID_HANDLE;
-    }
-    rtReleaseBuffer(entry->buffer, entry->size);
-    entry->buffer       = NULL;
-    entry->size         = 0;
-    entry->next_reclaim = NULL;
-    entry->prev_reclaim = NULL;
-}
-
-static void GarbageCollect(void) {
-    rtLockWrite(&_lock);
-    rt_asset_cache_entry *entry = _first_reclaim;
-    while (entry) {
-        assert(entry->refcount == 0);
-        rt_asset_cache_entry *next = entry->next_reclaim;
-        if (entry->state == CACHE_ENTRY_STATE_LOADED) {
-            ReleaseEntry(entry);
-            _first_reclaim = next;
-        }
-        entry          = next;
-    }
-    rtUnlockWrite(&_lock);
-}
-
-static rt_asset_cache_entry *GetEntry(rt_uid uid) {
-    /* Hash lookup */
-    unsigned int mod = (unsigned int)rt_AssetCacheSize.i - 1;
-    for (unsigned int i = 0; i < (unsigned int)rt_AssetCacheSize.i; ++i) {
-        unsigned int slot = (uid + i) & mod;
-        if (_uids[slot] == uid) {
-            return &_entries[slot];
-        } else if (_uids[slot] == RT_INVALID_UID) {
-            break;
-        }
-    }
-    return NULL;
-}
-
-
-static bool IsAssetLoaded(rt_uid uid) {
-    const rt_asset_cache_entry *entry = GetEntry(uid);
-    if (entry)
-        return entry->state == CACHE_ENTRY_STATE_LOADED ||
-               entry->state == CACHE_ENTRY_STATE_LOADING;
-    else
-        return false;
-}
-
-static int InsertEntry(rt_uid uid) {
-    unsigned int mod = (unsigned int)rt_AssetCacheSize.i - 1;
-    for (unsigned int i = 0; i < (unsigned int)rt_AssetCacheSize.i; ++i) {
-        unsigned int slot = (uid + i) & mod;
-        if (_uids[slot] == 0) {
-            return (int)slot;
-        }
-    }
-    return -1;
-}
-static rt_result InsertAndLoadAssets(const rt_uid *uids, size_t count) {
-    rt_load_batch batch = {.num_loads = 0};
-
-    rt_result res = RT_SUCCESS;
-
-    count = (count < RT_LOAD_BATCH_MAX_SIZE) ? count : RT_LOAD_BATCH_MAX_SIZE;
-    rt_asset_cache_entry *load_entries[RT_LOAD_BATCH_MAX_SIZE];
-
-    for (size_t i = 0; i < count; ++i) {
-        rtLockRead(&_lock);
-        bool needs_load = !IsAssetLoaded(uids[i]);
-        rtUnlockRead(&_lock);
-
-        if (!needs_load)
-            continue;
-
-        rtLockWrite(&_lock);
-        /* It's possible that another thread loaded the asset in the meantime */
-        if (!IsAssetLoaded(uids[i])) {
-            const rt_uid_data *data = rtGetUIDData(uids[i]);
-            if (!data) {
-                rtUnlockWrite(&_lock);
-                rtLog("ASSET_CACHE", "Failed to get uid data for uid %u", uids[i]);
-                res = RT_UNKNOWN_ASSET;
-                continue;
-            }
-
-            void *compressed_data = rtAllocBuffer(data->size);
-            if (!compressed_data) {
-                /* Try again after garbage collection */
-                rtUnlockWrite(&_lock);
-                GarbageCollect();
-                compressed_data = rtAllocBuffer(data->size);
-                if (!compressed_data) {
-                    rtLog("ASSET_CACHE",
-                          "Failed to allocate intermediate buffer for uid %u",
-                          uids[i]);
-                    res = RT_BUFFER_ALLOC_FAILED;
-                    continue;
-                }
-                rtLockWrite(&_lock);
-            }
-
-            int slot = InsertEntry(uids[i]);
-            if (slot == -1) {
-                rtUnlockWrite(&_lock);
-                rtLog("ASSET_CACHE", "Failed to insert new entry for uid %u", uids[i]);
-                res = RT_ASSET_CACHE_FULL;
-                break;
-            }
-            
-            rt_asset_cache_entry *entry = &_entries[slot];
-            load_entries[batch.num_loads] = entry;
-
-            /* We set the refcount to 0, but don't insert the entry
-             * into the reclaim list, to ensure that its buffer does not get freed
-             * while the load still executes. Setting the refcount to 0 ensures
-             * that the count is correct, once the asset is accessed the first time. */
-            entry->state        = CACHE_ENTRY_STATE_LOADING;
-            entry->refcount     = 0;
-            entry->buffer       = compressed_data;
-            entry->size         = data->size;
-            entry->next_reclaim = NULL;
-            entry->prev_reclaim = NULL;
-            entry->load         = RT_AIO_INVALID_HANDLE;
-
-            batch.loads[batch.num_loads].file      = data->pkg_file;
-            batch.loads[batch.num_loads].num_bytes = data->size;
-            batch.loads[batch.num_loads].offset    = data->offset;
-            batch.loads[batch.num_loads].dest      = compressed_data;
-            ++batch.num_loads;
-        }
-    }
-    rtUnlockWrite(&_lock);
-
-    /* Dispatch the load */
-    rt_aio_handle handles[RT_LOAD_BATCH_MAX_SIZE];
-    if ((res = rtSubmitLoadBatch(&batch, handles)) != RT_SUCCESS) {
-        rtLog("ASSET_CACHE", "Failed to submit %u asset loads.", batch.num_loads);
-        return res;
-    }
-
-    /* Set the aio handles of the inserted entries */
-    rtLockWrite(&_lock);
-    for (unsigned int i = 0; i < batch.num_loads; ++i) {
-        load_entries[batch.num_loads]->load = handles[i];
-    }
-    rtUnlockWrite(&_lock);
-
-    return res;
-}
-
-static bool DecompressEntry(rt_uid uid, rt_asset_cache_entry *entry) {
-    rtReleaseAIO(entry->load);
-    entry->load = RT_AIO_INVALID_HANDLE;
-
-    void *decompressed_buffer;
-    size_t decompressed_size;
-    rt_result dec_res =
-        DecompressAsset(entry->buffer, entry->size, &decompressed_buffer, &decompressed_size);
-    if (dec_res == RT_SUCCESS) {
-        rtReleaseBuffer(entry->buffer, entry->size);
-        entry->buffer = decompressed_buffer;
-        entry->size   = decompressed_size;
-        entry->state  = CACHE_ENTRY_STATE_LOADED;
-        return true;
-    } else if (dec_res == RT_BUFFER_ALLOC_FAILED) {
-        GarbageCollect();
-        /* Try again */
-        if (DecompressAsset(entry->buffer, entry->size, &decompressed_buffer, &decompressed_size) ==
-            RT_SUCCESS) {
-            rtReleaseBuffer(entry->buffer, entry->size);
-            entry->buffer = decompressed_buffer;
-            entry->size   = decompressed_size;
-            entry->state  = CACHE_ENTRY_STATE_LOADED;
-            return true;
-        }
-        /* Don't do anything yet. We might be able to to do this later, once some
-         * buffers become free. */
-        rtLog("ASSET_CACHE", "Failed to decompress asset %u", uid);
-        return false;
-    } else {
-        rtLog("ASSET_CACHE", "Failed to decompress asset %u", uid);
-        ReleaseEntry(entry);
-
-        ptrdiff_t idx = entry - _entries;
-        _uids[idx]    = RT_INVALID_UID;
-        return false;
-    }
-}
-
-static void CheckCompletedLoads(const rt_uid *uids, size_t count) {
-    for (size_t i = 0; i < count; ++i) {
-        rtLockRead(&_lock);
-        volatile rt_asset_cache_entry *entry = (volatile rt_asset_cache_entry *)GetEntry(uids[i]);
-        if (!entry) {
-            rtUnlockRead(&_lock);
-            rtLog("ASSET_CACHE", "Passed unknown uid %u to CheckCompletedLoads()", uids[i]);
-            continue;
-        }
-
-        if (entry->state != CACHE_ENTRY_STATE_LOADING) {
-            rtUnlockRead(&_lock);
-            continue;
-        }
-        bool load_finished = rtGetAIOState(entry->load) == RT_AIO_STATE_FINISHED;
-        rtUnlockRead(&_lock);
-       
-        if (load_finished) {
-            rtLockWrite(&_lock);
-            /* Ensure that no-one else handled this */
-            if (entry->state == CACHE_ENTRY_STATE_LOADING) {
-                DecompressEntry(uids[i], (rt_asset_cache_entry *)entry);
-            }
-            rtUnlockWrite(&_lock);
-        }
-    }
-}
-
-RT_DLLEXPORT rt_get_asset_result rtGetAsset(rt_uid uid) {
-    rt_get_asset_result result = {
-        .result = RT_SUCCESS,
-    };
-
-    rtLockRead(&_lock);
-    bool needs_load = !IsAssetLoaded(uid);
-    rtUnlockRead(&_lock);
-
-    if (needs_load) {
-        rt_uid load_uids[RT_LOAD_BATCH_MAX_SIZE];
-        size_t load_count = 1;
-        load_uids[0]      = uid;
-
-        rt_asset_dependency_list deps = rtGetAssetDependencies(uid);
-        for (size_t i = 0; i < deps.count && i < RT_LOAD_BATCH_MAX_SIZE - 1; ++i) {
-            load_uids[i + 1] = deps.dependencies[i];
-            ++load_count;
-        }
-
-        result.result = InsertAndLoadAssets(load_uids, load_count);
-        if (result.result == RT_SUCCESS) {
-            CheckCompletedLoads(load_uids, load_count);
-        }
-    }
-
-    rtLockWrite(&_lock);
-    rt_asset_cache_entry *entry = GetEntry(uid);
-    if (entry) {
-        if (entry->state == CACHE_ENTRY_STATE_LOADED) {
-            ++entry->refcount;
-            result.data = entry->buffer;
-            result.size = entry->size;
-        } else if (entry->state == CACHE_ENTRY_STATE_LOADING) {
-            if (entry->state == CACHE_ENTRY_STATE_LOADING) {
-                assert(entry->load != RT_AIO_INVALID_HANDLE);
-                ++entry->refcount;
-                if (rtWaitForAIOCompletion(entry->load) == RT_AIO_STATE_FINISHED) {
-                    if (DecompressEntry(uid, entry)) {
-                        result.data = entry->buffer;
-                        result.size = entry->size;
-                    } else {
-                        result.result = RT_LOAD_FAILED;
-                    }
-                } else {
-                    ReleaseEntry(entry);
-                    rtLog("ASSET_CACHE", "Failed to load asset %u", uid);
-                    result.result = RT_LOAD_FAILED;
-                }
-            }
-        }
-
-        /* Remove from the reclaim list */
-        if (_first_reclaim == entry)
-            _first_reclaim = entry->next_reclaim;
-        if (_last_reclaim == entry)
-            _last_reclaim = entry->prev_reclaim;
-        if (entry->next_reclaim)
-            entry->next_reclaim->prev_reclaim = entry->prev_reclaim;
-        if (entry->prev_reclaim)
-            entry->prev_reclaim->next_reclaim = entry->next_reclaim;
-    }
-    rtUnlockWrite(&_lock);
-
-    return result;
-}
-
-RT_DLLEXPORT void rtReleaseAsset(rt_uid uid) {
-    rtLockWrite(&_lock);
-    rt_asset_cache_entry *entry = GetEntry(uid);
-    if (entry && entry->refcount > 0) {
-        --entry->refcount;
-        if (entry->refcount == 0) {
-            /* add to the reclaim list */
-            if (_last_reclaim)
-                _last_reclaim->next_reclaim = entry;
-            if (!_first_reclaim)
-                _first_reclaim = entry;
-            entry->prev_reclaim = _last_reclaim;
-            entry->next_reclaim = NULL;
-            _last_reclaim       = entry;
-        }
-    }
-    rtUnlockWrite(&_lock);
-}

src/runtime/asset_compiler.c

@@ -0,0 +1,342 @@
+#include "runtime.h"
+#include "threading.h"
+#include "config.h"
+#include "fsutils.h"
+#include "file_tab.h"
+#include "mem_arena.h"
+#include "buffer_manager.h"
+
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#ifndef RT_BUILD_ASSET_COMPILER
+#error This should only be built when RT_BUILD_ASSET_COMPILER is defined.
+#endif
+
+typedef struct {
+    uint64_t last_processed;
+} rt_asset_data;
+
+typedef struct {
+    rt_file_id *files;
+    rt_asset_data *data;
+} rt_asset_db;
+
+typedef rt_result rt_asset_processor_fn(rt_file_id file, rt_arena *arena);
+
+typedef struct {
+    const char *file_ext;
+    rt_asset_processor_fn *proc;
+} rt_asset_processor;
+
+typedef struct {
+    unsigned int processor_index;
+    unsigned int db_index;
+    rt_file_id fid;
+} rt_processing_queue_entry;
+
+typedef struct {
+    rt_processing_queue_entry entries[1024];
+    unsigned int head;
+    unsigned int tail;
+    rt_condition_var *lock;
+} rt_processing_queue;
+
+RT_CVAR_S(rt_AssetDirectory, "Name of the asset directory. Default: assets", "assets");
+RT_CVAR_I(rt_AssetDBSize, "Size of the asset database. Default: 1024", 1024);
+RT_CVAR_I(rt_AssetProcessingThreads, "Number of asset processing threads. Default: 4", 4);
+RT_CVAR_I(rt_AssetProcessorArenaSize,
+          "Size of the per-thread asset processor arena. Default: 128 MiB",
+          (int)RT_MB(128));
+
+#define MAX_PROCESSING_THREADS 8
+static rt_thread *_compiler_thread;
+static rt_thread *_processing_threads[MAX_PROCESSING_THREADS];
+static bool _keep_running = true;
+
+static rt_asset_db _asset_db;
+
+static rt_processing_queue _processing_queue;
+
+extern rt_result PipelineProcessor(rt_file_id file, rt_arena *arena);
+
+static rt_asset_processor _processors[] = {
+    {.file_ext = ".pipeline", .proc = PipelineProcessor}
+};
+
+static void ProcessorThreadEntry(void *);
+static void CompilerThreadEntry(void *);
+
+rt_result InitAssetCompiler(void) {
+
+    unsigned int db_size = (unsigned int)rt_AssetDBSize.i;
+    void *mem            = malloc((sizeof(rt_file_id) + sizeof(rt_asset_data)) * db_size);
+    if (!mem)
+        return RT_OUT_OF_MEMORY;
+    _asset_db.files = mem;
+    _asset_db.data  = (rt_asset_data *)(_asset_db.files + db_size);
+    memset(mem, 0, (sizeof(rt_file_id) + sizeof(rt_asset_data)) * db_size);
+
+    _processing_queue.lock = rtCreateConditionVar();
+    if (!_processing_queue.lock) {
+        free(mem);
+        return RT_UNKNOWN_ERROR;
+    }
+
+    _keep_running    = true;
+    _compiler_thread = rtSpawnThread(CompilerThreadEntry, NULL, "AssetCompilerThread");
+    if (!_compiler_thread)
+        return RT_UNKNOWN_ERROR;
+
+    if (rt_AssetProcessingThreads.i > MAX_PROCESSING_THREADS)
+        rt_AssetProcessingThreads.i = MAX_PROCESSING_THREADS;
+    for (int i = 0; i < rt_AssetProcessingThreads.i; ++i) {
+        char name[64];
+        snprintf(name, 64, "AssetProcessorThread %d", i);
+        _processing_threads[i] = rtSpawnThread(ProcessorThreadEntry, NULL, name);
+        if (!_processing_threads[i]) {
+            /* Wake the processing threads */
+            rtLockConditionVar(_processing_queue.lock);
+            rtUnlockConditionVar(_processing_queue.lock, true);
+            _keep_running = false;
+            for (int j = 0; j < i; ++j)
+                rtJoinThread(_processing_threads[j]);
+            rtJoinThread(_compiler_thread);
+            free(mem);
+            rtDestroyConditionVar(_processing_queue.lock);
+            return RT_UNKNOWN_ERROR;
+        }
+    }
+
+    return RT_SUCCESS;
+}
+
+void ShutdownAssetCompiler(void) {
+    _keep_running = false;
+    rtJoinThread(_compiler_thread);
+    for (int i = 0; i < rt_AssetProcessingThreads.i; ++i)
+        rtJoinThread(_processing_threads[i]);
+    free(_asset_db.files);
+    rtDestroyConditionVar(_processing_queue.lock);
+}
+
+static int DiscoverAssets(void) {
+    /* Recursive descend into the asset directory */
+#define MAX_DISCOVERY_DEPTH 64
+#define MAX_FILENAME_LEN 260
+    static char directory_stack[MAX_DISCOVERY_DEPTH][MAX_FILENAME_LEN];
+    static unsigned int path_lens[MAX_DISCOVERY_DEPTH];
+    unsigned int top = 0;
+
+    memcpy(directory_stack[0],
+           rt_AssetDirectory.s,
+           (strlen(rt_AssetDirectory.s) <= (MAX_FILENAME_LEN - 1)) ? strlen(rt_AssetDirectory.s)
+                                                                   : MAX_FILENAME_LEN);
+    directory_stack[0][MAX_FILENAME_LEN - 1] = '\0';
+    ++top;
+    
+    int discovery_count = 0;
+
+    while (top > 0) {
+        /* Process the directory */
+        char dir[MAX_FILENAME_LEN];
+        memcpy(dir, directory_stack[top - 1], MAX_FILENAME_LEN);
+        --top;
+
+        /* Append to the path */
+        size_t dir_len = strlen(dir);
+
+        rt_scandir_handle *scan = rtScanDirectory(dir);
+        if (!scan)
+            continue;
+        rt_dirent entry;
+        do {
+            entry = rtNextDirectoryEntry(scan);
+            
+            if (entry.name[0] == '.')
+                continue;
+
+            size_t entry_name_len = strlen(entry.name);
+            if (dir_len + entry_name_len + 1 >= MAX_FILENAME_LEN) {
+                rtLog("AC",
+                      "Could not process %s\\%s because the name exceeds the maximum "
+                      "allowed path length %d.",
+                      dir,
+                      entry.name,
+                      MAX_FILENAME_LEN);
+                continue;
+            }
+
+            if (entry.type == RT_DIRENT_TYPE_DIRECTORY) {
+                /* Push on stack */
+                assert(MAX_FILENAME_LEN == RT_ARRAY_COUNT(entry.name));
+                memcpy(directory_stack[top], dir, MAX_FILENAME_LEN);
+                directory_stack[top][dir_len] = '/';
+                memcpy(&directory_stack[top][dir_len + 1], entry.name, entry_name_len);
+                directory_stack[top][dir_len + entry_name_len + 1] = '\0';
+                ++top;
+            } else {
+                /* Check if the asset is new */
+                char file[MAX_FILENAME_LEN];
+                memcpy(file, dir, MAX_FILENAME_LEN);
+                file[dir_len] = '/';
+                memcpy(&file[dir_len + 1], entry.name, entry_name_len);
+                file[dir_len + entry_name_len + 1] = '\0';
+
+                rt_file_id fid = rtAddFile(file);
+                unsigned int i = 0;
+                while (i < (unsigned int)rt_AssetDBSize.i) {
+                    unsigned int slot = (fid + i) % (unsigned int)rt_AssetDBSize.i;
+                    if (_asset_db.files[slot] == fid) {
+                        break;
+                    } else if (_asset_db.files[slot] == 0) {
+                        _asset_db.files[slot] = fid;
+                        _asset_db.data[slot].last_processed = 0;
+                        ++discovery_count;
+                        break;
+                    }
+                    ++i;
+                }
+                if (i == (unsigned int)rt_AssetDBSize.i) {
+                    rtLog("AC", "Failed to add %s to AssetDB, because no free slots are left.", file);
+                }
+            }
+        } while (!entry.is_last);
+        rtCloseDirectory(scan);
+    }
+    return discovery_count;
+#undef MAX_DISCOVERY_DEPTH
+#undef MAX_FILENAME_LEN
+}
+
+static int CheckUpdatedAssets(void) {
+    int updated_count = 0;
+    for (int i = 0; i < rt_AssetDBSize.i; ++i) {
+        if (_asset_db.files[i] == 0)
+            continue;
+        const char *path = rtGetFilePath(_asset_db.files[i]);
+        uint64_t last_changed = rtGetFileModificationTimestamp(path);
+        if (_asset_db.data[i].last_processed < last_changed) {
+            const char *ext = path + strlen(path);
+            while (*ext != '.' && ext != path)
+                --ext;
+            if (*ext != '.')
+                break;
+
+            for (unsigned int j = 0; j < RT_ARRAY_COUNT(_processors); ++j) {
+                if (strcmp(ext, _processors[j].file_ext) == 0) {
+                    rt_processing_queue_entry entry;
+                    entry.fid             = _asset_db.files[i];
+                    entry.processor_index = j;
+                    entry.db_index        = i;
+                    
+                    while (true) {
+                        bool inserted = false;
+                        rtLockConditionVar(_processing_queue.lock);
+                        unsigned int next_tail = (_processing_queue.tail + 1) %
+                                                 RT_ARRAY_COUNT(_processing_queue.entries);
+                        if (next_tail != _processing_queue.head) {
+                            _processing_queue.entries[_processing_queue.tail] = entry;
+                            _processing_queue.tail                            = next_tail;
+                            inserted = true;
+                        }
+                        rtUnlockConditionVar(_processing_queue.lock, inserted);
+                        if (inserted)
+                            break;
+                    }
+                }
+            }
+            ++updated_count;
+        }
+    }
+    return updated_count;
+}
+
+static void CompilerThreadEntry(void *param) {
+    RT_UNUSED(param);
+
+    /* Mainloop. Watch for changed or new assets, compile them
+     * and notify the appropriate system */
+    while (_keep_running) {
+        int d = DiscoverAssets();
+        int u = CheckUpdatedAssets();
+        if (d == 0 && u == 0)
+            rtSleep(100);
+    }
+}
+
+static void ProcessorThreadEntry(void *param) {
+    RT_UNUSED(param);
+
+    rt_create_arena_result arena_res = rtCreateArena(NULL, (size_t)rt_AssetProcessorArenaSize.i);
+    if (!arena_res.ok) {
+        rtLog("AC",
+              "Failed to allocate %d bytes for the processing arena.",
+              rt_AssetProcessorArenaSize.i);
+        return;
+    }
+    rt_arena arena = arena_res.arena;
+
+    while (_keep_running) {
+        rtLockConditionVar(_processing_queue.lock);
+
+        while (_keep_running && (_processing_queue.tail == _processing_queue.head))
+            rtWaitOnConditionVar(_processing_queue.lock);
+
+        bool got_entry = false;
+        rt_processing_queue_entry entry = {0};
+        if (_processing_queue.tail != _processing_queue.head) {
+            entry = _processing_queue.entries[_processing_queue.head];
+            _processing_queue.head =
+                (_processing_queue.head + 1) % RT_ARRAY_COUNT(_processing_queue.entries);
+            got_entry = true;
+        }
+
+        rtUnlockConditionVar(_processing_queue.lock, false);
+
+        if (!got_entry)
+            continue;
+
+        const char *path = rtGetFilePath(entry.fid);
+        rtLog("AC", "Processing %s", path);
+        rtArenaClear(&arena);
+        rt_result res = _processors[entry.processor_index].proc(entry.fid, &arena);
+        if (res != RT_SUCCESS) {
+            rtLog("AC", "Failed to process %s: %u", path, res);
+        }
+        _asset_db.data[entry.db_index].last_processed = rtGetCurrentTimestamp();
+    }
+}
+
+/* Utilities for asset processors*/
+
+#include "aio.h"
+#include "asset_compiler.h"
+
+rt_loaded_asset LoadAsset(rt_file_id file)
+{
+    const char *path = rtGetFilePath(file);
+    size_t file_size = rtGetFileSize(path);
+
+    void *buffer = rtAllocBuffer(file_size);
+    rtLog("AC", "Buffer ptr %llx", (uintptr_t)buffer);
+    if (!buffer) {
+        rtLog("AC", "Failed to allocate buffer for loading %s.", path);
+        return (rt_loaded_asset){.buffer = NULL, .size = 0};
+    }
+
+    rt_aio_state load_state = rtSubmitSingleLoadSync((rt_file_load){
+        .file      = file,
+        .offset    = 0,
+        .num_bytes = file_size,
+        .dest      = buffer,
+    });
+    if (load_state != RT_AIO_STATE_FINISHED) {
+        rtReleaseBuffer(buffer, file_size);
+        rtLog("AC", "Failed to load %s.", path);
+        return (rt_loaded_asset){.buffer = NULL, .size = 0};
+    }
+    return (rt_loaded_asset){.buffer = buffer, .size = file_size};
+}

src/runtime/asset_compiler.h

@@ -0,0 +1,22 @@
+#ifndef RT_ASSET_COMPILER_H
+#define RT_ASSET_COMPILER_H
+
+#ifndef RT_BUILD_ASSET_COMPILER
+#error This file should only be included if RT_BUILD_ASSET_COMPILER is defined. 
+#endif
+
+#include "file_tab.h"
+
+enum {
+    RT_ASSET_PROCESSING_FAILED = RT_CUSTOM_ERROR_START,
+};
+
+/* Allocated from the buffer manager */
+typedef struct {
+    void *buffer;
+    size_t size;
+} rt_loaded_asset;
+
+rt_loaded_asset LoadAsset(rt_file_id file);
+
+#endif

src/runtime/asset_dependencies.c

@@ -1,137 +0,0 @@
-#define RT_DEFINE_DEPENDENCY_FILE_STRUCTURES
-#include "asset_dependencies.h"
-
-#include "aio.h"
-#include "buffer_manager.h"
-
-#include <assert.h>
-#include <stdbool.h>
-#include <stdlib.h>
-#include <string.h>
-
-typedef struct {
-    uint32_t begin;
-    uint32_t count;
-} rt_dep_list;
-
-typedef struct {
-    rt_uid *uids;
-    rt_dep_list *lists;
-    uint32_t capacity;
-} rt_dep_map;
-
-static rt_dep_map _map;
-static rt_uid *_list_mem;
-
-rt_result LoadAssetDependencies(void) {
-    rt_dependency_file_header header;
-    rt_file_id fid = rtAddFile("data/deps.bin");
-
-    if (rtSubmitSingleLoadSync((rt_file_load){.dest      = &header,
-                                              .num_bytes = sizeof(header),
-                                              .offset    = 0,
-                                              .file      = fid}) != RT_AIO_STATE_FINISHED) {
-        rtReportError("core", "Failed to load deps.bin");
-        return RT_UNKNOWN_ERROR;
-    }
-    void *buffer = rtAllocBuffer(header.data_size);
-    if (rtSubmitSingleLoadSync((rt_file_load){.dest      = buffer,
-                                              .num_bytes = header.data_size,
-                                              .offset    = sizeof(header),
-                                              .file      = fid}) != RT_AIO_STATE_FINISHED) {
-        rtReportError("core", "Failed to load deps.bin");
-        return RT_UNKNOWN_ERROR;
-    }
-
-    /* We know the exact number of list entries */
-    uint64_t total_list_entries =
-        (header.data_size - header.num_lists * sizeof(rt_dependency_file_list_header)) /
-        sizeof(rt_uid);
-    _list_mem = malloc(total_list_entries * sizeof(rt_uid));
-    if (!_list_mem) {
-        rtReleaseBuffer(buffer, header.data_size);
-        rtReportError("core", "Failed to allocate dependency list storage.");
-        return RT_UNKNOWN_ERROR;
-    }
-
-    _map.capacity = rtNextPowerOfTwo32(header.num_lists);
-    _map.uids     = calloc(_map.capacity, sizeof(rt_uid));
-    if (!_map.uids) {
-        free(_list_mem);
-        rtReleaseBuffer(buffer, header.data_size);
-        rtReportError("core", "Failed to allocate dependency list storage.");
-        return RT_UNKNOWN_ERROR;
-    }
-
-    _map.lists = calloc(_map.capacity, sizeof(rt_dep_list));
-    if (!_map.uids) {
-        free(_list_mem);
-        free(_map.uids);
-        rtReleaseBuffer(buffer, header.data_size);
-        rtReportError("core", "Failed to allocate dependency list storage.");
-        return RT_UNKNOWN_ERROR;
-    }
-
-    uint32_t storage_at = 0;
-
-    rt_dependency_file_list_header *list = buffer;
-    for (uint32_t i = 0; i < header.num_lists; ++i) {
-        const rt_uid *entries = (rt_uid *)(list + 1);
-
-        /* Validate the checksum */
-        XXH64_hash_t file_hash = XXH64_hashFromCanonical(&list->checksum);
-        XXH64_hash_t calc_hash = XXH3_64bits(entries, sizeof(rt_uid) * list->num_entries);
-        if (file_hash != calc_hash) {
-            rtReportError("core", "Checksum mismatch in list %u", i);
-            rtReleaseBuffer(buffer, header.data_size);
-            return RT_UNKNOWN_ERROR;
-        }
-
-        /* Store the list */
-        memcpy(_list_mem + storage_at, entries, sizeof(rt_uid) * list->num_entries);
-        bool inserted = false;
-        for (uint32_t j = 0; j < _map.capacity; ++j) {
-            uint32_t slot = (list->uid + j) % _map.capacity;
-            if (_map.uids[slot] == RT_INVALID_UID) {
-                _map.uids[slot] = list->uid;
-                _map.lists[slot].begin = storage_at;
-                _map.lists[slot].count = list->num_entries;
-                inserted = true;
-                break;
-            }
-        }
-        storage_at += list->num_entries;
-        assert(inserted);
-        assert(storage_at <= total_list_entries);
-
-        list = (rt_dependency_file_list_header *)(entries + list->num_entries);
-    }
-
-    rtReleaseBuffer(buffer, header.data_size);
-
-    return RT_SUCCESS;
-}
-
-void ReleaseAssetDependencies(void) {
-    free(_list_mem);
-    free(_map.uids);
-    free(_map.lists);
-}
-
-RT_DLLEXPORT rt_asset_dependency_list rtGetAssetDependencies(rt_uid asset) {
-    rt_asset_dependency_list result = {
-        .dependencies = NULL,
-        .count        = 0,
-    };
-    for (uint32_t i = 0; i < _map.capacity; ++i) {
-        uint32_t slot = (asset + i) % _map.capacity;
-        if (_map.uids[slot] == asset) {
-            result.dependencies = &_list_mem[_map.lists[slot].begin];
-            result.count        = _map.lists[slot].count;
-            break;
-        } else if (_map.uids[slot] == RT_INVALID_UID) {
-            break;
-        }
-    }
-    return result;
-}

src/runtime/asset_dependencies.h

@@ -1,33 +0,0 @@
-#ifndef RT_ASSET_DEPENDENCIES_H
-#define RT_ASSET_DEPENDENCIES_H
-
-#include "assets.h"
-
-#ifdef RT_DEFINE_DEPENDENCY_FILE_STRUCTURES
-
-#include "xxhash/xxhash.h"
-
-#pragma pack(push, 1)
-typedef struct {
-    uint64_t data_size;
-    uint32_t num_lists;
-} rt_dependency_file_header;
-
-typedef struct {
-    rt_uid uid;
-    uint32_t num_entries;
-    XXH64_canonical_t checksum;
-} rt_dependency_file_list_header;
-
-#pragma pack(pop)
-
-#endif
-
-typedef struct {
-    const rt_uid *dependencies;
-    uint32_t count;
-} rt_asset_dependency_list;
-
-RT_DLLEXPORT rt_asset_dependency_list rtGetAssetDependencies(rt_uid asset);
-
-#endif

src/runtime/asset_loading.c

@@ -1,74 +0,0 @@
-#include "assets.h"
-#include "uidtab.h"
-#include "aio.h"
-#include "buffer_manager.h"
-
-#define RT_DEFINE_PACKAGE_FILE_STRUCTURES
-#include "packages.h"
-
-#include "lz4/lz4.h"
-
-rt_result DecompressAsset(void *compressed_buffer,
-                          size_t compressed_buffer_size,
-                          void **p_decompressed,
-                          size_t *p_decompressed_size) {
-
-    const rt_package_asset_header *header = compressed_buffer;
-
-    size_t compressed_size       = (compressed_buffer_size) - sizeof(*header);
-    XXH64_hash_t calculated_hash = XXH3_64bits((header + 1), compressed_size);
-    XXH64_hash_t file_hash       = XXH64_hashFromCanonical(&header->checksum);
-    if (calculated_hash != file_hash) {
-        rtLog("core", "Checksum mismatch for asset");
-        return RT_LOAD_FAILED;
-    }
-
-    size_t size               = (size_t)header->decompressed_size;
-    void *decompressed_buffer = rtAllocBuffer(size);
-    if (!decompressed_buffer) {
-        return RT_BUFFER_ALLOC_FAILED;
-    }
-
-    if (LZ4_decompress_safe((const char *)(header + 1),
-                            (char *)decompressed_buffer,
-                            (int)compressed_size,
-                            (int)size) < 0) {
-        return RT_UNKNOWN_ERROR;
-    }
-
-    *p_decompressed      = decompressed_buffer;
-    *p_decompressed_size = size;
-    return RT_SUCCESS;
-}
-
-
-RT_DLLEXPORT rt_result rtLoadAssetDirect(rt_uid uid, void **p_buffer, size_t *p_size) {
-    const rt_uid_data *data = rtGetUIDData(uid);
-    if (!data)
-        return RT_UNKNOWN_ASSET;
-
-    void *compressed_buffer = rtAllocBuffer(data->size);
-    if (!compressed_buffer) {
-        return RT_BUFFER_ALLOC_FAILED;
-    }
-
-    if (rtSubmitSingleLoadSync((rt_file_load) {
-            .file      = data->pkg_file,
-            .offset    = data->offset,
-            .num_bytes = data->size,
-            .dest      = compressed_buffer,
-        }) != RT_AIO_STATE_FINISHED) {
-        rtReleaseBuffer(compressed_buffer, data->size);
-        return RT_LOAD_FAILED;
-    }
-    
-    void *decompressed_buffer;
-    size_t decompressed_size;
-    rt_result res = DecompressAsset(compressed_buffer, data->size, &decompressed_buffer, &decompressed_size);
-
-    rtReleaseBuffer(compressed_buffer, data->size);
-    *p_buffer = decompressed_buffer;
-    *p_size   = decompressed_size;
-
-    return res;
-}

src/runtime/assets.h

@@ -1,40 +1,6 @@
 #ifndef RT_ASSETS_H
 #define RT_ASSETS_H
 
-#include <stdint.h>
+/* Asset system interface */
-
-#include "runtime.h"
-
-/* Unique identifier for an asset. */
-typedef uint32_t rt_uid;
-
-#define RT_INVALID_UID 0
-
-/* Used to identify renderer backend dependent assets. */
-enum {
-    RT_INVALID_RENDERER_BACKEND_CODE = 0,
-    RT_RENDERER_BACKEND_CODE_VK      = 1,
-
-    RT_RENDERER_BACKEND_CODE_ONE_PAST_LAST,
-};
-typedef uint8_t rt_renderer_backend_code;
-
-enum {
-    RT_UNKNOWN_ASSET = RT_CUSTOM_ERROR_START,
-    RT_BUFFER_ALLOC_FAILED,
-    RT_LOAD_FAILED,
-    RT_ASSET_CACHE_FULL,
-};
-
-/* Load an asset without using the cache */
-RT_DLLEXPORT rt_result rtLoadAssetDirect(rt_uid uid, void **buffer, size_t *size);
-
-typedef struct {
-    void *data;
-    size_t size;
-    rt_result result;
-} rt_get_asset_result;
-
-RT_DLLEXPORT rt_get_asset_result rtGetAsset(rt_uid uid);
 
 #endif

src/runtime/buffer_manager.c

@@ -7,7 +7,9 @@
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdlib.h>
+#include <string.h>
 
+#if 0
 typedef struct rt_buffer_region_s {
     void *memory;
     int16_t *refcounts; // One per block
@@ -15,6 +17,7 @@ typedef struct rt_buffer_region_s {
     size_t block_count;
     rt_mutex *guard;
 } rt_buffer_region;
+#endif
 
 /* Count leading zeroes.
  * Note that the return value of __builtin_clz(0) is undefined. */
@@ -31,7 +34,7 @@ static __forceinline uint32_t tzcnt32(uint32_t x) {
     return (uint32_t)i;
 }
 
-static inline bool IsLZCNTSupported(void) {
+static __forceinline bool IsLZCNTSupported(void) {
 #define Type 0x80000001
     int info[4];
     __cpuid(info, Type);
@@ -48,6 +51,7 @@ static inline bool IsLZCNTSupported(void) {
 
 #endif
 
+#if 0
 /* NOTE(Kevin): Keep these sorted! */
 static size_t _block_sizes[] = {RT_KB(512), RT_MB(1), RT_MB(4), RT_MB(8)};
 #define NUM_BLOCK_SIZES (sizeof(_block_sizes) / sizeof(_block_sizes[0]))
@@ -258,3 +262,155 @@ RT_DLLEXPORT void rtIncreaseBufferRefCount(const void *begin, size_t size) {
     }
     rtLog("BUFFERMGR", "Tried to increase the refcount of an invalid buffer");
 }
+#endif
+
+#define BLOCK_SIZE 4096u
+
+static uint32_t *_refcounts;
+static uint32_t *_bitmap;
+static char *_memory;
+static rt_mutex *_guard;
+
+static size_t _block_count;
+
+RT_CVAR_I(rt_BufferMemoryBudget,
+          "The amount of memory to allocate for the buffer manager. Default: 512MB",
+          RT_MB(512));
+
+extern rt_result InitBufferManager(void) {
+    _guard = rtCreateMutex();
+    if (!_guard) {
+        rtReportError("BUFFERMGR", "Failed to create the buffer manager mutex.");
+        return RT_UNKNOWN_ERROR;
+    }
+    if (!IsLZCNTSupported()) {
+        rtReportError("BUFFERMGR", "The required lzcnt intrinisc is not supported.");
+        return RT_UNKNOWN_ERROR;
+    }
+
+    size_t budget      = (size_t)rt_BufferMemoryBudget.i;
+    size_t block_count = budget / BLOCK_SIZE;
+    if ((budget % block_count) != 0) {
+        rtLog("BUFFERMGR",
+              "The configured buffer memory budget %zu is not dividable by the block size (4KB).",
+              budget);
+    }
+    size_t dword_count = (block_count + 31) / 32;
+    _block_count       = block_count;
+
+    _memory = malloc(budget + dword_count * sizeof(uint32_t) + block_count * sizeof(uint32_t));
+    if (!_memory) {
+        return RT_OUT_OF_MEMORY;
+    }
+    _bitmap = (uint32_t*)(_memory + budget);
+    memset(_bitmap, 0, sizeof(uint32_t) * dword_count);
+    _refcounts = _bitmap + dword_count;
+    memset(_refcounts, 0, sizeof(uint32_t) * block_count);
+
+    return RT_SUCCESS;
+}
+
+extern void ShutdownBufferManager(void) {
+    rtDestroyMutex(_guard);
+}
+
+/* Public API */
+
+RT_DLLEXPORT void *rtAllocBuffer(size_t size) {
+    size_t alloc_blocks = (size + BLOCK_SIZE - 1) / BLOCK_SIZE;
+    size_t dword_count  = (_block_count + 31) / 32;
+    void *result        = NULL;
+    size_t first_block  = 0;
+    rtLockMutex(_guard);
+    for (size_t i = 0; i < _block_count; ++i) {
+        size_t dword = i / 32;
+        if (_bitmap[dword] == 0 || tzcnt32(_bitmap[dword]) >= alloc_blocks) {
+            size_t mask = (1ull << alloc_blocks) - 1;
+            _bitmap[dword] |= mask;
+            result      = _memory + i * BLOCK_SIZE;
+            first_block = i;
+        }
+        else if (lzcnt32(_bitmap[dword]) >= alloc_blocks) {
+            size_t first = (_bitmap[dword] != 0) ? 32 - lzcnt32(_bitmap[dword]) : 0;
+            size_t mask  = ((1ull << alloc_blocks) - 1) << first;
+            _bitmap[dword] |= mask;
+            result      = _memory + (i + first) * BLOCK_SIZE;
+            first_block = i + first;
+            break;
+        } else if (_bitmap[dword] != UINT32_MAX) {
+            size_t first    = 32 - lzcnt32(_bitmap[dword]);
+            size_t leftover = alloc_blocks - lzcnt32(_bitmap[dword]);
+            if (dword == dword_count - 1) {
+                break; // Reached the end
+            }
+            if (leftover < 32) {
+                size_t next_dword_free = _bitmap[dword + 1] != 0 ? tzcnt32(_bitmap[dword + 1]) : 32;
+                if (next_dword_free < leftover)
+                    continue;
+                _bitmap[dword] = UINT32_MAX;
+                size_t mask    = (1ull << leftover) - 1;
+                _bitmap[dword + 1] |= mask;
+                result      = _memory + (i + first) * BLOCK_SIZE;
+                first_block = i + first;
+                break;
+            } else {
+                // Check each bit separately
+                bool free = true;
+                for (size_t j = i + first; j < i + first + alloc_blocks; ++j) {
+                    size_t dwordj = j / 32;
+                    size_t bitj   = j % 32;
+                    if ((_bitmap[dwordj] & (1u << bitj)) != 0) {
+                        free = false;
+                        break;
+                    }
+                }
+                if (free) {
+                    for (size_t j = i + first; j < i + first + alloc_blocks; ++j) {
+                        size_t dwordj = j / 32;
+                        size_t bitj   = j % 32;
+                        _bitmap[dwordj] |= (1u << bitj);
+                    }
+                    result      = _memory + (i + first) * BLOCK_SIZE;
+                    first_block = i + first;
+                }
+            }
+        } else {
+            /* These 32 blocks are all allocated. Go to the next dword */
+            assert((i % 32) == 0);
+            i += 31;
+        }
+    }
+
+    for (size_t i = first_block; i < first_block + alloc_blocks; ++i)
+        _refcounts[i] = 1;
+
+    rtUnlockMutex(_guard);
+    rtLog("BUFFERMGR", "Result ptr %llx", (uintptr_t)result);
+    return result;
+}
+
+RT_DLLEXPORT void rtReleaseBuffer(const void *begin, size_t size) {
+    size_t alloc_blocks = (size + BLOCK_SIZE - 1) / BLOCK_SIZE;
+    uintptr_t off       = (uintptr_t)begin - (uintptr_t)_memory;
+    uintptr_t first_block = off / BLOCK_SIZE;
+    rtLockMutex(_guard);
+    for (size_t i = first_block; i < first_block + alloc_blocks; ++i) {
+        if (--_refcounts[i] == 0) {
+            size_t dword = i / 32;
+            size_t bit   = i % 32;
+            _bitmap[dword] &= ~(1u << bit);
+        }
+    }
+    rtUnlockMutex(_guard);
+}
+
+RT_DLLEXPORT void rtIncreaseBufferRefCount(const void *begin, size_t size) {
+    size_t alloc_blocks   = (size + BLOCK_SIZE - 1) / BLOCK_SIZE;
+    uintptr_t off         = (uintptr_t)begin - (uintptr_t)_memory;
+    uintptr_t first_block = off / BLOCK_SIZE;
+    rtLockMutex(_guard);
+    for (size_t i = first_block; i < first_block + alloc_blocks; ++i) {
+        ++_refcounts[i];
+    }
+    rtUnlockMutex(_guard);
+}

src/runtime/description_parser.c

@@ -0,0 +1,252 @@
+#include "description_parser.h"
+#include "runtime.h"
+#include "mem_arena.h"
+
+#include <stdbool.h>
+#include <limits.h>
+
+extern int memcmp(const void *s1, const void *s2, size_t n);
+extern void *memcpy(void *dest, const void *src, size_t n);
+
+static bool IsAllowedChar(char c) {
+    return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+           (c == '.') || (c == '_') || (c == '/');
+}
+
+static bool IsWhitespace(char c) {
+    return c == ' ' || c == '\t' || c == '\n' || c == '\r';
+}
+
+static void SkipWhitespace(rt_parse_state *state) {
+    while (state->at < state->length && IsWhitespace(state->text[state->at])) {
+        if (state->text[state->at] == '\n')
+            ++state->line;
+        ++state->at;
+    }
+}
+
+static bool ParseAttribute(rt_parse_state *state, rt_text_span *_name) {
+    rt_text_span name;
+    name.start  = &state->text[state->at];
+    name.length = 0;
+    while (state->at < state->length && !IsWhitespace(state->text[state->at])) {
+        if (IsAllowedChar(state->text[state->at])) {
+            ++state->at;
+            ++name.length;
+        } else {
+            rtReportError("AC",
+                          "%s:%d Unexpected character %c",
+                          state->file,
+                          state->line,
+                          state->text[state->at]);
+            return false;
+        }
+    }
+    *_name = name;
+    return true;
+}
+
+static bool ParseValue(rt_parse_state *state, rt_text_span *_value) {
+    rt_text_span value;
+    value.start  = &state->text[state->at];
+    value.length = 0;
+    while (state->at < state->length && !IsWhitespace(state->text[state->at]) &&
+           state->text[state->at] != ';') {
+        if (IsAllowedChar(state->text[state->at])) {
+            ++state->at;
+            ++value.length;
+        } else {
+            rtReportError("AC",
+                          "%s:%d Unexpected character %c",
+                          state->file,
+                          state->line,
+                          state->text[state->at]);
+            return false;
+        }
+    }
+    *_value = value;
+    return true;
+}
+
+#define BLOCK_BEGIN        "BEGIN"
+#define BLOCK_BEGIN_LENGTH 5
+#define BLOCK_END        "END"
+#define BLOCK_END_LENGTH 3
+
+RT_INLINE static bool IsBlockBegin(rt_parse_state *state) {
+    return (state->length - state->at >= BLOCK_BEGIN_LENGTH) &&
+           (memcmp(&state->text[state->at], BLOCK_BEGIN, BLOCK_BEGIN_LENGTH) == 0);
+}
+
+RT_INLINE static bool IsBlockEnd(rt_parse_state *state) {
+    return (state->length - state->at >= BLOCK_END_LENGTH) &&
+           (memcmp(&state->text[state->at], BLOCK_END, BLOCK_END_LENGTH) == 0);
+}
+
+static bool ParseBlock(rt_parse_state *state, rt_text_span *p_value) {
+    rt_text_span value;
+    value.start = &state->text[state->at];
+    value.length = 0;
+    while (state->at < state->length) {
+        if (state->text[state->at] == BLOCK_END[0] && IsBlockEnd(state)) {
+            *p_value = value;
+            return true;
+        }
+        ++value.length;
+        ++state->at;
+    }
+    /* did not find END */
+    return false;
+}
+
+static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index, rt_arena *arena);
+
+static bool ParseStmt(rt_parse_state *state, unsigned int *stmt_index, rt_arena *arena) {
+    rt_parsed_stmt stmt;
+    stmt.next = UINT_MAX; /* end of list */
+
+    SkipWhitespace(state);
+    if (!ParseAttribute(state, &stmt.attribute))
+        return false;
+    SkipWhitespace(state);
+
+    if (state->at == state->length) {
+        rtReportError("AC", "%s:%d Expected either a value or '{'", state->file, state->line);
+        return false;
+    }
+
+    if (state->text[state->at] == '{') {
+        /* Consume '{' */
+        ++state->at;
+
+        stmt.form = RT_STMT_FORM_LIST;
+        if (!ParseStmtList(state, &stmt.list_index, arena))
+            return false;
+
+        /* Consume '}' */
+        if (state->at < state->length && state->text[state->at] == '}')
+            ++state->at;
+    } else if (IsBlockBegin(state)) {
+        /* Consume BEGIN */
+        state->at += BLOCK_BEGIN_LENGTH;
+
+        stmt.form = RT_STMT_FORM_BLOCK;
+        if (!ParseBlock(state, &stmt.block))
+            return false;
+
+        /* Consume END */
+        state->at += BLOCK_END_LENGTH;
+    } else {
+        stmt.form = RT_STMT_FORM_VALUE;
+        if (!ParseValue(state, &stmt.value))
+            return false;
+
+        /* Consume ';' */
+        if (state->at < state->length && state->text[state->at] == ';')
+            ++state->at;
+    }
+
+    SkipWhitespace(state);
+
+    /* Add statement to array */
+    if (state->statement_count == state->statement_capacity) {
+        unsigned int cap     = (state->statement_capacity > 0) ? state->statement_capacity * 2 : 64;
+        rt_parsed_stmt *temp = rtArenaPush(arena, sizeof(rt_parsed_stmt) * cap);
+        if (!temp) {
+            rtReportError("AC", "While parsing %s: Out of memory\n", state->file);
+            return false;
+        }
+        if (state->statement_capacity > 0)
+            memcpy(temp, state->statements, sizeof(rt_parsed_stmt) * state->statement_capacity);
+        state->statements         = temp;
+        state->statement_capacity = cap;
+    }
+    state->statements[state->statement_count] = stmt;
+    *stmt_index                               = state->statement_count++;
+
+    return true;
+}
+
+static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index, rt_arena *arena) {
+    rt_parsed_stmt_list list;
+    list.first = UINT_MAX;
+    list.count = 0;
+
+    unsigned int last = UINT_MAX;
+
+    while (state->at < state->length && state->text[state->at] != '}') {
+        unsigned int stmt;
+        if (!ParseStmt(state, &stmt, arena))
+            return false;
+
+        if (last != UINT_MAX)
+            state->statements[last].next = stmt;
+        else
+            list.first = stmt;
+        last = stmt;
+
+        ++list.count;
+    }
+
+    /* Add list to array */
+    if (state->statement_list_count == state->statement_list_capacity) {
+        unsigned int cap =
+            (state->statement_list_capacity > 0) ? state->statement_list_capacity * 2 : 64;
+        rt_parsed_stmt_list *temp = rtArenaPush(arena, sizeof(rt_parsed_stmt_list) * cap);
+        if (!temp) {
+            rtReportError("AC", "While parsing %s: Out of memory\n", state->file);
+            return false;
+        }
+        if (state->statement_list_capacity > 0)
+            memcpy(temp,
+                   state->statement_lists,
+                   sizeof(rt_parsed_stmt_list) * state->statement_list_capacity);
+        state->statement_lists         = temp;
+        state->statement_list_capacity = cap;
+    }
+    state->statement_lists[state->statement_list_count] = list;
+    *list_index                                         = state->statement_list_count++;
+    return true;
+}
+
+const rt_parsed_stmt * rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute) {
+    if (list_index >= state->statement_list_count)
+        return NULL;
+    const rt_parsed_stmt_list *list = &state->statement_lists[list_index];
+    unsigned int stmt_index         = list->first;
+    for (unsigned int i = 0; i < list->count; ++i) {
+        const rt_parsed_stmt *stmt = &state->statements[stmt_index];
+        if (rtCompareSpanToString(stmt->attribute, attribute) == 0)
+            return stmt;
+        stmt_index = stmt->next;
+    }
+    return NULL;
+}
+
+
+rt_result rtParseDescription(const char *text,
+                             size_t length,
+                             const char *file_path,
+                             unsigned int *_root_list,
+                             rt_parse_state *_state,
+                             struct rt_arena_s *arena) {
+
+    rt_parse_state state = {.text                    = text,
+                            .at                      = 0,
+                            .length                  = length,
+                            .line                    = 1,
+                            .file                    = file_path,
+                            .statements              = NULL,
+                            .statement_lists         = NULL,
+                            .statement_capacity      = 0,
+                            .statement_list_capacity = 0};
+
+    unsigned int root_list = 0;
+    if (!ParseStmtList(&state, &root_list, arena)) {
+        return 1;
+    }
+    *_root_list = root_list;
+    *_state     = state;
+
+    return RT_SUCCESS;
+}

src/runtime/description_parser.h

@@ -0,0 +1,56 @@
+#ifndef RT_ASSETC_DESCRIPTION_PARSER_H
+#define RT_ASSETC_DESCRIPTION_PARSER_H
+
+#include "runtime/runtime.h"
+
+struct rt_arena_s;
+
+typedef enum {
+    RT_STMT_FORM_VALUE,
+    RT_STMT_FORM_LIST,
+    RT_STMT_FORM_BLOCK,
+} rt_stmt_form;
+
+typedef struct {
+    unsigned int first;
+    unsigned int count;
+} rt_parsed_stmt_list;
+
+typedef struct {
+    rt_stmt_form form;
+    rt_text_span attribute;
+    union {
+        rt_text_span value;
+        rt_text_span block;
+        unsigned int list_index;
+    };
+    /* For lists */
+    unsigned int next;
+} rt_parsed_stmt;
+
+typedef struct {
+    const char *file;
+    const char *text;
+    size_t at;
+    size_t length;
+    int line;
+
+    rt_parsed_stmt *statements;
+    unsigned int statement_count;
+    unsigned int statement_capacity;
+
+    rt_parsed_stmt_list *statement_lists;
+    unsigned int statement_list_count;
+    unsigned int statement_list_capacity;
+} rt_parse_state;
+
+rt_result rtParseDescription(const char *text,
+                             size_t length,
+                             const char *file_path,
+                             unsigned int *root_list,
+                             rt_parse_state *state,
+                             struct rt_arena_s *arena);
+
+const rt_parsed_stmt *rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute);
+
+#endif

src/runtime/error_report.c

@@ -1,5 +1,4 @@
 #include <stdarg.h>
-#include <stdio.h>
 #include <stdbool.h>
 #include <stdio.h>
 
@@ -19,6 +18,17 @@
 
 /* TODO(Kevin): Log to file, show error message box, ... */
 
+void Win32ErrorToString(DWORD last_error, char *out, int bufsize) {
+    FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS |
+                       FORMAT_MESSAGE_ARGUMENT_ARRAY | FORMAT_MESSAGE_ALLOCATE_BUFFER,
+                   NULL,
+                   last_error,
+                   0,
+                   out,
+                   bufsize,
+                   NULL);
+}
+
 static bool DisplayErrorBox(const char *text) {
 #ifdef _WIN32
     WCHAR msg[256];

src/runtime/file_tab.c

@@ -1,8 +1,7 @@
 #include "file_tab.h"
 #include "threading.h"
 #include "config.h"
-
+#include "hashing.h"
-#include <xxhash/xxhash.h>
 
 #define NAME_CAP(cap) ((cap)*128)
 typedef struct {
@@ -53,8 +52,7 @@ RT_DLLEXPORT rt_file_id rtGetFileId(const char *path) {
 
 RT_DLLEXPORT rt_file_id rtGetFileIdFromSpan(rt_text_span path) {
     /* Randomly choosen, aka finger smash keyboard */
-    XXH64_hash_t seed = 15340978;
+    rt_file_id fid = (rt_file_id)rtHashBytes(path.start, path.length);
-    rt_file_id fid    = (rt_file_id)XXH3_64bits_withSeed(path.start, path.length, seed);
     if (fid == 0)
         fid = ~fid;
     return fid;

src/runtime/fsutils.c

@@ -0,0 +1,115 @@
+#include "fsutils.h"
+#include "runtime.h"
+#include "threading.h"
+
+#ifdef _WIN32
+
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+
+struct rt_scandir_handle_s {
+    HANDLE handle;
+    rt_dirent next;
+};
+
+static rt_scandir_handle _dirs[256];
+static unsigned int _next = 0;
+static rt_mutex *_guard = NULL;
+
+RT_DLLEXPORT rt_scandir_handle *rtScanDirectory(const char *path) {
+    char wildcard_path[MAX_PATH];
+    strncpy(wildcard_path, path, MAX_PATH);
+    strncat(wildcard_path, "\\*", MAX_PATH - strlen(path));
+    WCHAR wpath[MAX_PATH];
+    if (rtUTF8ToWStr(wildcard_path, wpath, MAX_PACKAGE_NAME) != RT_SUCCESS)
+        return NULL;
+    WIN32_FIND_DATAW data;
+    HANDLE h = FindFirstFileW(wpath, &data);
+    if (h == INVALID_HANDLE_VALUE)
+        return NULL;
+
+    if (!_guard) {
+        _guard = rtCreateMutex();
+        if (!_guard) {
+            FindClose(h);
+            return NULL;
+        }
+    }
+
+    rtLockMutex(_guard);
+    rt_scandir_handle *dir = &_dirs[_next];
+    _next                   = (_next + 1) % RT_ARRAY_COUNT(_dirs);
+    rtUnlockMutex(_guard);
+
+    if (dir->handle != NULL) {
+        rtLog("core", "Failed to acquire a scandir handle.");
+        FindClose(h);
+        return NULL;
+    }
+
+    dir->handle = h;
+    dir->next.is_last = false;
+    rtWStrToUTF8(data.cFileName, dir->next.name, 260);
+    if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+        dir->next.type = RT_DIRENT_TYPE_DIRECTORY;
+    else
+        dir->next.type = RT_DIRENT_TYPE_FILE;
+
+    return dir;
+}
+
+RT_DLLEXPORT rt_dirent rtNextDirectoryEntry(rt_scandir_handle *dir) {
+    rt_dirent current;
+    memcpy(&current, &dir->next, sizeof(rt_dirent));
+    
+    WIN32_FIND_DATAW data;
+    if (!FindNextFileW(dir->handle, &data)) {
+        current.is_last = true;
+        return current;
+    }
+    rtWStrToUTF8(data.cFileName, dir->next.name, 260);
+    if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+        dir->next.type = RT_DIRENT_TYPE_DIRECTORY;
+    else
+        dir->next.type = RT_DIRENT_TYPE_FILE;
+
+
+    return current;
+}
+
+RT_DLLEXPORT void rtCloseDirectory(rt_scandir_handle *dir) {
+    if (!dir)
+        return;
+    FindClose(dir->handle);
+    dir->handle = NULL;
+}
+
+RT_DLLEXPORT size_t rtGetFileSize(const char *path) {
+    WCHAR wpath[MAX_PATH];
+    MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH);
+    WIN32_FILE_ATTRIBUTE_DATA attribs;
+    if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs))
+        return 0;
+    return (size_t)attribs.nFileSizeHigh << 32 | (size_t)attribs.nFileSizeLow;
+}
+
+uint64_t rtGetCurrentTimestamp(void) {
+    FILETIME ft;
+    GetSystemTimeAsFileTime(&ft);
+    uint64_t ts = ft.dwLowDateTime;
+    ts |= (uint64_t)ft.dwHighDateTime << 32;
+    return ts;
+}
+
+uint64_t rtGetFileModificationTimestamp(const char *path) {
+    WCHAR wpath[MAX_PATH];
+    MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH);
+    WIN32_FILE_ATTRIBUTE_DATA attribs;
+    if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs))
+        return 0;
+    uint64_t ts = attribs.ftLastWriteTime.dwLowDateTime;
+    ts |= (uint64_t)attribs.ftLastWriteTime.dwHighDateTime << 32;
+    return ts;
+}
+
+#endif

src/runtime/fsutils.h

@@ -0,0 +1,35 @@
+#ifndef RT_FSUITLS_H
+#define RT_FSUTILS_H
+
+/* File system utilities */
+#include "runtime.h"
+#include <stdbool.h>
+
+typedef enum {
+    RT_DIRENT_TYPE_FILE,
+    RT_DIRENT_TYPE_DIRECTORY,
+    RT_DIRENT_TYPE_SYMLINK,
+} rt_dirent_type;
+
+typedef struct {
+    char name[260];
+    rt_dirent_type type;
+    bool is_last;
+} rt_dirent;
+
+typedef struct rt_scandir_handle_s rt_scandir_handle;
+
+RT_DLLEXPORT rt_scandir_handle *rtScanDirectory(const char *path);
+
+RT_DLLEXPORT rt_dirent rtNextDirectoryEntry(rt_scandir_handle *dir);
+
+RT_DLLEXPORT void rtCloseDirectory(rt_scandir_handle *dir);
+
+RT_DLLEXPORT size_t rtGetFileSize(const char *path);
+
+RT_DLLEXPORT uint64_t rtGetFileModificationTimestamp(const char *path);
+
+/* Does not really fit here, but it is mostly useful for comparison with file timestamps. */
+RT_DLLEXPORT uint64_t rtGetCurrentTimestamp(void);
+
+#endif

src/runtime/hashing.c

@@ -0,0 +1,10 @@
+#include "hashing.h"
+
+#include <xxhash/xxhash.h>
+#include <assert.h>
+
+static_assert(sizeof(rt_hash64) == sizeof(XXH64_hash_t), "Size mismatch between rt_hash64 and XXH64_hash_t!");
+
+RT_DLLEXPORT rt_hash64 rtHashBytes(const void *begin, size_t count) {
+    return XXH3_64bits(begin, count);
+}

src/runtime/hashing.h

@@ -0,0 +1,12 @@
+#ifndef RT_HASHING_H
+#define RT_HASHING_H
+
+/* Hash functions */
+
+#include "runtime.h"
+
+typedef uint64_t rt_hash64;
+
+RT_DLLEXPORT rt_hash64 rtHashBytes(const void *begin, size_t count);
+
+#endif

src/runtime/init.c

@@ -3,26 +3,30 @@
 #include "aio.h"
 #include "file_tab.h"
 #include "buffer_manager.h"
-#include "uidtab.h"
 
 extern rt_cvar rt_Renderer;
 extern rt_cvar rt_Fullscreen;
 extern rt_cvar rt_WindowWidth;
 extern rt_cvar rt_WindowHeight;
-extern rt_cvar rt_BufferManagerMemory;
+extern rt_cvar rt_BufferMemoryBudget;
 extern rt_cvar rt_FileTabCapacity;
 extern rt_cvar rt_MaxConcurrentAsyncIO;
-extern rt_cvar rt_AssetCacheSize;
+
+#ifdef RT_BUILD_ASSET_COMPILER
+extern rt_cvar rt_AssetDirectory;
+#endif
 
 void RegisterRuntimeCVars(void) {
     rtRegisterCVAR(&rt_Renderer);
     rtRegisterCVAR(&rt_Fullscreen);
     rtRegisterCVAR(&rt_WindowWidth);
     rtRegisterCVAR(&rt_WindowHeight);
-    rtRegisterCVAR(&rt_BufferManagerMemory);
+    rtRegisterCVAR(&rt_BufferMemoryBudget);
     rtRegisterCVAR(&rt_FileTabCapacity);
     rtRegisterCVAR(&rt_MaxConcurrentAsyncIO);
-    rtRegisterCVAR(&rt_AssetCacheSize);
+#ifdef RT_BUILD_ASSET_COMPILER
+    rtRegisterCVAR(&rt_AssetDirectory);
+#endif
 }
 
 extern void SetMainThreadId(void);
@@ -33,14 +37,11 @@ extern rt_result InitFileTab(void);
 extern void ShutdownFileTab(void);
 extern rt_result InitAIO(void);
 extern void ShutdownAIO(void);
-extern rt_result InitAssetCache(void);
-extern void ShutdownAssetCache(void);
 
-extern rt_result LoadUIDTable(void);
+#ifdef RT_BUILD_ASSET_COMPILER
-extern void ReleaseUIDTable(void);
+extern rt_result InitAssetCompiler(void);
-extern rt_result LoadAssetDependencies(void); 
+extern void ShutdownAssetCompiler(void);
-extern void ReleaseAssetDependencies(void);
+#endif
-extern rt_result LoadPackageNames(void);
 
 RT_DLLEXPORT rt_result rtInitRuntime(void) {
     SetMainThreadId();
@@ -62,31 +63,20 @@ RT_DLLEXPORT rt_result rtInitRuntime(void) {
         return res;
     }
 
-    if ((res = InitAssetCache()) != RT_SUCCESS) {
+#ifdef RT_BUILD_ASSET_COMPILER
-        rtReportError("ASSETCACHE", "Init failed.");
+    if ((res = InitAssetCompiler()) != RT_SUCCESS) {
+        rtReportError("AC", "Init failed.");
         return res;
     }
-
+#endif
-    if ((res = LoadUIDTable()) != RT_SUCCESS) {
-        rtLog("CORE", "LoadUIDTable returned result: %u", res);
-    }
-
-    if ((res = LoadAssetDependencies()) != RT_SUCCESS) {
-        rtReportError("ASSETDEP", "Init failed.");
-        return res;
-    }
-
-    if ((res = LoadPackageNames()) != RT_SUCCESS) {
-        rtLog("CORE", "LoadPackageNames returned result: %u", res);
-    }
 
     return RT_SUCCESS;
 }
 
 RT_DLLEXPORT void rtShutdownRuntime(void) {
-    ReleaseAssetDependencies();
+#ifdef RT_BUILD_ASSET_COMPILER
-    ReleaseUIDTable();
+    ShutdownAssetCompiler();
-    ShutdownAssetCache();
+#endif
     ShutdownAIO();
     ShutdownFileTab();
     ShutdownBufferManager();

src/runtime/mem_arena.c

@@ -0,0 +1,61 @@
+#include "mem_arena.h"
+
+#include <stdlib.h>
+
+#define ALIGNMENT 0xf
+#define ALIGN(n)  (((n) + ALIGNMENT) & ~ALIGNMENT)
+
+extern void *memset(void *, int, size_t);
+
+RT_DLLEXPORT rt_create_arena_result rtCreateArena(void *memory, size_t size) {
+    rt_arena arena = {.needs_free = 0};
+    if (!memory) {
+        size   = ALIGN(size);
+        memory = malloc(size);
+        if (!memory) {
+            return (rt_create_arena_result){
+                .ok = 0,
+            };
+        }
+        arena.needs_free = 1;
+    }
+    arena.base = memory;
+    arena.size = size;
+    arena.at   = 0;
+    return (rt_create_arena_result){
+        .arena = arena,
+        .ok    = 1,
+    };
+}
+
+RT_DLLEXPORT void *rtArenaPush(rt_arena *arena, size_t n) {
+    n = ALIGN(n);
+    if (arena->at + n > arena->size)
+        return NULL;
+    void *p = (char *)arena->base + arena->at;
+    arena->at += n;
+    return p;
+}
+
+RT_DLLEXPORT void *rtArenaPushZero(rt_arena *arena, size_t n) {
+    void *p = rtArenaPush(arena, n);
+    if (p)
+        memset(p, 0, ALIGN(n));
+    return p;
+}
+
+RT_DLLEXPORT void rtArenaPop(rt_arena *arena, size_t n) {
+    n = ALIGN(n);
+    if (arena->at < n)
+        return;
+    arena->at -= n;
+}
+
+RT_DLLEXPORT void rtReleaseArena(rt_arena *arena) {
+    if (arena->needs_free)
+        free(arena->base);
+    arena->base       = NULL;
+    arena->at         = 0;
+    arena->size       = 0;
+    arena->needs_free = 0;
+}

src/runtime/mem_arena.h

@@ -0,0 +1,57 @@
+#ifndef RT_MEM_ARENA_H
+#define RT_MEM_ARENA_H
+
+#include "runtime.h"
+
+typedef struct rt_arena_s {
+    void *base;
+    size_t at;
+    size_t size;
+    int needs_free;
+} rt_arena;
+
+typedef struct {
+    int ok;
+    rt_arena arena;
+} rt_create_arena_result;
+
+typedef struct {
+    size_t at;
+} rt_arena_rewindpoint;
+
+RT_DLLEXPORT rt_create_arena_result rtCreateArena(void *memory, size_t size);
+
+RT_DLLEXPORT void rtReleaseArena(rt_arena *arena);
+
+RT_DLLEXPORT void *rtArenaPush(rt_arena *arena, size_t n);
+
+RT_DLLEXPORT void *rtArenaPushZero(rt_arena *arena, size_t n);
+
+RT_DLLEXPORT void rtArenaPop(rt_arena *arena, size_t n);
+
+RT_INLINE void rtArenaClear(rt_arena *arena) {
+    arena->at = 0;
+}
+
+RT_INLINE rt_arena_rewindpoint rtGetArenaRewindPoint(rt_arena *arena) {
+    return (rt_arena_rewindpoint){.at = arena->at};
+}
+
+RT_INLINE void rtArenaRewind(rt_arena *arena, rt_arena_rewindpoint rewind) {
+#ifndef NDEBUG
+    if (rewind.at > arena->at)
+        rtReportError("CORE",
+                      "Tried to rewind an arena to a point that lies beyond the last allocation.");
+#endif
+    arena->at = rewind.at;
+}
+
+/* Helper macros */
+#define RT_ARENA_PUSH_STRUCT(_Arena, _Type)         rtArenaPush((_Arena), sizeof(_Type))
+#define RT_ARENA_PUSH_STRUCT_ZERO(_Arena, _Type)    rtArenaPushZero((_Arena), sizeof(_Type))
+#define RT_ARENA_POP_STRUCT(_Arena, _Type)          rtArenaPop((_Arena), sizeof(_Type))
+#define RT_ARENA_PUSH_ARRAY(_Arena, _Type, _N)      rtArenaPush((_Arena), sizeof(_Type) * (_N))
+#define RT_ARENA_PUSH_ARRAY_ZERO(_Arena, _Type, _N) rtArenaPushZero((_Arena), sizeof(_Type) * (_N))
+#define RT_ARENA_POP_ARRAY(_Arena, _Type, _N)       rtArenaPop((_Arena), sizeof(_Type) * (_N)
+
+#endif

src/runtime/pipeline_processor.c

@@ -0,0 +1,370 @@
+#include "runtime.h"
+#include "mem_arena.h"
+#include "description_parser.h"
+#include "buffer_manager.h"
+#include "asset_compiler.h"
+#include "shader_compiler.h"
+#include "gfx.h"
+#include "config.h"
+
+#include <limits.h>
+#include <assert.h>
+#include <string.h>
+
+typedef struct {
+    rt_attribute_binding *uniform_bindings;
+    rt_attribute_binding *storage_bindings;
+    rt_attribute_binding *texture_bindings;
+
+    rt_shader_bytecode vertex_shader;
+    rt_shader_bytecode fragment_shader;
+    rt_shader_bytecode compute_shader;
+
+    /* TODO Fixed function settings */
+
+    /* Sampler settings */
+
+    uint16_t uniform_binding_count;
+    uint16_t storage_binding_count;
+    uint16_t texture_binding_count;
+} rt_parsed_pipeline_data;
+
+enum {
+    RT_SHADER_NOT_PRESENT = RT_ASSET_PROCESSING_FAILED + 1
+};
+
+extern rt_cvar rt_Renderer;
+
+#if 0
+static void
+DbgPrintShaderFile(const rt_parse_state *state, unsigned int list_index, unsigned int indent) {
+    assert(list_index < state->statement_list_count);
+    const rt_parsed_stmt_list *list = &state->statement_lists[list_index];
+
+    unsigned int stmt_index = list->first;
+    for (unsigned int i = 0; i < list->count; ++i) {
+        const rt_parsed_stmt *stmt = &state->statements[stmt_index];
+        for (unsigned int j = 0; j < indent; ++j)
+            printf(" ");
+        printf("%.*s: ", stmt->attribute.length, stmt->attribute.start);
+        if (stmt->form == RT_STMT_FORM_VALUE) {
+            printf("%.*s\n", stmt->value.length, stmt->value.start);
+        } else {
+            printf("{\n");
+            DbgPrintShaderFile(state, stmt->list_index, indent + 2);
+            printf("}\n");
+        }
+        stmt_index = stmt->next;
+    }
+    assert(stmt_index == UINT_MAX || stmt_index == 0);
+}
+#endif
+
+static bool ParseBindingIndex(rt_text_span span, unsigned int *index) {
+    if (span.length == 0)
+        return false;
+    int at           = (int)span.length - 1;
+    unsigned int exp = 1;
+    unsigned int n   = 0;
+    while (at >= 0) {
+        if (span.start[at] >= '0' && span.start[at] <= '9') {
+            unsigned int digit = (unsigned int)(span.start[at] - '0');
+            n += digit * exp;
+        } else {
+            rtReportError("GFX", "Unexpected non-digit character in binding index");
+            return false;
+        }
+        --at;
+        exp *= 10;
+    }
+    *index = n;
+    return true;
+}
+
+static rt_attribute_value ParseBindingValue(rt_text_span span) {
+    if (rtCompareSpanToString(span, "MATERIAL_ALBEDO") == 0) {
+        return RT_ATTRIBUTE_VALUE_MATERIAL_ALBEDO;
+    } else if (rtCompareSpanToString(span, "MATERIAL_NORMAL") == 0) {
+        return RT_ATTRIBUTE_VALUE_MATERIAL_NORMAL;
+    }
+    rtReportError("GFX", "Unsupported binding value %*.s", span.length, span.start);
+    return RT_ATTRIBUTE_VALUE_UNDEFINED;
+}
+
+static bool ParseBindings(rt_parse_state *state,
+                          unsigned int root_list,
+                          const char *name,
+                          const char *file_path,
+                          rt_attribute_binding **p_bindings,
+                          uint16_t *p_binding_count) {
+    const rt_parsed_stmt *bindings = rtFindStatement(state, root_list, name);
+    if (bindings) {
+        if (bindings->form != RT_STMT_FORM_LIST) {
+            rtReportError("GFX",
+                          "Expected list of bindings as the value of "
+                          "\"%s\" in %s",
+                          name,
+                          file_path);
+            return false;
+        }
+        const rt_parsed_stmt_list *binding_list = &state->statement_lists[bindings->list_index];
+        rt_attribute_binding *shader_bindings =
+            rtAllocBuffer(sizeof(rt_attribute_binding) * binding_list->count);
+        if (!bindings) {
+            rtReportError("GFX", "Out of memory");
+            return false;
+        }
+        unsigned int binding_count = binding_list->count;
+
+        unsigned int stmt_index = binding_list->first;
+        for (unsigned int i = 0; i < binding_list->count; ++i) {
+            const rt_parsed_stmt *stmt = &state->statements[stmt_index];
+            if (!ParseBindingIndex(stmt->attribute, &shader_bindings[i].index)) {
+                rtReleaseBuffer(shader_bindings,
+                                sizeof(rt_attribute_binding) * binding_list->count);
+                return false;
+            }
+
+            shader_bindings[i].value = ParseBindingValue(stmt->value);
+            if (shader_bindings[i].value == RT_ATTRIBUTE_VALUE_UNDEFINED) {
+                rtReleaseBuffer(shader_bindings,
+                                sizeof(rt_attribute_binding) * binding_list->count);
+                return false;
+            }
+            stmt_index = stmt->next;
+        }
+
+        *p_bindings      = shader_bindings;
+        *p_binding_count = (uint16_t)binding_count;
+        return true;
+    } else {
+        *p_bindings      = NULL;
+        *p_binding_count = 0;
+        return true;
+    }
+}
+
+static rt_result ParseShader(rt_parse_state *state,
+                             unsigned int root_list,
+                             const char *name,
+                             const char *file_path,
+                             rt_shader_type type,
+                             rt_shader_stage stage,
+                             rt_shader_optimization_level optimization,
+                             rt_shader_bytecode *p_shader_bytecode,
+                             rt_arena *arena) {
+    const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, name);
+    if (stmt) {
+        if (stmt->form != RT_STMT_FORM_LIST) {
+            rtReportError("GFX",
+                          "Expected a list as the value of "
+                          "\"%s\" in %s",
+                          name,
+                          file_path);
+            return RT_ASSET_PROCESSING_FAILED;
+        }
+        const rt_parsed_stmt_list *shader_list = &state->statement_lists[stmt->list_index];
+
+        unsigned int stmt_index = shader_list->first;
+        for (unsigned int i = 0; i < shader_list->count; ++i) {
+            const rt_parsed_stmt *shader = &state->statements[stmt_index];
+            if (shader->form != RT_STMT_FORM_VALUE && shader->form != RT_STMT_FORM_BLOCK) {
+                rtReportError("GFX",
+                              "Expected a simple statement or block for "
+                              "\"%s.%*s\" in %s",
+                              name,
+                              (int)shader->attribute.length,
+                              shader->attribute.start,
+                              file_path);
+                return RT_ASSET_PROCESSING_FAILED;
+            }
+            rt_shader_type in_file_type = RT_SHADER_TYPE_INVALID;
+            if (rtCompareSpanToString(shader->attribute, "vk") == 0) {
+                in_file_type = RT_SHADER_TYPE_VULKAN;
+            } else {
+                rtReportError("GFX",
+                              "Invalid renderer backend"
+                              "\"%*s\" in %s of file %s",
+                              (int)shader->attribute.length,
+                              shader->attribute.start,
+                              name,
+                              file_path);
+                return RT_ASSET_PROCESSING_FAILED;
+            }
+
+            if (in_file_type == type) {
+                if (shader->form == RT_STMT_FORM_BLOCK) {
+                    /* Inline code */
+                    *p_shader_bytecode =
+                        CompileShader(type, stage, optimization, shader->block, file_path, arena);
+                    if (!p_shader_bytecode->bytes)
+                        return RT_ASSET_PROCESSING_FAILED;
+                    break;
+                } else if (shader->form != RT_STMT_FORM_VALUE) {
+                    /* A filename */
+                    rtLog("AC", "Shader source files not implemented yet!");
+                    return RT_ASSET_PROCESSING_FAILED;
+                }
+            }
+
+            stmt_index = shader->next;
+        }
+        return RT_SUCCESS;
+    }
+    return RT_SHADER_NOT_PRESENT;
+}
+
+static rt_shader_optimization_level
+ParseOptimizationLevel(rt_parse_state *state, unsigned int root_list, const char *file_path) {
+    const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, "optimization");
+    rt_shader_optimization_level optimization_level = RT_SHADER_OPTIMIZATION_NONE;
+    if (stmt) {
+        if (stmt->form != RT_STMT_FORM_VALUE) {
+            rtReportError("GFX",
+                          "Expected a simple statement for"
+                          "\"optimization\" in %s",
+                          file_path);
+            return RT_SHADER_OPTIMIZATION_NONE;
+        }
+
+        if (rtCompareSpanToString(stmt->value, "speed") == 0) {
+            optimization_level = RT_SHADER_OPTIMIZATION_SPEED;
+        } else if (rtCompareSpanToString(stmt->value, "size") == 0) {
+            optimization_level = RT_SHADER_OPTIMIZATION_SIZE;
+        } else if (rtCompareSpanToString(stmt->value, "none") == 0) {
+            optimization_level = RT_SHADER_OPTIMIZATION_NONE;
+        } else {
+            rtReportError("GFX",
+                          "Expected one of 'speed', 'size' and 'none' for \"optimization\" in %s",
+                          file_path);
+        }
+    }
+
+    return optimization_level;
+}
+
+static rt_result ParsePipelineFile(rt_file_id fid,
+                                   const char *text,
+                                   size_t length,
+                                   rt_parsed_pipeline_data *pipeline,
+                                   rt_arena *arena) {
+    /* This is the grammar for pipeline files:
+     * <stmt-list> ::= <stmt>*
+     * <stmt> ::= <attribute> ( ( <value> ';' ) | ( '{' <stmt-list> '}' ) )
+     * <attribute> ::= [:alnum:]*
+     * <value>:: = [:alnum:]* */
+    const char *file_path = rtGetFilePath(fid);
+    rt_parse_state state;
+    unsigned int root_list;
+    rt_result result = rtParseDescription(text, length, file_path, &root_list, &state, arena);
+    if (result != RT_SUCCESS) {
+        goto out;
+    }
+
+    /* We allow the pipeline file to overwrite the optimization level */
+    rt_shader_optimization_level optimization = ParseOptimizationLevel(&state, root_list, file_path);
+    
+    rt_shader_type type = RT_SHADER_TYPE_INVALID;
+    if (strcmp(rt_Renderer.s, "vk") == 0)
+        type = RT_SHADER_TYPE_VULKAN;
+
+    if (type == RT_SHADER_TYPE_INVALID) {
+        result = RT_ASSET_PROCESSING_FAILED;
+        rtLog("AC", "rt_Renderer (%s) could not be translated to a shader type.", rt_Renderer.s);
+        goto out;
+    }
+
+    /* Process shader stages */
+    if (ParseShader(&state,
+                    root_list,
+                    "vertex",
+                    file_path,
+                    type,
+                    RT_SHADER_STAGE_VERTEX,
+                    optimization,
+                    &pipeline->vertex_shader,
+                    arena) == RT_ASSET_PROCESSING_FAILED) {
+        result = RT_ASSET_PROCESSING_FAILED;
+        goto out;
+    }
+    if (ParseShader(&state,
+                    root_list,
+                    "fragment",
+                    file_path,
+                    type,
+                    RT_SHADER_STAGE_FRAGMENT,
+                    optimization,
+                    &pipeline->fragment_shader,
+                    arena) == RT_ASSET_PROCESSING_FAILED) {
+        result = RT_ASSET_PROCESSING_FAILED;
+        goto out;
+    }
+    if (ParseShader(&state,
+                    root_list,
+                    "compute",
+                    file_path,
+                    type,
+                    RT_SHADER_STAGE_COMPUTE,
+                    optimization,
+                    &pipeline->compute_shader,
+                    arena) == RT_ASSET_PROCESSING_FAILED) {
+        result = RT_ASSET_PROCESSING_FAILED;
+        goto out;
+    }
+
+    /* Process bindings */
+    pipeline->texture_bindings      = NULL;
+    pipeline->texture_binding_count = 0;
+    pipeline->uniform_bindings      = NULL;
+    pipeline->uniform_binding_count = 0;
+    pipeline->storage_bindings      = NULL;
+    pipeline->storage_binding_count = 0;
+
+    if (!ParseBindings(&state,
+                       root_list,
+                       "texture_bindings",
+                       file_path,
+                       &pipeline->texture_bindings,
+                       &pipeline->texture_binding_count)) {
+        result = RT_ASSET_PROCESSING_FAILED;
+        goto out;
+    }
+
+    if (!ParseBindings(&state,
+                       root_list,
+                       "uniform_bindings",
+                       file_path,
+                       &pipeline->uniform_bindings,
+                       &pipeline->uniform_binding_count)) {
+        result = RT_ASSET_PROCESSING_FAILED;
+        goto out;
+    }
+
+    if (!ParseBindings(&state,
+                       root_list,
+                       "storage_bindings",
+                       file_path,
+                       &pipeline->storage_bindings,
+                       &pipeline->storage_binding_count)) {
+        result = RT_ASSET_PROCESSING_FAILED;
+        goto out;
+    }
+
+out:
+    return result;
+}
+
+rt_result PipelineProcessor(rt_file_id file, rt_arena *arena) {
+    rt_loaded_asset asset = LoadAsset(file);
+    if (!asset.buffer)
+        return RT_UNKNOWN_ERROR;
+
+    rt_parsed_pipeline_data pipeline;
+    rt_result result = ParsePipelineFile(file, asset.buffer, asset.size, &pipeline, arena);
+    if (result != RT_SUCCESS)
+        goto out;
+
+out:
+    rtReleaseBuffer(asset.buffer, asset.size);
+    return result;
+}

src/runtime/renderer_api.h

@@ -27,9 +27,10 @@ struct rt_renderer_init_info_s {
 };
 
 typedef struct {
-    rt_uid vertex_shader;
+    /* rt_uid vertex_shader;
     rt_uid fragment_shader;
     rt_uid compute_shader;
+    */
 
     rt_relptr texture_bindings;
     rt_relptr uniform_bindings;

src/runtime/runtime.h

@@ -109,7 +109,6 @@ static RT_INLINE uint32_t rtNextPowerOfTwo32(uint32_t v) {
     return v;
 }
 
-
 /* Runtime init. Initializes basic systems.
  * You need to call this, even if you build a CLI only app. */
 RT_DLLEXPORT rt_result rtInitRuntime(void);

src/runtime/shader_compiler.c

@@ -0,0 +1,52 @@
+#include "shader_compiler.h"
+
+#ifdef RT_BUILD_VULKAN_SHADER_COMPILER
+extern rt_shader_bytecode CompileVulkanShader(rt_shader_stage stage,
+                                              rt_shader_optimization_level optimization,
+                                              rt_text_span code,
+    const char *file_path,
+                                              rt_arena *arena);
+#endif
+
+static rt_shader_bytecode CompileNullShader(rt_shader_stage stage,
+                                            rt_shader_optimization_level optimization,
+                                            rt_text_span code,
+    const char *file_path,
+                                            rt_arena *arena) {
+    RT_UNUSED(stage);
+    RT_UNUSED(optimization);
+    RT_UNUSED(code);
+    RT_UNUSED(arena);
+    rtLog("AC", "Attempted to compile shader %s with unsupported type.", file_path);
+    return (rt_shader_bytecode){
+        .bytes = NULL,
+        .len   = 0,
+    };
+}
+
+typedef rt_shader_bytecode
+shader_compiler_fn(rt_shader_stage, rt_shader_optimization_level, rt_text_span, const char *, rt_arena *);
+
+static shader_compiler_fn *_compiler_funcs[RT_SHADER_TYPE_COUNT] = {
+    CompileNullShader,
+
+#ifdef RT_BUILD_VULKAN_SHADER_COMPILER
+    CompileVulkanShader,
+#else
+    CompileNullShader,
+#endif
+
+};
+
+rt_shader_bytecode CompileShader(rt_shader_type type,
+                                 rt_shader_stage stage,
+                                 rt_shader_optimization_level optimization,
+                                 rt_text_span code,
+                                 const char *file_path,
+                                 rt_arena *arena) {
+    if (type >= RT_SHADER_TYPE_COUNT) {
+        rtLog("AC", "Invalid shader type %u", type);
+        return (rt_shader_bytecode){.bytes = NULL, .len = 0};
+    }
+    return _compiler_funcs[type](stage, optimization, code, file_path, arena);
+}

src/runtime/shader_compiler.h

@@ -0,0 +1,34 @@
+#ifndef RT_SHADER_COMPILER_H
+#define RT_SHADER_COMPILER_H
+
+#include "runtime.h"
+#include "mem_arena.h"
+
+typedef enum
+{
+    RT_SHADER_TYPE_INVALID,
+    RT_SHADER_TYPE_VULKAN,
+
+    RT_SHADER_TYPE_COUNT,
+} rt_shader_type;
+
+typedef enum {
+    RT_SHADER_STAGE_VERTEX,
+    RT_SHADER_STAGE_FRAGMENT,
+    RT_SHADER_STAGE_COMPUTE,
+} rt_shader_stage;
+
+typedef enum {
+    RT_SHADER_OPTIMIZATION_NONE,
+    RT_SHADER_OPTIMIZATION_SPEED,
+    RT_SHADER_OPTIMIZATION_SIZE,
+} rt_shader_optimization_level;
+
+typedef struct {
+    void *bytes;
+    size_t len;
+} rt_shader_bytecode;
+
+rt_shader_bytecode CompileShader(rt_shader_type type, rt_shader_stage stage, rt_shader_optimization_level optimization, rt_text_span code, const char *input_file, rt_arena *arena);
+
+#endif

src/runtime/threading.h

@@ -75,4 +75,6 @@ RT_DLLEXPORT rt_thread_id rtGetCurrentThreadId(void);
 
 RT_DLLEXPORT bool rtIsMainThread(void);
 
+RT_DLLEXPORT void rtSleep(unsigned int milliseconds);
+
 #endif

src/runtime/threading_mutex.c

@@ -41,19 +41,23 @@ RT_DLLEXPORT rt_mutex *rtCreateMutex(void) {
         rt_mutex *mtx   = &_mutex[_first_reusable];
         _first_reusable = mtx->next_reusable;
         mtx->owner      = 0;
+        ReleaseMutex(_guard);
         return mtx;
     } else if (_next < MAX_MUTEX) {
         rt_mutex *mtx = &_mutex[_next];
         mtx->handle   = CreateMutexW(NULL, FALSE, NULL);
         if (!mtx->handle) {
             rtLog("core", "Mutex creation failed: %u", GetLastError());
+            ReleaseMutex(_guard);
             return NULL;
         }
         mtx->next_reusable = MAX_MUTEX;
         mtx->owner         = 0;
         ++_next;
+        ReleaseMutex(_guard);
         return mtx;
     }
+    ReleaseMutex(_guard);
     rtReportError("core", "Ran out of mutex objects");
     return NULL;
 }

src/runtime/threading_thread.c

@@ -123,11 +123,18 @@ RT_DLLEXPORT bool rtIsMainThread(void) {
     return rtGetCurrentThreadId() == _main_thread_id;
 }
 
+RT_DLLEXPORT void rtSleep(unsigned int milliseconds) {
+    SleepEx(milliseconds, TRUE);
+}
+
 #elif defined(__linux__)
 
 #define _GNU_SOURCE
 #include <pthread.h>
 #include <unistd.h>
+#if _POSIX_C_SOURCE >= 199309L
+#include <time.h>
+#endif
 
 struct rt_thread_s {
     pthread_t handle;
@@ -208,5 +215,17 @@ RT_DLLEXPORT bool rtIsMainThread(void) {
     return rtGetCurrentThreadId() == _main_thread_id;
 }
 
+RT_DLLEXPORT void rtSleep(unsigned int milliseconds) {
+#if _POSIX_C_SOURCE >= 199309L
+    struct timespec ts;
+    ts.tv_sec  = milliseconds / 1000;
+    ts.tv_nsec = (milliseconds % 1000) * 1000000;
+    nanosleep(&ts, NULL);
+#else
+    if (milliseconds >= 1000)
+        sleep(milliseconds / 1000);
+    usleep((milliseconds % 1000) * 1000);
+#endif
+}
 
 #endif

src/runtime/uidtab.c

@@ -1,121 +0,0 @@
-#define RT_DEFINE_UIDTAB_FILE_STRUCTURES
-#include "uidtab.h"
-#include "aio.h"
-
-#include "xxhash/xxhash.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdbool.h>
-
-typedef struct {
-    rt_uid *uids;
-    rt_uid_data *data;
-    uint32_t slots;
-} rt_uidtab;
-
-static rt_uidtab _tab;
-
-rt_result LoadUIDTable(void) {
-    /* We use stdio here, because we cannot load any asset in parallel to this.
-     * This is because the uidtab is what tells us which assets exist.
-     */
-    FILE *f = fopen("data/uidtab.bin", "rb");
-    if (!f)
-        return RT_LOAD_FAILED;
-
-    rt_uidtab_header header;
-    if (fread(&header, sizeof(header), 1, f) != 1) {
-        fclose(f);
-        return RT_LOAD_FAILED;
-    }
-
-    /* TODO(Kevin): For some reason, the checksum calculation causes
-     *  Memory access errors .
-    XXH3_state_t *checksum = XXH3_createState();
-    if (!checksum) {
-        fclose(f);
-        return RT_UNKNOWN_ERROR;
-    }
-    */
-
-    _tab.slots = rtNextPowerOfTwo32(header.num_entries * 2);
-    void *mem  = malloc((sizeof(rt_uid) + sizeof(rt_uid_data)) * _tab.slots);
-    if (!mem) {
-        fclose(f);
-        _tab.slots = 0;
-        return RT_OUT_OF_MEMORY;
-    }
-    _tab.uids = mem;
-    _tab.data = (rt_uid_data *)(_tab.uids + _tab.slots);
-    memset(mem, 0, (sizeof(rt_uid) + sizeof(rt_uid_data)) * _tab.slots);
-
-    uint32_t mod = _tab.slots - 1; 
-    for (uint32_t i = 0; i < header.num_entries; ++i) {
-        rt_uidtab_entry entry;
-        if (fread(&entry, sizeof(entry), 1, f) != 1) {
-            free(mem);
-            _tab.slots = 0;
-            fclose(f);
-            return RT_LOAD_FAILED;
-        }
-        //XXH3_64bits_update(checksum, &entry, sizeof(entry));
-
-        /* Insert into hashtable */
-        bool inserted = false;
-        for (uint32_t j = 0; j < _tab.slots; ++j) {
-            uint32_t at = (entry.uid + j) & mod;
-            if (_tab.uids[at] == RT_INVALID_UID) {
-                _tab.uids[at] = entry.uid;
-                _tab.data[at].pkg_file = entry.file;
-                _tab.data[at].size     = entry.size;
-                _tab.data[at].offset   = entry.offset;
-                
-                inserted = true;
-                break;
-            }
-        }
-        if (!inserted) {
-            rtReportError("core",
-                          "Failed to insert an entry into the uid table. This should not happen.");
-            fclose(f);
-            free(mem);
-            _tab.slots = 0;
-            return RT_UNKNOWN_ERROR;
-        }
-    }
-
-    fclose(f);
-
-    /*
-    XXH64_hash_t checksum_hash = XXH3_64bits_digest(checksum);
-    XXH64_hash_t file_hash     = XXH64_hashFromCanonical(&header.checksum);
-    XXH3_freeState(checksum);
-
-    if (checksum_hash != file_hash) {
-        rtLog("core",
-              "WARNING: uidtab.bin checksum does not match calculated checksum of loaded entries.");
-    }
-    */
-    
-    return RT_SUCCESS;
-}
-
-void ReleaseUIDTable(void) {
-    free(_tab.uids);
-    _tab.slots = 0;
-}
-
-RT_DLLEXPORT const rt_uid_data *rtGetUIDData(rt_uid uid) {
-    uint32_t mod = _tab.slots - 1;
-    for (uint32_t j = 0; j < _tab.slots; ++j) {
-        uint32_t at = (uid + j) & mod;
-        if (_tab.uids[at] == uid) {
-            return &_tab.data[at];
-        } else if (_tab.uids[at] == RT_INVALID_UID) {
-            break;
-        }
-    }
-    return NULL;
-}

src/runtime/uidtab.h

@@ -1,36 +0,0 @@
-#ifndef RT_UIDTAB_H
-#define RT_UIDTAB_H
-
-#include "runtime.h"
-#include "file_tab.h"
-#include "assets.h"
-#include "xxhash/xxhash.h"
-
-#ifdef RT_DEFINE_UIDTAB_FILE_STRUCTURES
-#pragma pack(push, 1)
-typedef struct {
-    XXH64_canonical_t checksum;
-    uint32_t num_entries;
-} rt_uidtab_header;
-#pragma pack(pop)
-
-#pragma pack(push, 1)
-typedef struct {
-    rt_file_id file;
-    uint64_t offset;
-    uint64_t size;
-    rt_uid uid;
-} rt_uidtab_entry;
-#pragma pack(pop)
-#endif
-
-/* Data associated with an uid */
-typedef struct {
-    rt_file_id pkg_file;
-    uint64_t offset;
-    uint64_t size;
-} rt_uid_data;
-
-RT_DLLEXPORT const rt_uid_data *rtGetUIDData(rt_uid uid);
-
-#endif

src/runtime/vulkan_shader_compiler.c

@@ -0,0 +1,171 @@
+#include "asset_compiler.h"
+#include "config.h"
+#include "shader_compiler.h"
+#include "buffer_manager.h"
+
+#include <shaderc/shaderc.h>
+#include <string.h>
+
+RT_CVAR_S(rt_VkShaderCompilerShaderBasepath,
+          "Basepath for standard includes (<source>). Default: assets/shader/",
+          "assets/shader/");
+
+static shaderc_include_result *IncludeResolveCallback(void *user_data,
+                                                      const char *requested_source,
+                                                      int type,
+                                                      const char *requesting_source,
+                                                      size_t include_depth) {
+    RT_UNUSED(include_depth);
+    rt_arena *arena = user_data;
+
+    shaderc_include_result *result = RT_ARENA_PUSH_STRUCT_ZERO(arena, shaderc_include_result);
+    if (!result)
+        return NULL;
+
+    /* Resolve the path */
+    const char *basepath = NULL;
+    switch (type) {
+    case shaderc_include_type_standard:
+        basepath = rt_VkShaderCompilerShaderBasepath.s;
+        break;
+    case shaderc_include_type_relative: {
+        size_t len = strlen(requesting_source);
+        char *tmp  = rtArenaPush(arena, len + 1);
+        if (!tmp) {
+            rtReportError("AC", "Out of memory in the vulkan shader compiler.");
+            result->content = "Out of memory in the vulkan shader compiler.";
+            result->content_length = strlen(result->content);
+            return result;
+        }
+        memcpy(tmp, requesting_source, len + 1);
+        while (len > 0 && tmp[len] != '/' && tmp[len] != '\\')
+            --len;
+        tmp[len + 1] = '\0';
+        basepath     = tmp;
+    } break;
+    default:
+        rtReportError("AC", "Invalid include type reported in %s", requesting_source);
+        return result;
+    }
+    size_t basepath_len  = strlen(basepath);
+    size_t requested_len = strlen(requested_source);
+    char *full_path      = rtArenaPush(arena, basepath_len + requested_len + 1);
+    if (!full_path) {
+        rtReportError("AC", "Out of memory in vulkan shader compiler.");
+        result->content        = "Out of memory in the vulkan shader compiler.";
+        result->content_length = strlen(result->content);
+    }
+    memcpy(full_path, basepath, basepath_len);
+    memcpy(full_path + basepath_len, requested_source, requested_len);
+    full_path[basepath_len + requested_len] = '\0';
+
+    rt_loaded_asset source = LoadAsset(rtAddFile(full_path));
+    if (source.buffer) {
+        result->content        = source.buffer;
+        result->content_length = source.size;
+        result->source_name    = full_path;
+        result->source_name_length = basepath_len + requested_len + 1;
+    } else {
+        char *error_tmpl = "Could not open ";
+        char *error_msg = rtArenaPush(arena, strlen(error_tmpl) + basepath_len + requested_len + 1);
+        if (error_msg) {
+            memcpy(error_msg, error_tmpl, strlen(error_tmpl));
+            memcpy(error_msg + strlen(error_tmpl), full_path, basepath_len + requested_len + 1);
+            result->content = error_msg;
+            result->content_length = strlen(error_tmpl) + basepath_len + requested_len;
+        } else {
+            result->content        = "Could not open header file.";
+            result->content_length = strlen(result->content);
+        }
+    }
+
+    return result;
+}
+
+static void IncludeReleaseCallback(void *user_data, shaderc_include_result *include_result) {
+    RT_UNUSED(user_data);
+    if (include_result->source_name) {
+        /* Include result contains a valid file buffer */
+        rtReleaseBuffer(include_result->content, include_result->content_length);
+    }
+}
+
+rt_shader_bytecode CompileVulkanShader(rt_shader_stage stage,
+                                       rt_shader_optimization_level optimization,
+                                       rt_text_span code,
+                                       const char *file_path,
+                                       rt_arena *arena) {
+    rt_arena_rewindpoint arena_rewind = rtGetArenaRewindPoint(arena);
+
+    shaderc_shader_kind kind;
+    const char *entry;
+    switch (stage) {
+    case RT_SHADER_STAGE_VERTEX:
+        kind  = shaderc_vertex_shader;
+        entry = "VsMain";
+        break;
+    case RT_SHADER_STAGE_FRAGMENT:
+        kind  = shaderc_fragment_shader;
+        entry = "PsMain";
+        break;
+    case RT_SHADER_STAGE_COMPUTE:
+        kind  = shaderc_compute_shader;
+        entry = "CsMain";
+        break;
+    default:
+        rtLog("AC", "Invalid shader stage %u", stage);
+        return (rt_shader_bytecode){.bytes = NULL, .len = 0};
+    }
+
+    shaderc_compiler_t compiler = shaderc_compiler_initialize();
+    if (!compiler) {
+        rtLog("AC", "Failed to initialize the shaderc compiler.");
+        return (rt_shader_bytecode){.bytes = NULL, .len = 0};
+    }
+    shaderc_compile_options_t options = shaderc_compile_options_initialize();
+    if (!options) {
+        rtLog("AC", "Failed to initialize the shaderc compile options.");
+        shaderc_compiler_release(compiler);
+        return (rt_shader_bytecode){.bytes = NULL, .len = 0};
+    }
+    shaderc_compile_options_add_macro_definition(options, "RT_VULKAN", 9, NULL, 0);
+    shaderc_compile_options_set_source_language(options, shaderc_source_language_hlsl);
+    if (optimization == RT_SHADER_OPTIMIZATION_SIZE)
+        shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_size);
+    else if (optimization == RT_SHADER_OPTIMIZATION_SPEED)
+        shaderc_compile_options_set_optimization_level(options,
+                                                       shaderc_optimization_level_performance);
+    shaderc_compile_options_set_include_callbacks(options,
+                                                  IncludeResolveCallback,
+                                                  IncludeReleaseCallback,
+                                                  arena);
+
+    shaderc_compilation_result_t res = shaderc_compile_into_spv(compiler,
+                                                                code.start,
+                                                                code.length,
+                                                                kind,
+                                                                file_path,
+                                                                entry,
+                                                                options);
+    rt_shader_bytecode bytecode      = {.bytes = NULL, .len = 0};
+    if (shaderc_result_get_compilation_status(res) != shaderc_compilation_status_success) {
+        rtLog("AC", "Shader compilation failed: %s", shaderc_result_get_error_message(res));
+        goto out;
+    }
+    bytecode.len = shaderc_result_get_length(res);
+    if (bytecode.len > 0) {
+        bytecode.bytes = rtArenaPush(arena, bytecode.len);
+        if (!bytecode.bytes) {
+            rtLog("AC", "Failed to allocate %zu bytes for shader spv.", bytecode.len);
+            bytecode.len = 0;
+            goto out;
+        }
+        memcpy(bytecode.bytes, shaderc_result_get_bytes(res), bytecode.len);
+    }
+
+out:
+    shaderc_compile_options_release(options);
+    shaderc_compiler_release(compiler);
+    rtArenaRewind(arena, arena_rewind);
+    return bytecode;
+}