From 3254af37867c58e4a2be92683a871ea712db52c9 Mon Sep 17 00:00:00 2001 From: Kevin Trogant Date: Thu, 25 Jan 2024 09:45:23 +0100 Subject: [PATCH] Make progress towards the new builtin asset compiler Attempts to compile HLSL shaders (with includes) --- assets/shader/cell.as | 1 - assets/shader/cell.pipeline | 13 - assets/shader/cell_frag.as | 2 - assets/shader/cell_frag.glsl | 8 - assets/shader/cell_vert.as | 2 - assets/shader/cell_vert.glsl | 6 - assets/shader/test.as | 1 - assets/shader/test.hlsl | 1 + assets/shader/test.pipeline | 11 +- meson.build | 60 ++++- meson_options.txt | 1 + scripts/download_shaderc.bat | 1 + src/runtime/Source.cpp | 0 src/runtime/aio.c | 14 +- src/runtime/asset_cache.c | 379 --------------------------- src/runtime/asset_compiler.c | 342 ++++++++++++++++++++++++ src/runtime/asset_compiler.h | 22 ++ src/runtime/asset_dependencies.c | 137 ---------- src/runtime/asset_dependencies.h | 33 --- src/runtime/asset_loading.c | 74 ------ src/runtime/asset_manager.c | 0 src/runtime/assets.h | 38 +-- src/runtime/buffer_manager.c | 158 ++++++++++- src/runtime/description_parser.c | 252 ++++++++++++++++++ src/runtime/description_parser.h | 56 ++++ src/runtime/error_report.c | 14 +- src/runtime/file_tab.c | 6 +- src/runtime/fsutils.c | 115 ++++++++ src/runtime/fsutils.h | 35 +++ src/runtime/hashing.c | 10 + src/runtime/hashing.h | 12 + src/runtime/init.c | 50 ++-- src/runtime/mem_arena.c | 61 +++++ src/runtime/mem_arena.h | 57 ++++ src/runtime/pipeline_processor.c | 370 ++++++++++++++++++++++++++ src/runtime/renderer_api.h | 3 +- src/runtime/runtime.h | 1 - src/runtime/shader_compiler.c | 52 ++++ src/runtime/shader_compiler.h | 34 +++ src/runtime/threading.h | 2 + src/runtime/threading_mutex.c | 4 + src/runtime/threading_thread.c | 19 ++ src/runtime/uidtab.c | 121 --------- src/runtime/uidtab.h | 36 --- src/runtime/vulkan_shader_compiler.c | 171 ++++++++++++ 45 files changed, 1876 insertions(+), 909 deletions(-) delete mode 100644 assets/shader/cell.as delete mode 100644 assets/shader/cell.pipeline delete mode 100644 assets/shader/cell_frag.as delete mode 100644 assets/shader/cell_frag.glsl delete mode 100644 assets/shader/cell_vert.as delete mode 100644 assets/shader/cell_vert.glsl delete mode 100644 assets/shader/test.as create mode 100644 assets/shader/test.hlsl create mode 100644 src/runtime/Source.cpp delete mode 100644 src/runtime/asset_cache.c create mode 100644 src/runtime/asset_compiler.c create mode 100644 src/runtime/asset_compiler.h delete mode 100644 src/runtime/asset_dependencies.c delete mode 100644 src/runtime/asset_dependencies.h delete mode 100644 src/runtime/asset_loading.c create mode 100644 src/runtime/asset_manager.c create mode 100644 src/runtime/description_parser.c create mode 100644 src/runtime/description_parser.h create mode 100644 src/runtime/fsutils.c create mode 100644 src/runtime/fsutils.h create mode 100644 src/runtime/hashing.c create mode 100644 src/runtime/hashing.h create mode 100644 src/runtime/mem_arena.c create mode 100644 src/runtime/mem_arena.h create mode 100644 src/runtime/pipeline_processor.c create mode 100644 src/runtime/shader_compiler.c create mode 100644 src/runtime/shader_compiler.h delete mode 100644 src/runtime/uidtab.c delete mode 100644 src/runtime/uidtab.h create mode 100644 src/runtime/vulkan_shader_compiler.c diff --git a/assets/shader/cell.as b/assets/shader/cell.as deleted file mode 100644 index 82568d6..0000000 --- a/assets/shader/cell.as +++ /dev/null @@ -1 +0,0 @@ -package pipelines.pkg; diff --git a/assets/shader/cell.pipeline b/assets/shader/cell.pipeline deleted file mode 100644 index 79353e1..0000000 --- a/assets/shader/cell.pipeline +++ /dev/null @@ -1,13 +0,0 @@ -optimization speed; - -vertex { - vk assets/shader/cell_vert.glsl; -} -fragment { - vk assets/shader/cell_frag.glsl; -} - -texture_bindings { - 0 MATERIAL_ALBEDO; - 1 MATERIAL_NORMAL; -} diff --git a/assets/shader/cell_frag.as b/assets/shader/cell_frag.as deleted file mode 100644 index e434dd3..0000000 --- a/assets/shader/cell_frag.as +++ /dev/null @@ -1,2 +0,0 @@ -package shaders.pkg; -processing_flags 0002; \ No newline at end of file diff --git a/assets/shader/cell_frag.glsl b/assets/shader/cell_frag.glsl deleted file mode 100644 index 994ddf3..0000000 --- a/assets/shader/cell_frag.glsl +++ /dev/null @@ -1,8 +0,0 @@ -#version 450 -#pragma shader_stage(fragment) - -layout (location = 0) out vec3 color; - -void main() { - color = vec3(1, 1, 1); -} diff --git a/assets/shader/cell_vert.as b/assets/shader/cell_vert.as deleted file mode 100644 index 2dda451..0000000 --- a/assets/shader/cell_vert.as +++ /dev/null @@ -1,2 +0,0 @@ -package shaders.pkg; -processing_flags 0001; \ No newline at end of file diff --git a/assets/shader/cell_vert.glsl b/assets/shader/cell_vert.glsl deleted file mode 100644 index 8511547..0000000 --- a/assets/shader/cell_vert.glsl +++ /dev/null @@ -1,6 +0,0 @@ -#version 450 -#pragma shader_stage(vertex) - -void main() { - gl_Position = vec4(0, 0, 0, 1); -} diff --git a/assets/shader/test.as b/assets/shader/test.as deleted file mode 100644 index 82568d6..0000000 --- a/assets/shader/test.as +++ /dev/null @@ -1 +0,0 @@ -package pipelines.pkg; diff --git a/assets/shader/test.hlsl b/assets/shader/test.hlsl new file mode 100644 index 0000000..59a309a --- /dev/null +++ b/assets/shader/test.hlsl @@ -0,0 +1 @@ +void TestFn() {} \ No newline at end of file diff --git a/assets/shader/test.pipeline b/assets/shader/test.pipeline index ec6bbf8..fe6f33b 100644 --- a/assets/shader/test.pipeline +++ b/assets/shader/test.pipeline @@ -1,7 +1,10 @@ optimization speed; -vertex BEGIN -layout (location = 0) uniform vec3 foo; -void main() { +vertex { + vk BEGIN +#include "test.hlsl" + +void VsMain() { +} + END } -END diff --git a/meson.build b/meson.build index 1475939..13dade3 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,10 @@ project('voyage', 'c', - default_options: ['buildtype=debug', 'b_sanitize=address', 'c_std=c17', 'warning_level=3']) + default_options: ['buildtype=debug', + 'b_sanitize=address', + 'c_std=c17', + 'warning_level=3', + 'werror=true', + 'b_vscrt=static_from_buildtype']) compiler = meson.get_compiler('c') buildtype = get_option('buildtype') @@ -51,63 +56,90 @@ runtime_incdirs = common_incdirs runtime_linkargs = [] runtime_additional_sources = [] runtime_cargs = [] +runtime_deps = [thread_dep, m_dep, windowing_dep] + if get_option('build_asset_compiler') runtime_cargs += ['-DRT_BUILD_ASSET_COMPILER'] # Shaderc for shaders - shaderc_include = include_directories('contrib/shaderc/libshaderc/include') - shaderc_libdir = 'NONE' - if host_machine.system() == 'windows' - shaderc_libdir = meson.project_source_root() / 'contrib/shaderc/build-win/libshaderc/Release' + if get_option('enable_vulkan_shader_compiler') + shaderc_include = include_directories('contrib\\shaderc\\libshaderc\\include') + shaderc_libdir = 'NONE' + if host_machine.system() == 'windows' + if buildtype == 'debug' or buildtype == 'debugoptimized' + shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Debug' + else + shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Release' + endif + endif + shaderc_dep = declare_dependency(link_args : ['-L'+shaderc_libdir, '-lshaderc_combined'], + include_directories : shaderc_include) + runtime_deps += shaderc_dep + runtime_additional_sources += [ + 'src/runtime/vulkan_shader_compiler.c' + ] + runtime_cargs += ['-DRT_BUILD_VULKAN_SHADER_COMPILER'] endif - runtime_incdirs += shaderc_include - runtime_linkargs += ['-L'+shaderc_libdir, '-lshaderc_combined'] + + # Asset compiler sources + runtime_additional_sources += [ + 'src/runtime/asset_compiler.h', + 'src/runtime/description_parser.h', + 'src/runtime/shader_compiler.h', + + 'src/runtime/asset_compiler.c', + 'src/runtime/description_parser.c', + 'src/runtime/pipeline_processor.c', + 'src/runtime/shader_compiler.c', + ] endif runtime_lib = library('rt', # Project Sources 'src/runtime/aio.h', 'src/runtime/app.h', - 'src/runtime/asset_dependencies.h', 'src/runtime/assets.h', 'src/runtime/buffer_manager.h', 'src/runtime/config.h', 'src/runtime/dynamic_libs.h', 'src/runtime/file_tab.h', + 'src/runtime/fsutils.h', 'src/runtime/gfx.h', 'src/runtime/handles.h', + 'src/runtime/hashing.h', 'src/runtime/jobs.h', + 'src/runtime/mem_arena.h', 'src/runtime/packages.h', 'src/runtime/renderer_api.h', 'src/runtime/runtime.h', 'src/runtime/threading.h', - 'src/runtime/uidtab.h', 'src/runtime/aio.c', 'src/runtime/app.c', - 'src/runtime/asset_cache.c', - 'src/runtime/asset_dependencies.c', - 'src/runtime/asset_loading.c', + 'src/runtime/asset_manager.c', 'src/runtime/buffer_manager.c', 'src/runtime/config.c', 'src/runtime/dynamic_libs.c', 'src/runtime/error_report.c', 'src/runtime/file_tab.c', + 'src/runtime/fsutils.c', 'src/runtime/gfx_main.c', + 'src/runtime/hashing.c', 'src/runtime/init.c', 'src/runtime/jobs.c', + 'src/runtime/mem_arena.c', 'src/runtime/packages.c', 'src/runtime/text.c', 'src/runtime/threading_cond.c', 'src/runtime/threading_mutex.c', 'src/runtime/threading_rwlock.c', 'src/runtime/threading_thread.c', - 'src/runtime/uidtab.c', # Contrib Sources 'contrib/xxhash/xxhash.c', 'contrib/lz4/lz4.c', - dependencies : [thread_dep, m_dep, windowing_dep], + sources : runtime_additional_sources, + dependencies : runtime_deps, include_directories : runtime_incdirs, link_args : runtime_linkargs, c_args : runtime_cargs, diff --git a/meson_options.txt b/meson_options.txt index 6ce73ff..d830a50 120000 --- a/meson_options.txt +++ b/meson_options.txt @@ -1,3 +1,4 @@ option('use_xlib', type : 'boolean', value : false, description : 'Use Xlib for window creation under linux') option('error_report_debugbreak', type : 'boolean', value : true, description : 'Debugbreak in ReportError') option('build_asset_compiler', type : 'boolean', value : true, description : 'Enables or disables the asset compiler inside runtime.') +option('enable_vulkan_shader_compiler', type : 'boolean', value : true, description : 'Enables building the vulkan shader compiler (Requires shaderc).') diff --git a/scripts/download_shaderc.bat b/scripts/download_shaderc.bat index c694d04..482c46d 100644 --- a/scripts/download_shaderc.bat +++ b/scripts/download_shaderc.bat @@ -28,6 +28,7 @@ MKDIR build-win PUSHD build-win cmake .. cmake --build . --config Release +cmake --build . --config Debug POPD diff --git a/src/runtime/Source.cpp b/src/runtime/Source.cpp new file mode 100644 index 0000000..e69de29 diff --git a/src/runtime/aio.c b/src/runtime/aio.c index 886cece..ec5dd45 100644 --- a/src/runtime/aio.c +++ b/src/runtime/aio.c @@ -6,6 +6,8 @@ #define WIN32_LEAN_AND_MEAN #include +void Win32ErrorToString(DWORD last_error, char *out, int bufsize); + #elif defined(__linux__) #include @@ -183,10 +185,14 @@ RT_DLLEXPORT rt_result rtSubmitLoadBatch(const rt_load_batch *batch, rt_aio_hand FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, NULL); if (file_handle == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + char error_msg[256]; + Win32ErrorToString(err, error_msg, 256); rtReportError("aio", - "CreateFileW failed for file: %s with error code: %u", + "CreateFileW failed for file: %s with error code: %u (%s)", file_path, - GetLastError()); + err, + error_msg); op->state = RT_AIO_STATE_INVALID; handles[i] = RT_AIO_INVALID_HANDLE; continue; @@ -199,7 +205,9 @@ RT_DLLEXPORT rt_result rtSubmitLoadBatch(const rt_load_batch *batch, rt_aio_hand win32CompletionRoutine); DWORD err = GetLastError(); if (!result || err != ERROR_SUCCESS) { - rtReportError("aio", "ReadFileEx failed with error code: %u", err); + char error_msg[256]; + Win32ErrorToString(err, error_msg, 256); + rtReportError("aio", "ReadFileEx failed with error code: %u (%s)", err, error_msg); op->state = RT_AIO_STATE_FINISHED; handles[i] = RT_AIO_INVALID_HANDLE; CloseHandle(file_handle); diff --git a/src/runtime/asset_cache.c b/src/runtime/asset_cache.c deleted file mode 100644 index 9d6d462..0000000 --- a/src/runtime/asset_cache.c +++ /dev/null @@ -1,379 +0,0 @@ -#include "assets.h" -#include "asset_dependencies.h" -#include "aio.h" -#include "buffer_manager.h" -#include "config.h" -#include "threading.h" -#include "uidtab.h" - -#include -#include - -RT_CVAR_I(rt_AssetCacheSize, "Number of asset cache entries. Default: 1024.", 1024); - -/* asset_loading.c */ -extern rt_result DecompressAsset(void *compressed_buffer, - size_t compressed_buffer_size, - void **p_decompressed, - size_t *p_decompressed_size); - -typedef enum { - CACHE_ENTRY_STATE_FREE, - CACHE_ENTRY_STATE_LOADING, - CACHE_ENTRY_STATE_LOADED, -} rt_asset_cache_entry_state; - -typedef struct rt_asset_cache_entry_s { - rt_asset_cache_entry_state state; - rt_aio_handle load; - void *buffer; - size_t size; - int refcount; - - /* Reclaim list */ - struct rt_asset_cache_entry_s *prev_reclaim; - struct rt_asset_cache_entry_s *next_reclaim; -} rt_asset_cache_entry; - -static rt_uid *_uids; -static rt_asset_cache_entry *_entries; -static rt_asset_cache_entry *_first_reclaim; -static rt_asset_cache_entry *_last_reclaim; - -/* Locked as writer when modifiying entries, as reader when searching */ -static rt_rwlock _lock; - -rt_result InitAssetCache(void) { - _entries = calloc((size_t)rt_AssetCacheSize.i, sizeof(rt_asset_cache_entry)); - if (!_entries) { - return RT_BUFFER_ALLOC_FAILED; - } - _uids = calloc((size_t)rt_AssetCacheSize.i, sizeof(rt_uid)); - if (!_uids) { - free(_entries); - return RT_BUFFER_ALLOC_FAILED; - } - rt_create_rwlock_result lock_res = rtCreateRWLock(); - if (!lock_res.ok) { - free(_entries); - free(_uids); - return RT_UNKNOWN_ERROR; - } - _lock = lock_res.lock; - return RT_SUCCESS; -} - -void ShutdownAssetCache(void) { - free(_entries); - free(_uids); - rtDestroyRWLock(&_lock); - _first_reclaim = NULL; - _last_reclaim = NULL; -} - -static void ReleaseEntry(rt_asset_cache_entry *entry) { - if (entry->load != RT_AIO_INVALID_HANDLE) { - rtWaitForAIOCompletion(entry->load); - rtReleaseAIO(entry->load); - entry->load = RT_AIO_INVALID_HANDLE; - } - rtReleaseBuffer(entry->buffer, entry->size); - entry->buffer = NULL; - entry->size = 0; - entry->next_reclaim = NULL; - entry->prev_reclaim = NULL; -} - -static void GarbageCollect(void) { - rtLockWrite(&_lock); - rt_asset_cache_entry *entry = _first_reclaim; - while (entry) { - assert(entry->refcount == 0); - rt_asset_cache_entry *next = entry->next_reclaim; - if (entry->state == CACHE_ENTRY_STATE_LOADED) { - ReleaseEntry(entry); - _first_reclaim = next; - } - entry = next; - } - rtUnlockWrite(&_lock); -} - -static rt_asset_cache_entry *GetEntry(rt_uid uid) { - /* Hash lookup */ - unsigned int mod = (unsigned int)rt_AssetCacheSize.i - 1; - for (unsigned int i = 0; i < (unsigned int)rt_AssetCacheSize.i; ++i) { - unsigned int slot = (uid + i) & mod; - if (_uids[slot] == uid) { - return &_entries[slot]; - } else if (_uids[slot] == RT_INVALID_UID) { - break; - } - } - return NULL; -} - - -static bool IsAssetLoaded(rt_uid uid) { - const rt_asset_cache_entry *entry = GetEntry(uid); - if (entry) - return entry->state == CACHE_ENTRY_STATE_LOADED || - entry->state == CACHE_ENTRY_STATE_LOADING; - else - return false; -} - -static int InsertEntry(rt_uid uid) { - unsigned int mod = (unsigned int)rt_AssetCacheSize.i - 1; - for (unsigned int i = 0; i < (unsigned int)rt_AssetCacheSize.i; ++i) { - unsigned int slot = (uid + i) & mod; - if (_uids[slot] == 0) { - return (int)slot; - } - } - return -1; -} -static rt_result InsertAndLoadAssets(const rt_uid *uids, size_t count) { - rt_load_batch batch = {.num_loads = 0}; - - rt_result res = RT_SUCCESS; - - count = (count < RT_LOAD_BATCH_MAX_SIZE) ? count : RT_LOAD_BATCH_MAX_SIZE; - rt_asset_cache_entry *load_entries[RT_LOAD_BATCH_MAX_SIZE]; - - for (size_t i = 0; i < count; ++i) { - rtLockRead(&_lock); - bool needs_load = !IsAssetLoaded(uids[i]); - rtUnlockRead(&_lock); - - if (!needs_load) - continue; - - rtLockWrite(&_lock); - /* It's possible that another thread loaded the asset in the meantime */ - if (!IsAssetLoaded(uids[i])) { - const rt_uid_data *data = rtGetUIDData(uids[i]); - if (!data) { - rtUnlockWrite(&_lock); - rtLog("ASSET_CACHE", "Failed to get uid data for uid %u", uids[i]); - res = RT_UNKNOWN_ASSET; - continue; - } - - void *compressed_data = rtAllocBuffer(data->size); - if (!compressed_data) { - /* Try again after garbage collection */ - rtUnlockWrite(&_lock); - GarbageCollect(); - compressed_data = rtAllocBuffer(data->size); - if (!compressed_data) { - rtLog("ASSET_CACHE", - "Failed to allocate intermediate buffer for uid %u", - uids[i]); - res = RT_BUFFER_ALLOC_FAILED; - continue; - } - rtLockWrite(&_lock); - } - - int slot = InsertEntry(uids[i]); - if (slot == -1) { - rtUnlockWrite(&_lock); - rtLog("ASSET_CACHE", "Failed to insert new entry for uid %u", uids[i]); - res = RT_ASSET_CACHE_FULL; - break; - } - - rt_asset_cache_entry *entry = &_entries[slot]; - load_entries[batch.num_loads] = entry; - - /* We set the refcount to 0, but don't insert the entry - * into the reclaim list, to ensure that its buffer does not get freed - * while the load still executes. Setting the refcount to 0 ensures - * that the count is correct, once the asset is accessed the first time. */ - entry->state = CACHE_ENTRY_STATE_LOADING; - entry->refcount = 0; - entry->buffer = compressed_data; - entry->size = data->size; - entry->next_reclaim = NULL; - entry->prev_reclaim = NULL; - entry->load = RT_AIO_INVALID_HANDLE; - - batch.loads[batch.num_loads].file = data->pkg_file; - batch.loads[batch.num_loads].num_bytes = data->size; - batch.loads[batch.num_loads].offset = data->offset; - batch.loads[batch.num_loads].dest = compressed_data; - ++batch.num_loads; - } - } - rtUnlockWrite(&_lock); - - /* Dispatch the load */ - rt_aio_handle handles[RT_LOAD_BATCH_MAX_SIZE]; - if ((res = rtSubmitLoadBatch(&batch, handles)) != RT_SUCCESS) { - rtLog("ASSET_CACHE", "Failed to submit %u asset loads.", batch.num_loads); - return res; - } - - /* Set the aio handles of the inserted entries */ - rtLockWrite(&_lock); - for (unsigned int i = 0; i < batch.num_loads; ++i) { - load_entries[batch.num_loads]->load = handles[i]; - } - rtUnlockWrite(&_lock); - - return res; -} - -static bool DecompressEntry(rt_uid uid, rt_asset_cache_entry *entry) { - rtReleaseAIO(entry->load); - entry->load = RT_AIO_INVALID_HANDLE; - - void *decompressed_buffer; - size_t decompressed_size; - rt_result dec_res = - DecompressAsset(entry->buffer, entry->size, &decompressed_buffer, &decompressed_size); - if (dec_res == RT_SUCCESS) { - rtReleaseBuffer(entry->buffer, entry->size); - entry->buffer = decompressed_buffer; - entry->size = decompressed_size; - entry->state = CACHE_ENTRY_STATE_LOADED; - return true; - } else if (dec_res == RT_BUFFER_ALLOC_FAILED) { - GarbageCollect(); - /* Try again */ - if (DecompressAsset(entry->buffer, entry->size, &decompressed_buffer, &decompressed_size) == - RT_SUCCESS) { - rtReleaseBuffer(entry->buffer, entry->size); - entry->buffer = decompressed_buffer; - entry->size = decompressed_size; - entry->state = CACHE_ENTRY_STATE_LOADED; - return true; - } - /* Don't do anything yet. We might be able to to do this later, once some - * buffers become free. */ - rtLog("ASSET_CACHE", "Failed to decompress asset %u", uid); - return false; - } else { - rtLog("ASSET_CACHE", "Failed to decompress asset %u", uid); - ReleaseEntry(entry); - - ptrdiff_t idx = entry - _entries; - _uids[idx] = RT_INVALID_UID; - return false; - } -} - -static void CheckCompletedLoads(const rt_uid *uids, size_t count) { - for (size_t i = 0; i < count; ++i) { - rtLockRead(&_lock); - volatile rt_asset_cache_entry *entry = (volatile rt_asset_cache_entry *)GetEntry(uids[i]); - if (!entry) { - rtUnlockRead(&_lock); - rtLog("ASSET_CACHE", "Passed unknown uid %u to CheckCompletedLoads()", uids[i]); - continue; - } - - if (entry->state != CACHE_ENTRY_STATE_LOADING) { - rtUnlockRead(&_lock); - continue; - } - bool load_finished = rtGetAIOState(entry->load) == RT_AIO_STATE_FINISHED; - rtUnlockRead(&_lock); - - if (load_finished) { - rtLockWrite(&_lock); - /* Ensure that no-one else handled this */ - if (entry->state == CACHE_ENTRY_STATE_LOADING) { - DecompressEntry(uids[i], (rt_asset_cache_entry *)entry); - } - rtUnlockWrite(&_lock); - } - } -} - -RT_DLLEXPORT rt_get_asset_result rtGetAsset(rt_uid uid) { - rt_get_asset_result result = { - .result = RT_SUCCESS, - }; - - rtLockRead(&_lock); - bool needs_load = !IsAssetLoaded(uid); - rtUnlockRead(&_lock); - - if (needs_load) { - rt_uid load_uids[RT_LOAD_BATCH_MAX_SIZE]; - size_t load_count = 1; - load_uids[0] = uid; - - rt_asset_dependency_list deps = rtGetAssetDependencies(uid); - for (size_t i = 0; i < deps.count && i < RT_LOAD_BATCH_MAX_SIZE - 1; ++i) { - load_uids[i + 1] = deps.dependencies[i]; - ++load_count; - } - - result.result = InsertAndLoadAssets(load_uids, load_count); - if (result.result == RT_SUCCESS) { - CheckCompletedLoads(load_uids, load_count); - } - } - - rtLockWrite(&_lock); - rt_asset_cache_entry *entry = GetEntry(uid); - if (entry) { - if (entry->state == CACHE_ENTRY_STATE_LOADED) { - ++entry->refcount; - result.data = entry->buffer; - result.size = entry->size; - } else if (entry->state == CACHE_ENTRY_STATE_LOADING) { - if (entry->state == CACHE_ENTRY_STATE_LOADING) { - assert(entry->load != RT_AIO_INVALID_HANDLE); - ++entry->refcount; - if (rtWaitForAIOCompletion(entry->load) == RT_AIO_STATE_FINISHED) { - if (DecompressEntry(uid, entry)) { - result.data = entry->buffer; - result.size = entry->size; - } else { - result.result = RT_LOAD_FAILED; - } - } else { - ReleaseEntry(entry); - rtLog("ASSET_CACHE", "Failed to load asset %u", uid); - result.result = RT_LOAD_FAILED; - } - } - } - - /* Remove from the reclaim list */ - if (_first_reclaim == entry) - _first_reclaim = entry->next_reclaim; - if (_last_reclaim == entry) - _last_reclaim = entry->prev_reclaim; - if (entry->next_reclaim) - entry->next_reclaim->prev_reclaim = entry->prev_reclaim; - if (entry->prev_reclaim) - entry->prev_reclaim->next_reclaim = entry->next_reclaim; - } - rtUnlockWrite(&_lock); - - return result; -} - -RT_DLLEXPORT void rtReleaseAsset(rt_uid uid) { - rtLockWrite(&_lock); - rt_asset_cache_entry *entry = GetEntry(uid); - if (entry && entry->refcount > 0) { - --entry->refcount; - if (entry->refcount == 0) { - /* add to the reclaim list */ - if (_last_reclaim) - _last_reclaim->next_reclaim = entry; - if (!_first_reclaim) - _first_reclaim = entry; - entry->prev_reclaim = _last_reclaim; - entry->next_reclaim = NULL; - _last_reclaim = entry; - } - } - rtUnlockWrite(&_lock); -} \ No newline at end of file diff --git a/src/runtime/asset_compiler.c b/src/runtime/asset_compiler.c new file mode 100644 index 0000000..6a8df6f --- /dev/null +++ b/src/runtime/asset_compiler.c @@ -0,0 +1,342 @@ +#include "runtime.h" +#include "threading.h" +#include "config.h" +#include "fsutils.h" +#include "file_tab.h" +#include "mem_arena.h" +#include "buffer_manager.h" + +#include +#include +#include +#include + +#ifndef RT_BUILD_ASSET_COMPILER +#error This should only be built when RT_BUILD_ASSET_COMPILER is defined. +#endif + +typedef struct { + uint64_t last_processed; +} rt_asset_data; + +typedef struct { + rt_file_id *files; + rt_asset_data *data; +} rt_asset_db; + +typedef rt_result rt_asset_processor_fn(rt_file_id file, rt_arena *arena); + +typedef struct { + const char *file_ext; + rt_asset_processor_fn *proc; +} rt_asset_processor; + +typedef struct { + unsigned int processor_index; + unsigned int db_index; + rt_file_id fid; +} rt_processing_queue_entry; + +typedef struct { + rt_processing_queue_entry entries[1024]; + unsigned int head; + unsigned int tail; + rt_condition_var *lock; +} rt_processing_queue; + +RT_CVAR_S(rt_AssetDirectory, "Name of the asset directory. Default: assets", "assets"); +RT_CVAR_I(rt_AssetDBSize, "Size of the asset database. Default: 1024", 1024); +RT_CVAR_I(rt_AssetProcessingThreads, "Number of asset processing threads. Default: 4", 4); +RT_CVAR_I(rt_AssetProcessorArenaSize, + "Size of the per-thread asset processor arena. Default: 128 MiB", + (int)RT_MB(128)); + +#define MAX_PROCESSING_THREADS 8 +static rt_thread *_compiler_thread; +static rt_thread *_processing_threads[MAX_PROCESSING_THREADS]; +static bool _keep_running = true; + +static rt_asset_db _asset_db; + +static rt_processing_queue _processing_queue; + +extern rt_result PipelineProcessor(rt_file_id file, rt_arena *arena); + +static rt_asset_processor _processors[] = { + {.file_ext = ".pipeline", .proc = PipelineProcessor} +}; + +static void ProcessorThreadEntry(void *); +static void CompilerThreadEntry(void *); + +rt_result InitAssetCompiler(void) { + + unsigned int db_size = (unsigned int)rt_AssetDBSize.i; + void *mem = malloc((sizeof(rt_file_id) + sizeof(rt_asset_data)) * db_size); + if (!mem) + return RT_OUT_OF_MEMORY; + _asset_db.files = mem; + _asset_db.data = (rt_asset_data *)(_asset_db.files + db_size); + memset(mem, 0, (sizeof(rt_file_id) + sizeof(rt_asset_data)) * db_size); + + _processing_queue.lock = rtCreateConditionVar(); + if (!_processing_queue.lock) { + free(mem); + return RT_UNKNOWN_ERROR; + } + + _keep_running = true; + _compiler_thread = rtSpawnThread(CompilerThreadEntry, NULL, "AssetCompilerThread"); + if (!_compiler_thread) + return RT_UNKNOWN_ERROR; + + if (rt_AssetProcessingThreads.i > MAX_PROCESSING_THREADS) + rt_AssetProcessingThreads.i = MAX_PROCESSING_THREADS; + for (int i = 0; i < rt_AssetProcessingThreads.i; ++i) { + char name[64]; + snprintf(name, 64, "AssetProcessorThread %d", i); + _processing_threads[i] = rtSpawnThread(ProcessorThreadEntry, NULL, name); + if (!_processing_threads[i]) { + /* Wake the processing threads */ + rtLockConditionVar(_processing_queue.lock); + rtUnlockConditionVar(_processing_queue.lock, true); + _keep_running = false; + for (int j = 0; j < i; ++j) + rtJoinThread(_processing_threads[j]); + rtJoinThread(_compiler_thread); + free(mem); + rtDestroyConditionVar(_processing_queue.lock); + return RT_UNKNOWN_ERROR; + } + } + + return RT_SUCCESS; +} + +void ShutdownAssetCompiler(void) { + _keep_running = false; + rtJoinThread(_compiler_thread); + for (int i = 0; i < rt_AssetProcessingThreads.i; ++i) + rtJoinThread(_processing_threads[i]); + free(_asset_db.files); + rtDestroyConditionVar(_processing_queue.lock); +} + +static int DiscoverAssets(void) { + /* Recursive descend into the asset directory */ +#define MAX_DISCOVERY_DEPTH 64 +#define MAX_FILENAME_LEN 260 + static char directory_stack[MAX_DISCOVERY_DEPTH][MAX_FILENAME_LEN]; + static unsigned int path_lens[MAX_DISCOVERY_DEPTH]; + unsigned int top = 0; + + memcpy(directory_stack[0], + rt_AssetDirectory.s, + (strlen(rt_AssetDirectory.s) <= (MAX_FILENAME_LEN - 1)) ? strlen(rt_AssetDirectory.s) + : MAX_FILENAME_LEN); + directory_stack[0][MAX_FILENAME_LEN - 1] = '\0'; + ++top; + + int discovery_count = 0; + + while (top > 0) { + /* Process the directory */ + char dir[MAX_FILENAME_LEN]; + memcpy(dir, directory_stack[top - 1], MAX_FILENAME_LEN); + --top; + + /* Append to the path */ + size_t dir_len = strlen(dir); + + rt_scandir_handle *scan = rtScanDirectory(dir); + if (!scan) + continue; + rt_dirent entry; + do { + entry = rtNextDirectoryEntry(scan); + + if (entry.name[0] == '.') + continue; + + size_t entry_name_len = strlen(entry.name); + if (dir_len + entry_name_len + 1 >= MAX_FILENAME_LEN) { + rtLog("AC", + "Could not process %s\\%s because the name exceeds the maximum " + "allowed path length %d.", + dir, + entry.name, + MAX_FILENAME_LEN); + continue; + } + + if (entry.type == RT_DIRENT_TYPE_DIRECTORY) { + /* Push on stack */ + assert(MAX_FILENAME_LEN == RT_ARRAY_COUNT(entry.name)); + memcpy(directory_stack[top], dir, MAX_FILENAME_LEN); + directory_stack[top][dir_len] = '/'; + memcpy(&directory_stack[top][dir_len + 1], entry.name, entry_name_len); + directory_stack[top][dir_len + entry_name_len + 1] = '\0'; + ++top; + } else { + /* Check if the asset is new */ + char file[MAX_FILENAME_LEN]; + memcpy(file, dir, MAX_FILENAME_LEN); + file[dir_len] = '/'; + memcpy(&file[dir_len + 1], entry.name, entry_name_len); + file[dir_len + entry_name_len + 1] = '\0'; + + rt_file_id fid = rtAddFile(file); + unsigned int i = 0; + while (i < (unsigned int)rt_AssetDBSize.i) { + unsigned int slot = (fid + i) % (unsigned int)rt_AssetDBSize.i; + if (_asset_db.files[slot] == fid) { + break; + } else if (_asset_db.files[slot] == 0) { + _asset_db.files[slot] = fid; + _asset_db.data[slot].last_processed = 0; + ++discovery_count; + break; + } + ++i; + } + if (i == (unsigned int)rt_AssetDBSize.i) { + rtLog("AC", "Failed to add %s to AssetDB, because no free slots are left.", file); + } + } + } while (!entry.is_last); + rtCloseDirectory(scan); + } + return discovery_count; +#undef MAX_DISCOVERY_DEPTH +#undef MAX_FILENAME_LEN +} + +static int CheckUpdatedAssets(void) { + int updated_count = 0; + for (int i = 0; i < rt_AssetDBSize.i; ++i) { + if (_asset_db.files[i] == 0) + continue; + const char *path = rtGetFilePath(_asset_db.files[i]); + uint64_t last_changed = rtGetFileModificationTimestamp(path); + if (_asset_db.data[i].last_processed < last_changed) { + const char *ext = path + strlen(path); + while (*ext != '.' && ext != path) + --ext; + if (*ext != '.') + break; + + for (unsigned int j = 0; j < RT_ARRAY_COUNT(_processors); ++j) { + if (strcmp(ext, _processors[j].file_ext) == 0) { + rt_processing_queue_entry entry; + entry.fid = _asset_db.files[i]; + entry.processor_index = j; + entry.db_index = i; + + while (true) { + bool inserted = false; + rtLockConditionVar(_processing_queue.lock); + unsigned int next_tail = (_processing_queue.tail + 1) % + RT_ARRAY_COUNT(_processing_queue.entries); + if (next_tail != _processing_queue.head) { + _processing_queue.entries[_processing_queue.tail] = entry; + _processing_queue.tail = next_tail; + inserted = true; + } + rtUnlockConditionVar(_processing_queue.lock, inserted); + if (inserted) + break; + } + } + } + ++updated_count; + } + } + return updated_count; +} + +static void CompilerThreadEntry(void *param) { + RT_UNUSED(param); + + /* Mainloop. Watch for changed or new assets, compile them + * and notify the appropriate system */ + while (_keep_running) { + int d = DiscoverAssets(); + int u = CheckUpdatedAssets(); + if (d == 0 && u == 0) + rtSleep(100); + } +} + +static void ProcessorThreadEntry(void *param) { + RT_UNUSED(param); + + rt_create_arena_result arena_res = rtCreateArena(NULL, (size_t)rt_AssetProcessorArenaSize.i); + if (!arena_res.ok) { + rtLog("AC", + "Failed to allocate %d bytes for the processing arena.", + rt_AssetProcessorArenaSize.i); + return; + } + rt_arena arena = arena_res.arena; + + while (_keep_running) { + rtLockConditionVar(_processing_queue.lock); + + while (_keep_running && (_processing_queue.tail == _processing_queue.head)) + rtWaitOnConditionVar(_processing_queue.lock); + + bool got_entry = false; + rt_processing_queue_entry entry = {0}; + if (_processing_queue.tail != _processing_queue.head) { + entry = _processing_queue.entries[_processing_queue.head]; + _processing_queue.head = + (_processing_queue.head + 1) % RT_ARRAY_COUNT(_processing_queue.entries); + got_entry = true; + } + + rtUnlockConditionVar(_processing_queue.lock, false); + + if (!got_entry) + continue; + + const char *path = rtGetFilePath(entry.fid); + rtLog("AC", "Processing %s", path); + rtArenaClear(&arena); + rt_result res = _processors[entry.processor_index].proc(entry.fid, &arena); + if (res != RT_SUCCESS) { + rtLog("AC", "Failed to process %s: %u", path, res); + } + _asset_db.data[entry.db_index].last_processed = rtGetCurrentTimestamp(); + } +} + +/* Utilities for asset processors*/ + +#include "aio.h" +#include "asset_compiler.h" + +rt_loaded_asset LoadAsset(rt_file_id file) +{ + const char *path = rtGetFilePath(file); + size_t file_size = rtGetFileSize(path); + + void *buffer = rtAllocBuffer(file_size); + rtLog("AC", "Buffer ptr %llx", (uintptr_t)buffer); + if (!buffer) { + rtLog("AC", "Failed to allocate buffer for loading %s.", path); + return (rt_loaded_asset){.buffer = NULL, .size = 0}; + } + + rt_aio_state load_state = rtSubmitSingleLoadSync((rt_file_load){ + .file = file, + .offset = 0, + .num_bytes = file_size, + .dest = buffer, + }); + if (load_state != RT_AIO_STATE_FINISHED) { + rtReleaseBuffer(buffer, file_size); + rtLog("AC", "Failed to load %s.", path); + return (rt_loaded_asset){.buffer = NULL, .size = 0}; + } + return (rt_loaded_asset){.buffer = buffer, .size = file_size}; +} diff --git a/src/runtime/asset_compiler.h b/src/runtime/asset_compiler.h new file mode 100644 index 0000000..0c8eca2 --- /dev/null +++ b/src/runtime/asset_compiler.h @@ -0,0 +1,22 @@ +#ifndef RT_ASSET_COMPILER_H +#define RT_ASSET_COMPILER_H + +#ifndef RT_BUILD_ASSET_COMPILER +#error This file should only be included if RT_BUILD_ASSET_COMPILER is defined. +#endif + +#include "file_tab.h" + +enum { + RT_ASSET_PROCESSING_FAILED = RT_CUSTOM_ERROR_START, +}; + +/* Allocated from the buffer manager */ +typedef struct { + void *buffer; + size_t size; +} rt_loaded_asset; + +rt_loaded_asset LoadAsset(rt_file_id file); + +#endif \ No newline at end of file diff --git a/src/runtime/asset_dependencies.c b/src/runtime/asset_dependencies.c deleted file mode 100644 index 8307acb..0000000 --- a/src/runtime/asset_dependencies.c +++ /dev/null @@ -1,137 +0,0 @@ -#define RT_DEFINE_DEPENDENCY_FILE_STRUCTURES -#include "asset_dependencies.h" - -#include "aio.h" -#include "buffer_manager.h" - -#include -#include -#include -#include - -typedef struct { - uint32_t begin; - uint32_t count; -} rt_dep_list; - -typedef struct { - rt_uid *uids; - rt_dep_list *lists; - uint32_t capacity; -} rt_dep_map; - -static rt_dep_map _map; -static rt_uid *_list_mem; - -rt_result LoadAssetDependencies(void) { - rt_dependency_file_header header; - rt_file_id fid = rtAddFile("data/deps.bin"); - - if (rtSubmitSingleLoadSync((rt_file_load){.dest = &header, - .num_bytes = sizeof(header), - .offset = 0, - .file = fid}) != RT_AIO_STATE_FINISHED) { - rtReportError("core", "Failed to load deps.bin"); - return RT_UNKNOWN_ERROR; - } - void *buffer = rtAllocBuffer(header.data_size); - if (rtSubmitSingleLoadSync((rt_file_load){.dest = buffer, - .num_bytes = header.data_size, - .offset = sizeof(header), - .file = fid}) != RT_AIO_STATE_FINISHED) { - rtReportError("core", "Failed to load deps.bin"); - return RT_UNKNOWN_ERROR; - } - - /* We know the exact number of list entries */ - uint64_t total_list_entries = - (header.data_size - header.num_lists * sizeof(rt_dependency_file_list_header)) / - sizeof(rt_uid); - _list_mem = malloc(total_list_entries * sizeof(rt_uid)); - if (!_list_mem) { - rtReleaseBuffer(buffer, header.data_size); - rtReportError("core", "Failed to allocate dependency list storage."); - return RT_UNKNOWN_ERROR; - } - - _map.capacity = rtNextPowerOfTwo32(header.num_lists); - _map.uids = calloc(_map.capacity, sizeof(rt_uid)); - if (!_map.uids) { - free(_list_mem); - rtReleaseBuffer(buffer, header.data_size); - rtReportError("core", "Failed to allocate dependency list storage."); - return RT_UNKNOWN_ERROR; - } - - _map.lists = calloc(_map.capacity, sizeof(rt_dep_list)); - if (!_map.uids) { - free(_list_mem); - free(_map.uids); - rtReleaseBuffer(buffer, header.data_size); - rtReportError("core", "Failed to allocate dependency list storage."); - return RT_UNKNOWN_ERROR; - } - - uint32_t storage_at = 0; - - rt_dependency_file_list_header *list = buffer; - for (uint32_t i = 0; i < header.num_lists; ++i) { - const rt_uid *entries = (rt_uid *)(list + 1); - - /* Validate the checksum */ - XXH64_hash_t file_hash = XXH64_hashFromCanonical(&list->checksum); - XXH64_hash_t calc_hash = XXH3_64bits(entries, sizeof(rt_uid) * list->num_entries); - if (file_hash != calc_hash) { - rtReportError("core", "Checksum mismatch in list %u", i); - rtReleaseBuffer(buffer, header.data_size); - return RT_UNKNOWN_ERROR; - } - - /* Store the list */ - memcpy(_list_mem + storage_at, entries, sizeof(rt_uid) * list->num_entries); - bool inserted = false; - for (uint32_t j = 0; j < _map.capacity; ++j) { - uint32_t slot = (list->uid + j) % _map.capacity; - if (_map.uids[slot] == RT_INVALID_UID) { - _map.uids[slot] = list->uid; - _map.lists[slot].begin = storage_at; - _map.lists[slot].count = list->num_entries; - inserted = true; - break; - } - } - storage_at += list->num_entries; - assert(inserted); - assert(storage_at <= total_list_entries); - - list = (rt_dependency_file_list_header *)(entries + list->num_entries); - } - - rtReleaseBuffer(buffer, header.data_size); - - return RT_SUCCESS; -} - -void ReleaseAssetDependencies(void) { - free(_list_mem); - free(_map.uids); - free(_map.lists); -} - -RT_DLLEXPORT rt_asset_dependency_list rtGetAssetDependencies(rt_uid asset) { - rt_asset_dependency_list result = { - .dependencies = NULL, - .count = 0, - }; - for (uint32_t i = 0; i < _map.capacity; ++i) { - uint32_t slot = (asset + i) % _map.capacity; - if (_map.uids[slot] == asset) { - result.dependencies = &_list_mem[_map.lists[slot].begin]; - result.count = _map.lists[slot].count; - break; - } else if (_map.uids[slot] == RT_INVALID_UID) { - break; - } - } - return result; -} \ No newline at end of file diff --git a/src/runtime/asset_dependencies.h b/src/runtime/asset_dependencies.h deleted file mode 100644 index 0e531dd..0000000 --- a/src/runtime/asset_dependencies.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef RT_ASSET_DEPENDENCIES_H -#define RT_ASSET_DEPENDENCIES_H - -#include "assets.h" - -#ifdef RT_DEFINE_DEPENDENCY_FILE_STRUCTURES - -#include "xxhash/xxhash.h" - -#pragma pack(push, 1) -typedef struct { - uint64_t data_size; - uint32_t num_lists; -} rt_dependency_file_header; - -typedef struct { - rt_uid uid; - uint32_t num_entries; - XXH64_canonical_t checksum; -} rt_dependency_file_list_header; - -#pragma pack(pop) - -#endif - -typedef struct { - const rt_uid *dependencies; - uint32_t count; -} rt_asset_dependency_list; - -RT_DLLEXPORT rt_asset_dependency_list rtGetAssetDependencies(rt_uid asset); - -#endif diff --git a/src/runtime/asset_loading.c b/src/runtime/asset_loading.c deleted file mode 100644 index 293ceb6..0000000 --- a/src/runtime/asset_loading.c +++ /dev/null @@ -1,74 +0,0 @@ -#include "assets.h" -#include "uidtab.h" -#include "aio.h" -#include "buffer_manager.h" - -#define RT_DEFINE_PACKAGE_FILE_STRUCTURES -#include "packages.h" - -#include "lz4/lz4.h" - -rt_result DecompressAsset(void *compressed_buffer, - size_t compressed_buffer_size, - void **p_decompressed, - size_t *p_decompressed_size) { - - const rt_package_asset_header *header = compressed_buffer; - - size_t compressed_size = (compressed_buffer_size) - sizeof(*header); - XXH64_hash_t calculated_hash = XXH3_64bits((header + 1), compressed_size); - XXH64_hash_t file_hash = XXH64_hashFromCanonical(&header->checksum); - if (calculated_hash != file_hash) { - rtLog("core", "Checksum mismatch for asset"); - return RT_LOAD_FAILED; - } - - size_t size = (size_t)header->decompressed_size; - void *decompressed_buffer = rtAllocBuffer(size); - if (!decompressed_buffer) { - return RT_BUFFER_ALLOC_FAILED; - } - - if (LZ4_decompress_safe((const char *)(header + 1), - (char *)decompressed_buffer, - (int)compressed_size, - (int)size) < 0) { - return RT_UNKNOWN_ERROR; - } - - *p_decompressed = decompressed_buffer; - *p_decompressed_size = size; - return RT_SUCCESS; -} - - -RT_DLLEXPORT rt_result rtLoadAssetDirect(rt_uid uid, void **p_buffer, size_t *p_size) { - const rt_uid_data *data = rtGetUIDData(uid); - if (!data) - return RT_UNKNOWN_ASSET; - - void *compressed_buffer = rtAllocBuffer(data->size); - if (!compressed_buffer) { - return RT_BUFFER_ALLOC_FAILED; - } - - if (rtSubmitSingleLoadSync((rt_file_load) { - .file = data->pkg_file, - .offset = data->offset, - .num_bytes = data->size, - .dest = compressed_buffer, - }) != RT_AIO_STATE_FINISHED) { - rtReleaseBuffer(compressed_buffer, data->size); - return RT_LOAD_FAILED; - } - - void *decompressed_buffer; - size_t decompressed_size; - rt_result res = DecompressAsset(compressed_buffer, data->size, &decompressed_buffer, &decompressed_size); - - rtReleaseBuffer(compressed_buffer, data->size); - *p_buffer = decompressed_buffer; - *p_size = decompressed_size; - - return res; -} \ No newline at end of file diff --git a/src/runtime/asset_manager.c b/src/runtime/asset_manager.c new file mode 100644 index 0000000..e69de29 diff --git a/src/runtime/assets.h b/src/runtime/assets.h index 8466768..e11d5c7 100644 --- a/src/runtime/assets.h +++ b/src/runtime/assets.h @@ -1,40 +1,6 @@ #ifndef RT_ASSETS_H #define RT_ASSETS_H -#include +/* Asset system interface */ -#include "runtime.h" - -/* Unique identifier for an asset. */ -typedef uint32_t rt_uid; - -#define RT_INVALID_UID 0 - -/* Used to identify renderer backend dependent assets. */ -enum { - RT_INVALID_RENDERER_BACKEND_CODE = 0, - RT_RENDERER_BACKEND_CODE_VK = 1, - - RT_RENDERER_BACKEND_CODE_ONE_PAST_LAST, -}; -typedef uint8_t rt_renderer_backend_code; - -enum { - RT_UNKNOWN_ASSET = RT_CUSTOM_ERROR_START, - RT_BUFFER_ALLOC_FAILED, - RT_LOAD_FAILED, - RT_ASSET_CACHE_FULL, -}; - -/* Load an asset without using the cache */ -RT_DLLEXPORT rt_result rtLoadAssetDirect(rt_uid uid, void **buffer, size_t *size); - -typedef struct { - void *data; - size_t size; - rt_result result; -} rt_get_asset_result; - -RT_DLLEXPORT rt_get_asset_result rtGetAsset(rt_uid uid); - -#endif \ No newline at end of file +#endif diff --git a/src/runtime/buffer_manager.c b/src/runtime/buffer_manager.c index 510eb18..5980f77 100644 --- a/src/runtime/buffer_manager.c +++ b/src/runtime/buffer_manager.c @@ -7,7 +7,9 @@ #include #include #include +#include +#if 0 typedef struct rt_buffer_region_s { void *memory; int16_t *refcounts; // One per block @@ -15,6 +17,7 @@ typedef struct rt_buffer_region_s { size_t block_count; rt_mutex *guard; } rt_buffer_region; +#endif /* Count leading zeroes. * Note that the return value of __builtin_clz(0) is undefined. */ @@ -31,7 +34,7 @@ static __forceinline uint32_t tzcnt32(uint32_t x) { return (uint32_t)i; } -static inline bool IsLZCNTSupported(void) { +static __forceinline bool IsLZCNTSupported(void) { #define Type 0x80000001 int info[4]; __cpuid(info, Type); @@ -48,6 +51,7 @@ static inline bool IsLZCNTSupported(void) { #endif +#if 0 /* NOTE(Kevin): Keep these sorted! */ static size_t _block_sizes[] = {RT_KB(512), RT_MB(1), RT_MB(4), RT_MB(8)}; #define NUM_BLOCK_SIZES (sizeof(_block_sizes) / sizeof(_block_sizes[0])) @@ -257,4 +261,156 @@ RT_DLLEXPORT void rtIncreaseBufferRefCount(const void *begin, size_t size) { } } rtLog("BUFFERMGR", "Tried to increase the refcount of an invalid buffer"); +} +#endif + +#define BLOCK_SIZE 4096u + +static uint32_t *_refcounts; +static uint32_t *_bitmap; +static char *_memory; +static rt_mutex *_guard; + +static size_t _block_count; + +RT_CVAR_I(rt_BufferMemoryBudget, + "The amount of memory to allocate for the buffer manager. Default: 512MB", + RT_MB(512)); + +extern rt_result InitBufferManager(void) { + _guard = rtCreateMutex(); + if (!_guard) { + rtReportError("BUFFERMGR", "Failed to create the buffer manager mutex."); + return RT_UNKNOWN_ERROR; + } + if (!IsLZCNTSupported()) { + rtReportError("BUFFERMGR", "The required lzcnt intrinisc is not supported."); + return RT_UNKNOWN_ERROR; + } + + size_t budget = (size_t)rt_BufferMemoryBudget.i; + size_t block_count = budget / BLOCK_SIZE; + if ((budget % block_count) != 0) { + rtLog("BUFFERMGR", + "The configured buffer memory budget %zu is not dividable by the block size (4KB).", + budget); + } + size_t dword_count = (block_count + 31) / 32; + _block_count = block_count; + + _memory = malloc(budget + dword_count * sizeof(uint32_t) + block_count * sizeof(uint32_t)); + if (!_memory) { + return RT_OUT_OF_MEMORY; + } + _bitmap = (uint32_t*)(_memory + budget); + memset(_bitmap, 0, sizeof(uint32_t) * dword_count); + _refcounts = _bitmap + dword_count; + memset(_refcounts, 0, sizeof(uint32_t) * block_count); + + return RT_SUCCESS; +} + +extern void ShutdownBufferManager(void) { + rtDestroyMutex(_guard); +} + +/* Public API */ + +RT_DLLEXPORT void *rtAllocBuffer(size_t size) { + size_t alloc_blocks = (size + BLOCK_SIZE - 1) / BLOCK_SIZE; + size_t dword_count = (_block_count + 31) / 32; + void *result = NULL; + size_t first_block = 0; + rtLockMutex(_guard); + for (size_t i = 0; i < _block_count; ++i) { + size_t dword = i / 32; + if (_bitmap[dword] == 0 || tzcnt32(_bitmap[dword]) >= alloc_blocks) { + size_t mask = (1ull << alloc_blocks) - 1; + _bitmap[dword] |= mask; + result = _memory + i * BLOCK_SIZE; + first_block = i; + } + else if (lzcnt32(_bitmap[dword]) >= alloc_blocks) { + size_t first = (_bitmap[dword] != 0) ? 32 - lzcnt32(_bitmap[dword]) : 0; + size_t mask = ((1ull << alloc_blocks) - 1) << first; + _bitmap[dword] |= mask; + result = _memory + (i + first) * BLOCK_SIZE; + first_block = i + first; + break; + } else if (_bitmap[dword] != UINT32_MAX) { + size_t first = 32 - lzcnt32(_bitmap[dword]); + size_t leftover = alloc_blocks - lzcnt32(_bitmap[dword]); + if (dword == dword_count - 1) { + break; // Reached the end + } + if (leftover < 32) { + size_t next_dword_free = _bitmap[dword + 1] != 0 ? tzcnt32(_bitmap[dword + 1]) : 32; + if (next_dword_free < leftover) + continue; + _bitmap[dword] = UINT32_MAX; + size_t mask = (1ull << leftover) - 1; + _bitmap[dword + 1] |= mask; + result = _memory + (i + first) * BLOCK_SIZE; + first_block = i + first; + break; + } else { + // Check each bit separately + bool free = true; + for (size_t j = i + first; j < i + first + alloc_blocks; ++j) { + size_t dwordj = j / 32; + size_t bitj = j % 32; + if ((_bitmap[dwordj] & (1u << bitj)) != 0) { + free = false; + break; + } + } + if (free) { + for (size_t j = i + first; j < i + first + alloc_blocks; ++j) { + size_t dwordj = j / 32; + size_t bitj = j % 32; + _bitmap[dwordj] |= (1u << bitj); + } + result = _memory + (i + first) * BLOCK_SIZE; + first_block = i + first; + } + } + } else { + /* These 32 blocks are all allocated. Go to the next dword */ + assert((i % 32) == 0); + i += 31; + } + } + + for (size_t i = first_block; i < first_block + alloc_blocks; ++i) + _refcounts[i] = 1; + + rtUnlockMutex(_guard); + rtLog("BUFFERMGR", "Result ptr %llx", (uintptr_t)result); + return result; +} + +RT_DLLEXPORT void rtReleaseBuffer(const void *begin, size_t size) { + size_t alloc_blocks = (size + BLOCK_SIZE - 1) / BLOCK_SIZE; + uintptr_t off = (uintptr_t)begin - (uintptr_t)_memory; + uintptr_t first_block = off / BLOCK_SIZE; + rtLockMutex(_guard); + for (size_t i = first_block; i < first_block + alloc_blocks; ++i) { + if (--_refcounts[i] == 0) { + size_t dword = i / 32; + size_t bit = i % 32; + _bitmap[dword] &= ~(1u << bit); + } + } + rtUnlockMutex(_guard); +} + +RT_DLLEXPORT void rtIncreaseBufferRefCount(const void *begin, size_t size) { + size_t alloc_blocks = (size + BLOCK_SIZE - 1) / BLOCK_SIZE; + uintptr_t off = (uintptr_t)begin - (uintptr_t)_memory; + uintptr_t first_block = off / BLOCK_SIZE; + rtLockMutex(_guard); + for (size_t i = first_block; i < first_block + alloc_blocks; ++i) { + ++_refcounts[i]; + } + rtUnlockMutex(_guard); } \ No newline at end of file diff --git a/src/runtime/description_parser.c b/src/runtime/description_parser.c new file mode 100644 index 0000000..f5b01d8 --- /dev/null +++ b/src/runtime/description_parser.c @@ -0,0 +1,252 @@ +#include "description_parser.h" +#include "runtime.h" +#include "mem_arena.h" + +#include +#include + +extern int memcmp(const void *s1, const void *s2, size_t n); +extern void *memcpy(void *dest, const void *src, size_t n); + +static bool IsAllowedChar(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c == '.') || (c == '_') || (c == '/'); +} + +static bool IsWhitespace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + +static void SkipWhitespace(rt_parse_state *state) { + while (state->at < state->length && IsWhitespace(state->text[state->at])) { + if (state->text[state->at] == '\n') + ++state->line; + ++state->at; + } +} + +static bool ParseAttribute(rt_parse_state *state, rt_text_span *_name) { + rt_text_span name; + name.start = &state->text[state->at]; + name.length = 0; + while (state->at < state->length && !IsWhitespace(state->text[state->at])) { + if (IsAllowedChar(state->text[state->at])) { + ++state->at; + ++name.length; + } else { + rtReportError("AC", + "%s:%d Unexpected character %c", + state->file, + state->line, + state->text[state->at]); + return false; + } + } + *_name = name; + return true; +} + +static bool ParseValue(rt_parse_state *state, rt_text_span *_value) { + rt_text_span value; + value.start = &state->text[state->at]; + value.length = 0; + while (state->at < state->length && !IsWhitespace(state->text[state->at]) && + state->text[state->at] != ';') { + if (IsAllowedChar(state->text[state->at])) { + ++state->at; + ++value.length; + } else { + rtReportError("AC", + "%s:%d Unexpected character %c", + state->file, + state->line, + state->text[state->at]); + return false; + } + } + *_value = value; + return true; +} + +#define BLOCK_BEGIN "BEGIN" +#define BLOCK_BEGIN_LENGTH 5 +#define BLOCK_END "END" +#define BLOCK_END_LENGTH 3 + +RT_INLINE static bool IsBlockBegin(rt_parse_state *state) { + return (state->length - state->at >= BLOCK_BEGIN_LENGTH) && + (memcmp(&state->text[state->at], BLOCK_BEGIN, BLOCK_BEGIN_LENGTH) == 0); +} + +RT_INLINE static bool IsBlockEnd(rt_parse_state *state) { + return (state->length - state->at >= BLOCK_END_LENGTH) && + (memcmp(&state->text[state->at], BLOCK_END, BLOCK_END_LENGTH) == 0); +} + +static bool ParseBlock(rt_parse_state *state, rt_text_span *p_value) { + rt_text_span value; + value.start = &state->text[state->at]; + value.length = 0; + while (state->at < state->length) { + if (state->text[state->at] == BLOCK_END[0] && IsBlockEnd(state)) { + *p_value = value; + return true; + } + ++value.length; + ++state->at; + } + /* did not find END */ + return false; +} + +static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index, rt_arena *arena); + +static bool ParseStmt(rt_parse_state *state, unsigned int *stmt_index, rt_arena *arena) { + rt_parsed_stmt stmt; + stmt.next = UINT_MAX; /* end of list */ + + SkipWhitespace(state); + if (!ParseAttribute(state, &stmt.attribute)) + return false; + SkipWhitespace(state); + + if (state->at == state->length) { + rtReportError("AC", "%s:%d Expected either a value or '{'", state->file, state->line); + return false; + } + + if (state->text[state->at] == '{') { + /* Consume '{' */ + ++state->at; + + stmt.form = RT_STMT_FORM_LIST; + if (!ParseStmtList(state, &stmt.list_index, arena)) + return false; + + /* Consume '}' */ + if (state->at < state->length && state->text[state->at] == '}') + ++state->at; + } else if (IsBlockBegin(state)) { + /* Consume BEGIN */ + state->at += BLOCK_BEGIN_LENGTH; + + stmt.form = RT_STMT_FORM_BLOCK; + if (!ParseBlock(state, &stmt.block)) + return false; + + /* Consume END */ + state->at += BLOCK_END_LENGTH; + } else { + stmt.form = RT_STMT_FORM_VALUE; + if (!ParseValue(state, &stmt.value)) + return false; + + /* Consume ';' */ + if (state->at < state->length && state->text[state->at] == ';') + ++state->at; + } + + SkipWhitespace(state); + + /* Add statement to array */ + if (state->statement_count == state->statement_capacity) { + unsigned int cap = (state->statement_capacity > 0) ? state->statement_capacity * 2 : 64; + rt_parsed_stmt *temp = rtArenaPush(arena, sizeof(rt_parsed_stmt) * cap); + if (!temp) { + rtReportError("AC", "While parsing %s: Out of memory\n", state->file); + return false; + } + if (state->statement_capacity > 0) + memcpy(temp, state->statements, sizeof(rt_parsed_stmt) * state->statement_capacity); + state->statements = temp; + state->statement_capacity = cap; + } + state->statements[state->statement_count] = stmt; + *stmt_index = state->statement_count++; + + return true; +} + +static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index, rt_arena *arena) { + rt_parsed_stmt_list list; + list.first = UINT_MAX; + list.count = 0; + + unsigned int last = UINT_MAX; + + while (state->at < state->length && state->text[state->at] != '}') { + unsigned int stmt; + if (!ParseStmt(state, &stmt, arena)) + return false; + + if (last != UINT_MAX) + state->statements[last].next = stmt; + else + list.first = stmt; + last = stmt; + + ++list.count; + } + + /* Add list to array */ + if (state->statement_list_count == state->statement_list_capacity) { + unsigned int cap = + (state->statement_list_capacity > 0) ? state->statement_list_capacity * 2 : 64; + rt_parsed_stmt_list *temp = rtArenaPush(arena, sizeof(rt_parsed_stmt_list) * cap); + if (!temp) { + rtReportError("AC", "While parsing %s: Out of memory\n", state->file); + return false; + } + if (state->statement_list_capacity > 0) + memcpy(temp, + state->statement_lists, + sizeof(rt_parsed_stmt_list) * state->statement_list_capacity); + state->statement_lists = temp; + state->statement_list_capacity = cap; + } + state->statement_lists[state->statement_list_count] = list; + *list_index = state->statement_list_count++; + return true; +} + +const rt_parsed_stmt * rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute) { + if (list_index >= state->statement_list_count) + return NULL; + const rt_parsed_stmt_list *list = &state->statement_lists[list_index]; + unsigned int stmt_index = list->first; + for (unsigned int i = 0; i < list->count; ++i) { + const rt_parsed_stmt *stmt = &state->statements[stmt_index]; + if (rtCompareSpanToString(stmt->attribute, attribute) == 0) + return stmt; + stmt_index = stmt->next; + } + return NULL; +} + + +rt_result rtParseDescription(const char *text, + size_t length, + const char *file_path, + unsigned int *_root_list, + rt_parse_state *_state, + struct rt_arena_s *arena) { + + rt_parse_state state = {.text = text, + .at = 0, + .length = length, + .line = 1, + .file = file_path, + .statements = NULL, + .statement_lists = NULL, + .statement_capacity = 0, + .statement_list_capacity = 0}; + + unsigned int root_list = 0; + if (!ParseStmtList(&state, &root_list, arena)) { + return 1; + } + *_root_list = root_list; + *_state = state; + + return RT_SUCCESS; +} diff --git a/src/runtime/description_parser.h b/src/runtime/description_parser.h new file mode 100644 index 0000000..7ee7127 --- /dev/null +++ b/src/runtime/description_parser.h @@ -0,0 +1,56 @@ +#ifndef RT_ASSETC_DESCRIPTION_PARSER_H +#define RT_ASSETC_DESCRIPTION_PARSER_H + +#include "runtime/runtime.h" + +struct rt_arena_s; + +typedef enum { + RT_STMT_FORM_VALUE, + RT_STMT_FORM_LIST, + RT_STMT_FORM_BLOCK, +} rt_stmt_form; + +typedef struct { + unsigned int first; + unsigned int count; +} rt_parsed_stmt_list; + +typedef struct { + rt_stmt_form form; + rt_text_span attribute; + union { + rt_text_span value; + rt_text_span block; + unsigned int list_index; + }; + /* For lists */ + unsigned int next; +} rt_parsed_stmt; + +typedef struct { + const char *file; + const char *text; + size_t at; + size_t length; + int line; + + rt_parsed_stmt *statements; + unsigned int statement_count; + unsigned int statement_capacity; + + rt_parsed_stmt_list *statement_lists; + unsigned int statement_list_count; + unsigned int statement_list_capacity; +} rt_parse_state; + +rt_result rtParseDescription(const char *text, + size_t length, + const char *file_path, + unsigned int *root_list, + rt_parse_state *state, + struct rt_arena_s *arena); + +const rt_parsed_stmt *rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute); + +#endif diff --git a/src/runtime/error_report.c b/src/runtime/error_report.c index 28d7ec9..85e17eb 100644 --- a/src/runtime/error_report.c +++ b/src/runtime/error_report.c @@ -1,5 +1,4 @@ #include -#include #include #include @@ -14,11 +13,22 @@ #define DbgBreak __builtin_trap() #else /* Not available */ -#define DbgBreak +#define DbgBreak #endif /* TODO(Kevin): Log to file, show error message box, ... */ +void Win32ErrorToString(DWORD last_error, char *out, int bufsize) { + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | + FORMAT_MESSAGE_ARGUMENT_ARRAY | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + last_error, + 0, + out, + bufsize, + NULL); +} + static bool DisplayErrorBox(const char *text) { #ifdef _WIN32 WCHAR msg[256]; diff --git a/src/runtime/file_tab.c b/src/runtime/file_tab.c index 5d55d11..55deaa3 100644 --- a/src/runtime/file_tab.c +++ b/src/runtime/file_tab.c @@ -1,8 +1,7 @@ #include "file_tab.h" #include "threading.h" #include "config.h" - -#include +#include "hashing.h" #define NAME_CAP(cap) ((cap)*128) typedef struct { @@ -53,8 +52,7 @@ RT_DLLEXPORT rt_file_id rtGetFileId(const char *path) { RT_DLLEXPORT rt_file_id rtGetFileIdFromSpan(rt_text_span path) { /* Randomly choosen, aka finger smash keyboard */ - XXH64_hash_t seed = 15340978; - rt_file_id fid = (rt_file_id)XXH3_64bits_withSeed(path.start, path.length, seed); + rt_file_id fid = (rt_file_id)rtHashBytes(path.start, path.length); if (fid == 0) fid = ~fid; return fid; diff --git a/src/runtime/fsutils.c b/src/runtime/fsutils.c new file mode 100644 index 0000000..2488122 --- /dev/null +++ b/src/runtime/fsutils.c @@ -0,0 +1,115 @@ +#include "fsutils.h" +#include "runtime.h" +#include "threading.h" + +#ifdef _WIN32 + +#define WIN32_LEAN_AND_MEAN +#include + +struct rt_scandir_handle_s { + HANDLE handle; + rt_dirent next; +}; + +static rt_scandir_handle _dirs[256]; +static unsigned int _next = 0; +static rt_mutex *_guard = NULL; + +RT_DLLEXPORT rt_scandir_handle *rtScanDirectory(const char *path) { + char wildcard_path[MAX_PATH]; + strncpy(wildcard_path, path, MAX_PATH); + strncat(wildcard_path, "\\*", MAX_PATH - strlen(path)); + WCHAR wpath[MAX_PATH]; + if (rtUTF8ToWStr(wildcard_path, wpath, MAX_PACKAGE_NAME) != RT_SUCCESS) + return NULL; + WIN32_FIND_DATAW data; + HANDLE h = FindFirstFileW(wpath, &data); + if (h == INVALID_HANDLE_VALUE) + return NULL; + + if (!_guard) { + _guard = rtCreateMutex(); + if (!_guard) { + FindClose(h); + return NULL; + } + } + + rtLockMutex(_guard); + rt_scandir_handle *dir = &_dirs[_next]; + _next = (_next + 1) % RT_ARRAY_COUNT(_dirs); + rtUnlockMutex(_guard); + + if (dir->handle != NULL) { + rtLog("core", "Failed to acquire a scandir handle."); + FindClose(h); + return NULL; + } + + dir->handle = h; + dir->next.is_last = false; + rtWStrToUTF8(data.cFileName, dir->next.name, 260); + if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + dir->next.type = RT_DIRENT_TYPE_DIRECTORY; + else + dir->next.type = RT_DIRENT_TYPE_FILE; + + return dir; +} + +RT_DLLEXPORT rt_dirent rtNextDirectoryEntry(rt_scandir_handle *dir) { + rt_dirent current; + memcpy(¤t, &dir->next, sizeof(rt_dirent)); + + WIN32_FIND_DATAW data; + if (!FindNextFileW(dir->handle, &data)) { + current.is_last = true; + return current; + } + rtWStrToUTF8(data.cFileName, dir->next.name, 260); + if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + dir->next.type = RT_DIRENT_TYPE_DIRECTORY; + else + dir->next.type = RT_DIRENT_TYPE_FILE; + + + return current; +} + +RT_DLLEXPORT void rtCloseDirectory(rt_scandir_handle *dir) { + if (!dir) + return; + FindClose(dir->handle); + dir->handle = NULL; +} + +RT_DLLEXPORT size_t rtGetFileSize(const char *path) { + WCHAR wpath[MAX_PATH]; + MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH); + WIN32_FILE_ATTRIBUTE_DATA attribs; + if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs)) + return 0; + return (size_t)attribs.nFileSizeHigh << 32 | (size_t)attribs.nFileSizeLow; +} + +uint64_t rtGetCurrentTimestamp(void) { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + uint64_t ts = ft.dwLowDateTime; + ts |= (uint64_t)ft.dwHighDateTime << 32; + return ts; +} + +uint64_t rtGetFileModificationTimestamp(const char *path) { + WCHAR wpath[MAX_PATH]; + MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH); + WIN32_FILE_ATTRIBUTE_DATA attribs; + if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs)) + return 0; + uint64_t ts = attribs.ftLastWriteTime.dwLowDateTime; + ts |= (uint64_t)attribs.ftLastWriteTime.dwHighDateTime << 32; + return ts; +} + +#endif \ No newline at end of file diff --git a/src/runtime/fsutils.h b/src/runtime/fsutils.h new file mode 100644 index 0000000..6c6bcf9 --- /dev/null +++ b/src/runtime/fsutils.h @@ -0,0 +1,35 @@ +#ifndef RT_FSUITLS_H +#define RT_FSUTILS_H + +/* File system utilities */ +#include "runtime.h" +#include + +typedef enum { + RT_DIRENT_TYPE_FILE, + RT_DIRENT_TYPE_DIRECTORY, + RT_DIRENT_TYPE_SYMLINK, +} rt_dirent_type; + +typedef struct { + char name[260]; + rt_dirent_type type; + bool is_last; +} rt_dirent; + +typedef struct rt_scandir_handle_s rt_scandir_handle; + +RT_DLLEXPORT rt_scandir_handle *rtScanDirectory(const char *path); + +RT_DLLEXPORT rt_dirent rtNextDirectoryEntry(rt_scandir_handle *dir); + +RT_DLLEXPORT void rtCloseDirectory(rt_scandir_handle *dir); + +RT_DLLEXPORT size_t rtGetFileSize(const char *path); + +RT_DLLEXPORT uint64_t rtGetFileModificationTimestamp(const char *path); + +/* Does not really fit here, but it is mostly useful for comparison with file timestamps. */ +RT_DLLEXPORT uint64_t rtGetCurrentTimestamp(void); + +#endif diff --git a/src/runtime/hashing.c b/src/runtime/hashing.c new file mode 100644 index 0000000..cd8ad37 --- /dev/null +++ b/src/runtime/hashing.c @@ -0,0 +1,10 @@ +#include "hashing.h" + +#include +#include + +static_assert(sizeof(rt_hash64) == sizeof(XXH64_hash_t), "Size mismatch between rt_hash64 and XXH64_hash_t!"); + +RT_DLLEXPORT rt_hash64 rtHashBytes(const void *begin, size_t count) { + return XXH3_64bits(begin, count); +} \ No newline at end of file diff --git a/src/runtime/hashing.h b/src/runtime/hashing.h new file mode 100644 index 0000000..708a834 --- /dev/null +++ b/src/runtime/hashing.h @@ -0,0 +1,12 @@ +#ifndef RT_HASHING_H +#define RT_HASHING_H + +/* Hash functions */ + +#include "runtime.h" + +typedef uint64_t rt_hash64; + +RT_DLLEXPORT rt_hash64 rtHashBytes(const void *begin, size_t count); + +#endif diff --git a/src/runtime/init.c b/src/runtime/init.c index 5572685..2dd7b89 100644 --- a/src/runtime/init.c +++ b/src/runtime/init.c @@ -3,26 +3,30 @@ #include "aio.h" #include "file_tab.h" #include "buffer_manager.h" -#include "uidtab.h" extern rt_cvar rt_Renderer; extern rt_cvar rt_Fullscreen; extern rt_cvar rt_WindowWidth; extern rt_cvar rt_WindowHeight; -extern rt_cvar rt_BufferManagerMemory; +extern rt_cvar rt_BufferMemoryBudget; extern rt_cvar rt_FileTabCapacity; extern rt_cvar rt_MaxConcurrentAsyncIO; -extern rt_cvar rt_AssetCacheSize; + +#ifdef RT_BUILD_ASSET_COMPILER +extern rt_cvar rt_AssetDirectory; +#endif void RegisterRuntimeCVars(void) { rtRegisterCVAR(&rt_Renderer); rtRegisterCVAR(&rt_Fullscreen); rtRegisterCVAR(&rt_WindowWidth); rtRegisterCVAR(&rt_WindowHeight); - rtRegisterCVAR(&rt_BufferManagerMemory); + rtRegisterCVAR(&rt_BufferMemoryBudget); rtRegisterCVAR(&rt_FileTabCapacity); rtRegisterCVAR(&rt_MaxConcurrentAsyncIO); - rtRegisterCVAR(&rt_AssetCacheSize); +#ifdef RT_BUILD_ASSET_COMPILER + rtRegisterCVAR(&rt_AssetDirectory); +#endif } extern void SetMainThreadId(void); @@ -33,14 +37,11 @@ extern rt_result InitFileTab(void); extern void ShutdownFileTab(void); extern rt_result InitAIO(void); extern void ShutdownAIO(void); -extern rt_result InitAssetCache(void); -extern void ShutdownAssetCache(void); -extern rt_result LoadUIDTable(void); -extern void ReleaseUIDTable(void); -extern rt_result LoadAssetDependencies(void); -extern void ReleaseAssetDependencies(void); -extern rt_result LoadPackageNames(void); +#ifdef RT_BUILD_ASSET_COMPILER +extern rt_result InitAssetCompiler(void); +extern void ShutdownAssetCompiler(void); +#endif RT_DLLEXPORT rt_result rtInitRuntime(void) { SetMainThreadId(); @@ -62,31 +63,20 @@ RT_DLLEXPORT rt_result rtInitRuntime(void) { return res; } - if ((res = InitAssetCache()) != RT_SUCCESS) { - rtReportError("ASSETCACHE", "Init failed."); +#ifdef RT_BUILD_ASSET_COMPILER + if ((res = InitAssetCompiler()) != RT_SUCCESS) { + rtReportError("AC", "Init failed."); return res; } - - if ((res = LoadUIDTable()) != RT_SUCCESS) { - rtLog("CORE", "LoadUIDTable returned result: %u", res); - } - - if ((res = LoadAssetDependencies()) != RT_SUCCESS) { - rtReportError("ASSETDEP", "Init failed."); - return res; - } - - if ((res = LoadPackageNames()) != RT_SUCCESS) { - rtLog("CORE", "LoadPackageNames returned result: %u", res); - } +#endif return RT_SUCCESS; } RT_DLLEXPORT void rtShutdownRuntime(void) { - ReleaseAssetDependencies(); - ReleaseUIDTable(); - ShutdownAssetCache(); +#ifdef RT_BUILD_ASSET_COMPILER + ShutdownAssetCompiler(); +#endif ShutdownAIO(); ShutdownFileTab(); ShutdownBufferManager(); diff --git a/src/runtime/mem_arena.c b/src/runtime/mem_arena.c new file mode 100644 index 0000000..7fcddf4 --- /dev/null +++ b/src/runtime/mem_arena.c @@ -0,0 +1,61 @@ +#include "mem_arena.h" + +#include + +#define ALIGNMENT 0xf +#define ALIGN(n) (((n) + ALIGNMENT) & ~ALIGNMENT) + +extern void *memset(void *, int, size_t); + +RT_DLLEXPORT rt_create_arena_result rtCreateArena(void *memory, size_t size) { + rt_arena arena = {.needs_free = 0}; + if (!memory) { + size = ALIGN(size); + memory = malloc(size); + if (!memory) { + return (rt_create_arena_result){ + .ok = 0, + }; + } + arena.needs_free = 1; + } + arena.base = memory; + arena.size = size; + arena.at = 0; + return (rt_create_arena_result){ + .arena = arena, + .ok = 1, + }; +} + +RT_DLLEXPORT void *rtArenaPush(rt_arena *arena, size_t n) { + n = ALIGN(n); + if (arena->at + n > arena->size) + return NULL; + void *p = (char *)arena->base + arena->at; + arena->at += n; + return p; +} + +RT_DLLEXPORT void *rtArenaPushZero(rt_arena *arena, size_t n) { + void *p = rtArenaPush(arena, n); + if (p) + memset(p, 0, ALIGN(n)); + return p; +} + +RT_DLLEXPORT void rtArenaPop(rt_arena *arena, size_t n) { + n = ALIGN(n); + if (arena->at < n) + return; + arena->at -= n; +} + +RT_DLLEXPORT void rtReleaseArena(rt_arena *arena) { + if (arena->needs_free) + free(arena->base); + arena->base = NULL; + arena->at = 0; + arena->size = 0; + arena->needs_free = 0; +} diff --git a/src/runtime/mem_arena.h b/src/runtime/mem_arena.h new file mode 100644 index 0000000..902381b --- /dev/null +++ b/src/runtime/mem_arena.h @@ -0,0 +1,57 @@ +#ifndef RT_MEM_ARENA_H +#define RT_MEM_ARENA_H + +#include "runtime.h" + +typedef struct rt_arena_s { + void *base; + size_t at; + size_t size; + int needs_free; +} rt_arena; + +typedef struct { + int ok; + rt_arena arena; +} rt_create_arena_result; + +typedef struct { + size_t at; +} rt_arena_rewindpoint; + +RT_DLLEXPORT rt_create_arena_result rtCreateArena(void *memory, size_t size); + +RT_DLLEXPORT void rtReleaseArena(rt_arena *arena); + +RT_DLLEXPORT void *rtArenaPush(rt_arena *arena, size_t n); + +RT_DLLEXPORT void *rtArenaPushZero(rt_arena *arena, size_t n); + +RT_DLLEXPORT void rtArenaPop(rt_arena *arena, size_t n); + +RT_INLINE void rtArenaClear(rt_arena *arena) { + arena->at = 0; +} + +RT_INLINE rt_arena_rewindpoint rtGetArenaRewindPoint(rt_arena *arena) { + return (rt_arena_rewindpoint){.at = arena->at}; +} + +RT_INLINE void rtArenaRewind(rt_arena *arena, rt_arena_rewindpoint rewind) { +#ifndef NDEBUG + if (rewind.at > arena->at) + rtReportError("CORE", + "Tried to rewind an arena to a point that lies beyond the last allocation."); +#endif + arena->at = rewind.at; +} + +/* Helper macros */ +#define RT_ARENA_PUSH_STRUCT(_Arena, _Type) rtArenaPush((_Arena), sizeof(_Type)) +#define RT_ARENA_PUSH_STRUCT_ZERO(_Arena, _Type) rtArenaPushZero((_Arena), sizeof(_Type)) +#define RT_ARENA_POP_STRUCT(_Arena, _Type) rtArenaPop((_Arena), sizeof(_Type)) +#define RT_ARENA_PUSH_ARRAY(_Arena, _Type, _N) rtArenaPush((_Arena), sizeof(_Type) * (_N)) +#define RT_ARENA_PUSH_ARRAY_ZERO(_Arena, _Type, _N) rtArenaPushZero((_Arena), sizeof(_Type) * (_N)) +#define RT_ARENA_POP_ARRAY(_Arena, _Type, _N) rtArenaPop((_Arena), sizeof(_Type) * (_N) + +#endif diff --git a/src/runtime/pipeline_processor.c b/src/runtime/pipeline_processor.c new file mode 100644 index 0000000..d4e8aff --- /dev/null +++ b/src/runtime/pipeline_processor.c @@ -0,0 +1,370 @@ +#include "runtime.h" +#include "mem_arena.h" +#include "description_parser.h" +#include "buffer_manager.h" +#include "asset_compiler.h" +#include "shader_compiler.h" +#include "gfx.h" +#include "config.h" + +#include +#include +#include + +typedef struct { + rt_attribute_binding *uniform_bindings; + rt_attribute_binding *storage_bindings; + rt_attribute_binding *texture_bindings; + + rt_shader_bytecode vertex_shader; + rt_shader_bytecode fragment_shader; + rt_shader_bytecode compute_shader; + + /* TODO Fixed function settings */ + + /* Sampler settings */ + + uint16_t uniform_binding_count; + uint16_t storage_binding_count; + uint16_t texture_binding_count; +} rt_parsed_pipeline_data; + +enum { + RT_SHADER_NOT_PRESENT = RT_ASSET_PROCESSING_FAILED + 1 +}; + +extern rt_cvar rt_Renderer; + +#if 0 +static void +DbgPrintShaderFile(const rt_parse_state *state, unsigned int list_index, unsigned int indent) { + assert(list_index < state->statement_list_count); + const rt_parsed_stmt_list *list = &state->statement_lists[list_index]; + + unsigned int stmt_index = list->first; + for (unsigned int i = 0; i < list->count; ++i) { + const rt_parsed_stmt *stmt = &state->statements[stmt_index]; + for (unsigned int j = 0; j < indent; ++j) + printf(" "); + printf("%.*s: ", stmt->attribute.length, stmt->attribute.start); + if (stmt->form == RT_STMT_FORM_VALUE) { + printf("%.*s\n", stmt->value.length, stmt->value.start); + } else { + printf("{\n"); + DbgPrintShaderFile(state, stmt->list_index, indent + 2); + printf("}\n"); + } + stmt_index = stmt->next; + } + assert(stmt_index == UINT_MAX || stmt_index == 0); +} +#endif + +static bool ParseBindingIndex(rt_text_span span, unsigned int *index) { + if (span.length == 0) + return false; + int at = (int)span.length - 1; + unsigned int exp = 1; + unsigned int n = 0; + while (at >= 0) { + if (span.start[at] >= '0' && span.start[at] <= '9') { + unsigned int digit = (unsigned int)(span.start[at] - '0'); + n += digit * exp; + } else { + rtReportError("GFX", "Unexpected non-digit character in binding index"); + return false; + } + --at; + exp *= 10; + } + *index = n; + return true; +} + +static rt_attribute_value ParseBindingValue(rt_text_span span) { + if (rtCompareSpanToString(span, "MATERIAL_ALBEDO") == 0) { + return RT_ATTRIBUTE_VALUE_MATERIAL_ALBEDO; + } else if (rtCompareSpanToString(span, "MATERIAL_NORMAL") == 0) { + return RT_ATTRIBUTE_VALUE_MATERIAL_NORMAL; + } + rtReportError("GFX", "Unsupported binding value %*.s", span.length, span.start); + return RT_ATTRIBUTE_VALUE_UNDEFINED; +} + +static bool ParseBindings(rt_parse_state *state, + unsigned int root_list, + const char *name, + const char *file_path, + rt_attribute_binding **p_bindings, + uint16_t *p_binding_count) { + const rt_parsed_stmt *bindings = rtFindStatement(state, root_list, name); + if (bindings) { + if (bindings->form != RT_STMT_FORM_LIST) { + rtReportError("GFX", + "Expected list of bindings as the value of " + "\"%s\" in %s", + name, + file_path); + return false; + } + const rt_parsed_stmt_list *binding_list = &state->statement_lists[bindings->list_index]; + rt_attribute_binding *shader_bindings = + rtAllocBuffer(sizeof(rt_attribute_binding) * binding_list->count); + if (!bindings) { + rtReportError("GFX", "Out of memory"); + return false; + } + unsigned int binding_count = binding_list->count; + + unsigned int stmt_index = binding_list->first; + for (unsigned int i = 0; i < binding_list->count; ++i) { + const rt_parsed_stmt *stmt = &state->statements[stmt_index]; + if (!ParseBindingIndex(stmt->attribute, &shader_bindings[i].index)) { + rtReleaseBuffer(shader_bindings, + sizeof(rt_attribute_binding) * binding_list->count); + return false; + } + + shader_bindings[i].value = ParseBindingValue(stmt->value); + if (shader_bindings[i].value == RT_ATTRIBUTE_VALUE_UNDEFINED) { + rtReleaseBuffer(shader_bindings, + sizeof(rt_attribute_binding) * binding_list->count); + return false; + } + stmt_index = stmt->next; + } + + *p_bindings = shader_bindings; + *p_binding_count = (uint16_t)binding_count; + return true; + } else { + *p_bindings = NULL; + *p_binding_count = 0; + return true; + } +} + +static rt_result ParseShader(rt_parse_state *state, + unsigned int root_list, + const char *name, + const char *file_path, + rt_shader_type type, + rt_shader_stage stage, + rt_shader_optimization_level optimization, + rt_shader_bytecode *p_shader_bytecode, + rt_arena *arena) { + const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, name); + if (stmt) { + if (stmt->form != RT_STMT_FORM_LIST) { + rtReportError("GFX", + "Expected a list as the value of " + "\"%s\" in %s", + name, + file_path); + return RT_ASSET_PROCESSING_FAILED; + } + const rt_parsed_stmt_list *shader_list = &state->statement_lists[stmt->list_index]; + + unsigned int stmt_index = shader_list->first; + for (unsigned int i = 0; i < shader_list->count; ++i) { + const rt_parsed_stmt *shader = &state->statements[stmt_index]; + if (shader->form != RT_STMT_FORM_VALUE && shader->form != RT_STMT_FORM_BLOCK) { + rtReportError("GFX", + "Expected a simple statement or block for " + "\"%s.%*s\" in %s", + name, + (int)shader->attribute.length, + shader->attribute.start, + file_path); + return RT_ASSET_PROCESSING_FAILED; + } + rt_shader_type in_file_type = RT_SHADER_TYPE_INVALID; + if (rtCompareSpanToString(shader->attribute, "vk") == 0) { + in_file_type = RT_SHADER_TYPE_VULKAN; + } else { + rtReportError("GFX", + "Invalid renderer backend" + "\"%*s\" in %s of file %s", + (int)shader->attribute.length, + shader->attribute.start, + name, + file_path); + return RT_ASSET_PROCESSING_FAILED; + } + + if (in_file_type == type) { + if (shader->form == RT_STMT_FORM_BLOCK) { + /* Inline code */ + *p_shader_bytecode = + CompileShader(type, stage, optimization, shader->block, file_path, arena); + if (!p_shader_bytecode->bytes) + return RT_ASSET_PROCESSING_FAILED; + break; + } else if (shader->form != RT_STMT_FORM_VALUE) { + /* A filename */ + rtLog("AC", "Shader source files not implemented yet!"); + return RT_ASSET_PROCESSING_FAILED; + } + } + + stmt_index = shader->next; + } + return RT_SUCCESS; + } + return RT_SHADER_NOT_PRESENT; +} + +static rt_shader_optimization_level +ParseOptimizationLevel(rt_parse_state *state, unsigned int root_list, const char *file_path) { + const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, "optimization"); + rt_shader_optimization_level optimization_level = RT_SHADER_OPTIMIZATION_NONE; + if (stmt) { + if (stmt->form != RT_STMT_FORM_VALUE) { + rtReportError("GFX", + "Expected a simple statement for" + "\"optimization\" in %s", + file_path); + return RT_SHADER_OPTIMIZATION_NONE; + } + + if (rtCompareSpanToString(stmt->value, "speed") == 0) { + optimization_level = RT_SHADER_OPTIMIZATION_SPEED; + } else if (rtCompareSpanToString(stmt->value, "size") == 0) { + optimization_level = RT_SHADER_OPTIMIZATION_SIZE; + } else if (rtCompareSpanToString(stmt->value, "none") == 0) { + optimization_level = RT_SHADER_OPTIMIZATION_NONE; + } else { + rtReportError("GFX", + "Expected one of 'speed', 'size' and 'none' for \"optimization\" in %s", + file_path); + } + } + + return optimization_level; +} + +static rt_result ParsePipelineFile(rt_file_id fid, + const char *text, + size_t length, + rt_parsed_pipeline_data *pipeline, + rt_arena *arena) { + /* This is the grammar for pipeline files: + * ::= * + * ::= ( ( ';' ) | ( '{' '}' ) ) + * ::= [:alnum:]* + * :: = [:alnum:]* */ + const char *file_path = rtGetFilePath(fid); + rt_parse_state state; + unsigned int root_list; + rt_result result = rtParseDescription(text, length, file_path, &root_list, &state, arena); + if (result != RT_SUCCESS) { + goto out; + } + + /* We allow the pipeline file to overwrite the optimization level */ + rt_shader_optimization_level optimization = ParseOptimizationLevel(&state, root_list, file_path); + + rt_shader_type type = RT_SHADER_TYPE_INVALID; + if (strcmp(rt_Renderer.s, "vk") == 0) + type = RT_SHADER_TYPE_VULKAN; + + if (type == RT_SHADER_TYPE_INVALID) { + result = RT_ASSET_PROCESSING_FAILED; + rtLog("AC", "rt_Renderer (%s) could not be translated to a shader type.", rt_Renderer.s); + goto out; + } + + /* Process shader stages */ + if (ParseShader(&state, + root_list, + "vertex", + file_path, + type, + RT_SHADER_STAGE_VERTEX, + optimization, + &pipeline->vertex_shader, + arena) == RT_ASSET_PROCESSING_FAILED) { + result = RT_ASSET_PROCESSING_FAILED; + goto out; + } + if (ParseShader(&state, + root_list, + "fragment", + file_path, + type, + RT_SHADER_STAGE_FRAGMENT, + optimization, + &pipeline->fragment_shader, + arena) == RT_ASSET_PROCESSING_FAILED) { + result = RT_ASSET_PROCESSING_FAILED; + goto out; + } + if (ParseShader(&state, + root_list, + "compute", + file_path, + type, + RT_SHADER_STAGE_COMPUTE, + optimization, + &pipeline->compute_shader, + arena) == RT_ASSET_PROCESSING_FAILED) { + result = RT_ASSET_PROCESSING_FAILED; + goto out; + } + + /* Process bindings */ + pipeline->texture_bindings = NULL; + pipeline->texture_binding_count = 0; + pipeline->uniform_bindings = NULL; + pipeline->uniform_binding_count = 0; + pipeline->storage_bindings = NULL; + pipeline->storage_binding_count = 0; + + if (!ParseBindings(&state, + root_list, + "texture_bindings", + file_path, + &pipeline->texture_bindings, + &pipeline->texture_binding_count)) { + result = RT_ASSET_PROCESSING_FAILED; + goto out; + } + + if (!ParseBindings(&state, + root_list, + "uniform_bindings", + file_path, + &pipeline->uniform_bindings, + &pipeline->uniform_binding_count)) { + result = RT_ASSET_PROCESSING_FAILED; + goto out; + } + + if (!ParseBindings(&state, + root_list, + "storage_bindings", + file_path, + &pipeline->storage_bindings, + &pipeline->storage_binding_count)) { + result = RT_ASSET_PROCESSING_FAILED; + goto out; + } + +out: + return result; +} + +rt_result PipelineProcessor(rt_file_id file, rt_arena *arena) { + rt_loaded_asset asset = LoadAsset(file); + if (!asset.buffer) + return RT_UNKNOWN_ERROR; + + rt_parsed_pipeline_data pipeline; + rt_result result = ParsePipelineFile(file, asset.buffer, asset.size, &pipeline, arena); + if (result != RT_SUCCESS) + goto out; + +out: + rtReleaseBuffer(asset.buffer, asset.size); + return result; +} \ No newline at end of file diff --git a/src/runtime/renderer_api.h b/src/runtime/renderer_api.h index e13f9f6..9b305ef 100644 --- a/src/runtime/renderer_api.h +++ b/src/runtime/renderer_api.h @@ -27,9 +27,10 @@ struct rt_renderer_init_info_s { }; typedef struct { - rt_uid vertex_shader; + /* rt_uid vertex_shader; rt_uid fragment_shader; rt_uid compute_shader; + */ rt_relptr texture_bindings; rt_relptr uniform_bindings; diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index 680fd03..b2aa84b 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -109,7 +109,6 @@ static RT_INLINE uint32_t rtNextPowerOfTwo32(uint32_t v) { return v; } - /* Runtime init. Initializes basic systems. * You need to call this, even if you build a CLI only app. */ RT_DLLEXPORT rt_result rtInitRuntime(void); diff --git a/src/runtime/shader_compiler.c b/src/runtime/shader_compiler.c new file mode 100644 index 0000000..6455f03 --- /dev/null +++ b/src/runtime/shader_compiler.c @@ -0,0 +1,52 @@ +#include "shader_compiler.h" + +#ifdef RT_BUILD_VULKAN_SHADER_COMPILER +extern rt_shader_bytecode CompileVulkanShader(rt_shader_stage stage, + rt_shader_optimization_level optimization, + rt_text_span code, + const char *file_path, + rt_arena *arena); +#endif + +static rt_shader_bytecode CompileNullShader(rt_shader_stage stage, + rt_shader_optimization_level optimization, + rt_text_span code, + const char *file_path, + rt_arena *arena) { + RT_UNUSED(stage); + RT_UNUSED(optimization); + RT_UNUSED(code); + RT_UNUSED(arena); + rtLog("AC", "Attempted to compile shader %s with unsupported type.", file_path); + return (rt_shader_bytecode){ + .bytes = NULL, + .len = 0, + }; +} + +typedef rt_shader_bytecode +shader_compiler_fn(rt_shader_stage, rt_shader_optimization_level, rt_text_span, const char *, rt_arena *); + +static shader_compiler_fn *_compiler_funcs[RT_SHADER_TYPE_COUNT] = { + CompileNullShader, + +#ifdef RT_BUILD_VULKAN_SHADER_COMPILER + CompileVulkanShader, +#else + CompileNullShader, +#endif + +}; + +rt_shader_bytecode CompileShader(rt_shader_type type, + rt_shader_stage stage, + rt_shader_optimization_level optimization, + rt_text_span code, + const char *file_path, + rt_arena *arena) { + if (type >= RT_SHADER_TYPE_COUNT) { + rtLog("AC", "Invalid shader type %u", type); + return (rt_shader_bytecode){.bytes = NULL, .len = 0}; + } + return _compiler_funcs[type](stage, optimization, code, file_path, arena); +} \ No newline at end of file diff --git a/src/runtime/shader_compiler.h b/src/runtime/shader_compiler.h new file mode 100644 index 0000000..a20abb5 --- /dev/null +++ b/src/runtime/shader_compiler.h @@ -0,0 +1,34 @@ +#ifndef RT_SHADER_COMPILER_H +#define RT_SHADER_COMPILER_H + +#include "runtime.h" +#include "mem_arena.h" + +typedef enum +{ + RT_SHADER_TYPE_INVALID, + RT_SHADER_TYPE_VULKAN, + + RT_SHADER_TYPE_COUNT, +} rt_shader_type; + +typedef enum { + RT_SHADER_STAGE_VERTEX, + RT_SHADER_STAGE_FRAGMENT, + RT_SHADER_STAGE_COMPUTE, +} rt_shader_stage; + +typedef enum { + RT_SHADER_OPTIMIZATION_NONE, + RT_SHADER_OPTIMIZATION_SPEED, + RT_SHADER_OPTIMIZATION_SIZE, +} rt_shader_optimization_level; + +typedef struct { + void *bytes; + size_t len; +} rt_shader_bytecode; + +rt_shader_bytecode CompileShader(rt_shader_type type, rt_shader_stage stage, rt_shader_optimization_level optimization, rt_text_span code, const char *input_file, rt_arena *arena); + +#endif diff --git a/src/runtime/threading.h b/src/runtime/threading.h index 60030a2..3d9ac36 100644 --- a/src/runtime/threading.h +++ b/src/runtime/threading.h @@ -75,4 +75,6 @@ RT_DLLEXPORT rt_thread_id rtGetCurrentThreadId(void); RT_DLLEXPORT bool rtIsMainThread(void); +RT_DLLEXPORT void rtSleep(unsigned int milliseconds); + #endif diff --git a/src/runtime/threading_mutex.c b/src/runtime/threading_mutex.c index 203faaf..3e9d430 100644 --- a/src/runtime/threading_mutex.c +++ b/src/runtime/threading_mutex.c @@ -41,19 +41,23 @@ RT_DLLEXPORT rt_mutex *rtCreateMutex(void) { rt_mutex *mtx = &_mutex[_first_reusable]; _first_reusable = mtx->next_reusable; mtx->owner = 0; + ReleaseMutex(_guard); return mtx; } else if (_next < MAX_MUTEX) { rt_mutex *mtx = &_mutex[_next]; mtx->handle = CreateMutexW(NULL, FALSE, NULL); if (!mtx->handle) { rtLog("core", "Mutex creation failed: %u", GetLastError()); + ReleaseMutex(_guard); return NULL; } mtx->next_reusable = MAX_MUTEX; mtx->owner = 0; ++_next; + ReleaseMutex(_guard); return mtx; } + ReleaseMutex(_guard); rtReportError("core", "Ran out of mutex objects"); return NULL; } diff --git a/src/runtime/threading_thread.c b/src/runtime/threading_thread.c index 41a85f6..d6d5658 100644 --- a/src/runtime/threading_thread.c +++ b/src/runtime/threading_thread.c @@ -123,11 +123,18 @@ RT_DLLEXPORT bool rtIsMainThread(void) { return rtGetCurrentThreadId() == _main_thread_id; } +RT_DLLEXPORT void rtSleep(unsigned int milliseconds) { + SleepEx(milliseconds, TRUE); +} + #elif defined(__linux__) #define _GNU_SOURCE #include #include +#if _POSIX_C_SOURCE >= 199309L +#include +#endif struct rt_thread_s { pthread_t handle; @@ -208,5 +215,17 @@ RT_DLLEXPORT bool rtIsMainThread(void) { return rtGetCurrentThreadId() == _main_thread_id; } +RT_DLLEXPORT void rtSleep(unsigned int milliseconds) { +#if _POSIX_C_SOURCE >= 199309L + struct timespec ts; + ts.tv_sec = milliseconds / 1000; + ts.tv_nsec = (milliseconds % 1000) * 1000000; + nanosleep(&ts, NULL); +#else + if (milliseconds >= 1000) + sleep(milliseconds / 1000); + usleep((milliseconds % 1000) * 1000); +#endif +} #endif diff --git a/src/runtime/uidtab.c b/src/runtime/uidtab.c deleted file mode 100644 index a26d685..0000000 --- a/src/runtime/uidtab.c +++ /dev/null @@ -1,121 +0,0 @@ -#define RT_DEFINE_UIDTAB_FILE_STRUCTURES -#include "uidtab.h" -#include "aio.h" - -#include "xxhash/xxhash.h" - -#include -#include -#include -#include - -typedef struct { - rt_uid *uids; - rt_uid_data *data; - uint32_t slots; -} rt_uidtab; - -static rt_uidtab _tab; - -rt_result LoadUIDTable(void) { - /* We use stdio here, because we cannot load any asset in parallel to this. - * This is because the uidtab is what tells us which assets exist. - */ - FILE *f = fopen("data/uidtab.bin", "rb"); - if (!f) - return RT_LOAD_FAILED; - - rt_uidtab_header header; - if (fread(&header, sizeof(header), 1, f) != 1) { - fclose(f); - return RT_LOAD_FAILED; - } - - /* TODO(Kevin): For some reason, the checksum calculation causes - * Memory access errors . - XXH3_state_t *checksum = XXH3_createState(); - if (!checksum) { - fclose(f); - return RT_UNKNOWN_ERROR; - } - */ - - _tab.slots = rtNextPowerOfTwo32(header.num_entries * 2); - void *mem = malloc((sizeof(rt_uid) + sizeof(rt_uid_data)) * _tab.slots); - if (!mem) { - fclose(f); - _tab.slots = 0; - return RT_OUT_OF_MEMORY; - } - _tab.uids = mem; - _tab.data = (rt_uid_data *)(_tab.uids + _tab.slots); - memset(mem, 0, (sizeof(rt_uid) + sizeof(rt_uid_data)) * _tab.slots); - - uint32_t mod = _tab.slots - 1; - for (uint32_t i = 0; i < header.num_entries; ++i) { - rt_uidtab_entry entry; - if (fread(&entry, sizeof(entry), 1, f) != 1) { - free(mem); - _tab.slots = 0; - fclose(f); - return RT_LOAD_FAILED; - } - //XXH3_64bits_update(checksum, &entry, sizeof(entry)); - - /* Insert into hashtable */ - bool inserted = false; - for (uint32_t j = 0; j < _tab.slots; ++j) { - uint32_t at = (entry.uid + j) & mod; - if (_tab.uids[at] == RT_INVALID_UID) { - _tab.uids[at] = entry.uid; - _tab.data[at].pkg_file = entry.file; - _tab.data[at].size = entry.size; - _tab.data[at].offset = entry.offset; - - inserted = true; - break; - } - } - if (!inserted) { - rtReportError("core", - "Failed to insert an entry into the uid table. This should not happen."); - fclose(f); - free(mem); - _tab.slots = 0; - return RT_UNKNOWN_ERROR; - } - } - - fclose(f); - - /* - XXH64_hash_t checksum_hash = XXH3_64bits_digest(checksum); - XXH64_hash_t file_hash = XXH64_hashFromCanonical(&header.checksum); - XXH3_freeState(checksum); - - if (checksum_hash != file_hash) { - rtLog("core", - "WARNING: uidtab.bin checksum does not match calculated checksum of loaded entries."); - } - */ - - return RT_SUCCESS; -} - -void ReleaseUIDTable(void) { - free(_tab.uids); - _tab.slots = 0; -} - -RT_DLLEXPORT const rt_uid_data *rtGetUIDData(rt_uid uid) { - uint32_t mod = _tab.slots - 1; - for (uint32_t j = 0; j < _tab.slots; ++j) { - uint32_t at = (uid + j) & mod; - if (_tab.uids[at] == uid) { - return &_tab.data[at]; - } else if (_tab.uids[at] == RT_INVALID_UID) { - break; - } - } - return NULL; -} \ No newline at end of file diff --git a/src/runtime/uidtab.h b/src/runtime/uidtab.h deleted file mode 100644 index 4202fe4..0000000 --- a/src/runtime/uidtab.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef RT_UIDTAB_H -#define RT_UIDTAB_H - -#include "runtime.h" -#include "file_tab.h" -#include "assets.h" -#include "xxhash/xxhash.h" - -#ifdef RT_DEFINE_UIDTAB_FILE_STRUCTURES -#pragma pack(push, 1) -typedef struct { - XXH64_canonical_t checksum; - uint32_t num_entries; -} rt_uidtab_header; -#pragma pack(pop) - -#pragma pack(push, 1) -typedef struct { - rt_file_id file; - uint64_t offset; - uint64_t size; - rt_uid uid; -} rt_uidtab_entry; -#pragma pack(pop) -#endif - -/* Data associated with an uid */ -typedef struct { - rt_file_id pkg_file; - uint64_t offset; - uint64_t size; -} rt_uid_data; - -RT_DLLEXPORT const rt_uid_data *rtGetUIDData(rt_uid uid); - -#endif \ No newline at end of file diff --git a/src/runtime/vulkan_shader_compiler.c b/src/runtime/vulkan_shader_compiler.c new file mode 100644 index 0000000..79727cd --- /dev/null +++ b/src/runtime/vulkan_shader_compiler.c @@ -0,0 +1,171 @@ +#include "asset_compiler.h" +#include "config.h" +#include "shader_compiler.h" +#include "buffer_manager.h" + +#include +#include + +RT_CVAR_S(rt_VkShaderCompilerShaderBasepath, + "Basepath for standard includes (). Default: assets/shader/", + "assets/shader/"); + +static shaderc_include_result *IncludeResolveCallback(void *user_data, + const char *requested_source, + int type, + const char *requesting_source, + size_t include_depth) { + RT_UNUSED(include_depth); + rt_arena *arena = user_data; + + shaderc_include_result *result = RT_ARENA_PUSH_STRUCT_ZERO(arena, shaderc_include_result); + if (!result) + return NULL; + + /* Resolve the path */ + const char *basepath = NULL; + switch (type) { + case shaderc_include_type_standard: + basepath = rt_VkShaderCompilerShaderBasepath.s; + break; + case shaderc_include_type_relative: { + size_t len = strlen(requesting_source); + char *tmp = rtArenaPush(arena, len + 1); + if (!tmp) { + rtReportError("AC", "Out of memory in the vulkan shader compiler."); + result->content = "Out of memory in the vulkan shader compiler."; + result->content_length = strlen(result->content); + return result; + } + memcpy(tmp, requesting_source, len + 1); + while (len > 0 && tmp[len] != '/' && tmp[len] != '\\') + --len; + tmp[len + 1] = '\0'; + basepath = tmp; + } break; + default: + rtReportError("AC", "Invalid include type reported in %s", requesting_source); + return result; + } + size_t basepath_len = strlen(basepath); + size_t requested_len = strlen(requested_source); + char *full_path = rtArenaPush(arena, basepath_len + requested_len + 1); + if (!full_path) { + rtReportError("AC", "Out of memory in vulkan shader compiler."); + result->content = "Out of memory in the vulkan shader compiler."; + result->content_length = strlen(result->content); + } + memcpy(full_path, basepath, basepath_len); + memcpy(full_path + basepath_len, requested_source, requested_len); + full_path[basepath_len + requested_len] = '\0'; + + rt_loaded_asset source = LoadAsset(rtAddFile(full_path)); + if (source.buffer) { + result->content = source.buffer; + result->content_length = source.size; + result->source_name = full_path; + result->source_name_length = basepath_len + requested_len + 1; + } else { + char *error_tmpl = "Could not open "; + char *error_msg = rtArenaPush(arena, strlen(error_tmpl) + basepath_len + requested_len + 1); + if (error_msg) { + memcpy(error_msg, error_tmpl, strlen(error_tmpl)); + memcpy(error_msg + strlen(error_tmpl), full_path, basepath_len + requested_len + 1); + result->content = error_msg; + result->content_length = strlen(error_tmpl) + basepath_len + requested_len; + } else { + result->content = "Could not open header file."; + result->content_length = strlen(result->content); + } + } + + return result; +} + +static void IncludeReleaseCallback(void *user_data, shaderc_include_result *include_result) { + RT_UNUSED(user_data); + if (include_result->source_name) { + /* Include result contains a valid file buffer */ + rtReleaseBuffer(include_result->content, include_result->content_length); + } +} + +rt_shader_bytecode CompileVulkanShader(rt_shader_stage stage, + rt_shader_optimization_level optimization, + rt_text_span code, + const char *file_path, + rt_arena *arena) { + rt_arena_rewindpoint arena_rewind = rtGetArenaRewindPoint(arena); + + shaderc_shader_kind kind; + const char *entry; + switch (stage) { + case RT_SHADER_STAGE_VERTEX: + kind = shaderc_vertex_shader; + entry = "VsMain"; + break; + case RT_SHADER_STAGE_FRAGMENT: + kind = shaderc_fragment_shader; + entry = "PsMain"; + break; + case RT_SHADER_STAGE_COMPUTE: + kind = shaderc_compute_shader; + entry = "CsMain"; + break; + default: + rtLog("AC", "Invalid shader stage %u", stage); + return (rt_shader_bytecode){.bytes = NULL, .len = 0}; + } + + shaderc_compiler_t compiler = shaderc_compiler_initialize(); + if (!compiler) { + rtLog("AC", "Failed to initialize the shaderc compiler."); + return (rt_shader_bytecode){.bytes = NULL, .len = 0}; + } + shaderc_compile_options_t options = shaderc_compile_options_initialize(); + if (!options) { + rtLog("AC", "Failed to initialize the shaderc compile options."); + shaderc_compiler_release(compiler); + return (rt_shader_bytecode){.bytes = NULL, .len = 0}; + } + shaderc_compile_options_add_macro_definition(options, "RT_VULKAN", 9, NULL, 0); + shaderc_compile_options_set_source_language(options, shaderc_source_language_hlsl); + if (optimization == RT_SHADER_OPTIMIZATION_SIZE) + shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_size); + else if (optimization == RT_SHADER_OPTIMIZATION_SPEED) + shaderc_compile_options_set_optimization_level(options, + shaderc_optimization_level_performance); + shaderc_compile_options_set_include_callbacks(options, + IncludeResolveCallback, + IncludeReleaseCallback, + arena); + + shaderc_compilation_result_t res = shaderc_compile_into_spv(compiler, + code.start, + code.length, + kind, + file_path, + entry, + options); + rt_shader_bytecode bytecode = {.bytes = NULL, .len = 0}; + if (shaderc_result_get_compilation_status(res) != shaderc_compilation_status_success) { + rtLog("AC", "Shader compilation failed: %s", shaderc_result_get_error_message(res)); + goto out; + } + bytecode.len = shaderc_result_get_length(res); + if (bytecode.len > 0) { + bytecode.bytes = rtArenaPush(arena, bytecode.len); + if (!bytecode.bytes) { + rtLog("AC", "Failed to allocate %zu bytes for shader spv.", bytecode.len); + bytecode.len = 0; + goto out; + } + memcpy(bytecode.bytes, shaderc_result_get_bytes(res), bytecode.len); + } + +out: + shaderc_compile_options_release(options); + shaderc_compiler_release(compiler); + rtArenaRewind(arena, arena_rewind); + return bytecode; +} \ No newline at end of file