From 870f238410b3729de629ba8a3f7bdb27e58182e1 Mon Sep 17 00:00:00 2001 From: Kevin Trogant Date: Sun, 4 Feb 2024 18:31:02 +0100 Subject: [PATCH] Add a resource namespace file format --- src/runtime/asset_compiler.c | 1 - src/runtime/buffer_manager.c | 1 - src/runtime/resource_manager.c | 170 +++++++++++++++++++++++++++++---- src/runtime/resources.h | 2 + 4 files changed, 152 insertions(+), 22 deletions(-) diff --git a/src/runtime/asset_compiler.c b/src/runtime/asset_compiler.c index d4a798c..40f152a 100644 --- a/src/runtime/asset_compiler.c +++ b/src/runtime/asset_compiler.c @@ -389,7 +389,6 @@ rt_loaded_asset LoadAsset(rt_file_id file) { size_t file_size = rtGetFileSize(path); void *buffer = rtAllocBuffer(file_size); - rtLog("AC", "Buffer ptr %llx", (uintptr_t)buffer); if (!buffer) { rtLog("AC", "Failed to allocate buffer for loading %s.", path); return (rt_loaded_asset){.buffer = NULL, .size = 0}; diff --git a/src/runtime/buffer_manager.c b/src/runtime/buffer_manager.c index 7bb7b8a..ccce030 100644 --- a/src/runtime/buffer_manager.c +++ b/src/runtime/buffer_manager.c @@ -156,7 +156,6 @@ RT_DLLEXPORT void *rtAllocBuffer(size_t size) { } rtUnlockMutex(_guard); - rtLog("BUFFERMGR", "Result ptr %llx", (uintptr_t)result); return result; } diff --git a/src/runtime/resource_manager.c b/src/runtime/resource_manager.c index ebb7668..15652b5 100644 --- a/src/runtime/resource_manager.c +++ b/src/runtime/resource_manager.c @@ -25,6 +25,9 @@ RT_CVAR_I(rt_MaxCachedResources, RT_CVAR_I(rt_ResourceNamespaceSize, "The maximum number of resources that can exist. Default: 1.048.576", 1048576); +RT_CVAR_I(rt_DisableResourceNamespaceLoad, + "Disables the load of the saved resource namespace. Default: 0 (off)", + 0); #define RT_TOMBSTONE_ID 1 @@ -72,6 +75,18 @@ typedef struct { rt_rwlock lock; } rt_resource_namespace; +#pragma pack(push, 1) +typedef struct { + rt_hash64 checksum; + uint32_t num_entries; +} rt_namespace_file_header; + +typedef struct { + rt_resource_id id; + rt_resource_ref ref; +} rt_namespace_file_entry; +#pragma pack(pop) + /* ~~~ Utilities ~~~ */ static size_t GetResourceDataSize(const rt_resource *resource) { @@ -107,25 +122,6 @@ static void CopyResourceData(const rt_resource *resource, void *dest) { } } -#if 0 -static rt_resource_ref *GetResourceRefPtr(rt_resource_id id) { - rt_resource_ref *ref = NULL; - rtLockRead(&_namespace.lock); - size_t ns_size = (size_t)rt_ResourceNamespaceSize.i; - for (size_t j = 0; j < ns_size; ++j) { - size_t at = (id + j) % ns_size; - if (_namespace.ids[at] == RT_INVALID_RESOURCE_ID) { - break; - } else if (_namespace.ids[at] == id) { - ref = &_namespace.refs[at]; - break; - } - } - rtUnlockRead(&_namespace.lock); - return ref; -} -#endif - /* Fills the passed write struct with the necessary information to save the resource to a file */ static bool PrepareResourceFlushToFile(rt_resource_id id, const rt_resource *resource, @@ -441,6 +437,79 @@ static void ShutdownNamespace(void) { memset(&_namespace, 0, sizeof(_namespace)); } +static void LoadNamespace(void) { + char path[260]; + rtSPrint(path, RT_ARRAY_COUNT(path), "%s/namespace.bin", rt_ResourceDirectory.s); + rt_file_id fid = rtAddFile(path); + size_t file_size = rtGetFileSize(path); + if (file_size == 0) { + rtLog("RESMGR", "Unable to determine size of %s", path); + return; + } + + rt_temp_arena temp = rtGetTemporaryArena(NULL, 0); + if (!temp.arena) { + rtLog("RESMGR", "Unable to get temporary arena for loading the namespace."); + return; + } + void *dest = rtArenaPush(temp.arena, file_size); + if (!dest) { + rtReturnTemporaryArena(temp); + rtLog("RESMGR", "Unable to allocate temporary space for loading the namespace."); + return; + } + + rt_aio_state state = rtSubmitSingleLoadSync( + (rt_file_load){.file = fid, .num_bytes = file_size, .offset = 0, .dest = dest}); + if (state == RT_AIO_STATE_FINISHED) { + const rt_namespace_file_header *header = dest; + const rt_namespace_file_entry *entries = (const rt_namespace_file_entry *)(header + 1); + + if ((header->num_entries * sizeof(rt_namespace_file_entry)) <= + (file_size - sizeof(*header))) { + rt_hash64 entries_hash = + rtHashBytes(entries, sizeof(rt_namespace_file_entry) * header->num_entries); + if (entries_hash == header->checksum) { + size_t ns_size = (size_t)rt_ResourceNamespaceSize.i; + for (uint32_t i = 0; i < header->num_entries; ++i) { + bool inserted = false; + for (size_t j = 0; j < ns_size; ++j) { + size_t at = (entries[i].id + j) % ns_size; + if (_namespace.ids[at] == RT_INVALID_RESOURCE_ID) { + inserted = true; + _namespace.ids[at] = entries[i].id; + _namespace.refs[at] = entries[i].ref; + break; + } else if (_namespace.ids[at] == entries[i].id) { + rtReportError( + "RESMGR", + "Resource ID (%llx) collision detected in namespace file %s", + entries[i].id, + path); + rtReturnTemporaryArena(temp); + return; + } + } + if (!inserted) { + rtReportError("RESMGR", + "Failed to insert namespace entry %llx", + entries[i].id); + break; + } + } + } else { + rtLog("RESMGR", "Checksum mismatch in %s", path); + } + } else { + rtLog("RESMGR", "Number of entries in %s is inconsistent with the file size.", path); + } + } else { + rtLog("RESMGR", "Failed to load %s.", path); + } + + rtReturnTemporaryArena(temp); +} + static rt_resource_ref GetResourceRef(rt_resource_id id) { rt_resource_ref ref = {.file = RT_INVALID_FILE_ID}; rtLockRead(&_namespace.lock); @@ -472,6 +541,9 @@ rt_result InitResourceManager(void) { ShutdownResourceCache(); return res; } + if (!rt_DisableResourceNamespaceLoad.i) { + LoadNamespace(); + } return RT_SUCCESS; } @@ -736,7 +808,7 @@ RT_DLLEXPORT rt_result rtCreateResources(uint32_t count, "Resource ID collision occured with resource %s.\nID: %llx", names[i], id); - result = RT_INVALID_FILE_ID; + result = RT_INVALID_VALUE; goto out; } } @@ -858,3 +930,61 @@ RT_DLLEXPORT void rDebugLogResource(rt_resource_id id, const rt_resource *resour rtLog("RESMGR", " unknown data at: %llx", (uintptr_t)resource->data); } } + +RT_DLLEXPORT void rtSaveResourceNamespace(void) { + rt_temp_arena temp = rtGetTemporaryArena(NULL, 0); + rtLockRead(&_namespace.lock); + uint32_t entry_count = 0; + for (size_t i = 0; i < (size_t)rt_ResourceNamespaceSize.i; ++i) { + if (_namespace.ids[i] != RT_INVALID_RESOURCE_ID) + ++entry_count; + } + + size_t buffer_size = + sizeof(rt_namespace_file_header) + entry_count * sizeof(rt_namespace_file_entry); + void *buffer = rtArenaPush(temp.arena, buffer_size); + if (!buffer) { + rtReportError( + "RESMGR", + "Failed to allocate temporary buffer (%zu bytes) for writing the namespace file"); + goto out; + } + + rt_namespace_file_header *header = buffer; + rt_namespace_file_entry *entries = (rt_namespace_file_entry *)(header + 1); + size_t at = 0; + for (size_t i = 0; i < (size_t)rt_ResourceNamespaceSize.i; ++i) { + if (_namespace.ids[i] != RT_INVALID_RESOURCE_ID) { + entries[at].id = _namespace.ids[i]; + entries[at].ref = _namespace.refs[i]; + ++at; + } + } + RT_ASSERT(at == entry_count, ""); + header->num_entries = entry_count; + header->checksum = rtHashBytes(entries, entry_count * sizeof(rt_namespace_file_entry)); + + char path[260]; + rtSPrint(path, RT_ARRAY_COUNT(path), "%s/namespace.bin", rt_ResourceDirectory.s); + + rt_write_batch write = {.num_writes = 1}; + write.writes[0] = (rt_file_write){ + .file = rtAddFile(path), + .buffer = buffer, + .num_bytes = buffer_size, + .offset = 0, + }; + rt_aio_handle handle; + if (rtSubmitWriteBatch(&write, &handle) != RT_SUCCESS) { + rtReportError("RESMGR", "Failed to submit the write for %s", path); + goto out; + } + rt_aio_state state = rtWaitForAIOCompletion(handle); + rtReleaseAIO(handle); + if (state != RT_AIO_STATE_FINISHED) + rtReportError("RESMGR", "Write to %s failed: %u", path, state); + +out: + rtUnlockRead(&_namespace.lock); + rtReturnTemporaryArena(temp); +} \ No newline at end of file diff --git a/src/runtime/resources.h b/src/runtime/resources.h index bfe48d7..76f90f1 100644 --- a/src/runtime/resources.h +++ b/src/runtime/resources.h @@ -77,6 +77,8 @@ RT_DLLEXPORT size_t rtGetResourceSize(rt_resource_id id); /* Logs information about a resource. Useful for debugging */ RT_DLLEXPORT void rDebugLogResource(rt_resource_id id, const rt_resource *resource); +RT_DLLEXPORT void rtSaveResourceNamespace(void); + /* Registers resources with the resource manager, making them available to the system. * * The runtime will create a standalone file for each resource in the resource directory.