kevin/rtengine
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

Add a resource namespace file format

Browse Source
This commit is contained in:
Kevin Trogant 2024-02-04 18:31:02 +01:00
parent 0b6e855cef
commit 870f238410
4 changed files with 152 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/runtime/asset_compiler.c
View File

@ -389,7 +389,6 @@ rt_loaded_asset LoadAsset(rt_file_id file) {
size_t file_size = rtGetFileSize(path);
void *buffer = rtAllocBuffer(file_size);
rtLog("AC", "Buffer ptr %llx", (uintptr_t)buffer);
if (!buffer) {
rtLog("AC", "Failed to allocate buffer for loading %s.", path);
return (rt_loaded_asset){.buffer = NULL, .size = 0};

1
src/runtime/buffer_manager.c
View File

@ -156,7 +156,6 @@ RT_DLLEXPORT void *rtAllocBuffer(size_t size) {
}
rtUnlockMutex(_guard);
rtLog("BUFFERMGR", "Result ptr %llx", (uintptr_t)result);
return result;
}

170
src/runtime/resource_manager.c
View File

@ -25,6 +25,9 @@ RT_CVAR_I(rt_MaxCachedResources,
RT_CVAR_I(rt_ResourceNamespaceSize,
"The maximum number of resources that can exist. Default: 1.048.576",
1048576);
RT_CVAR_I(rt_DisableResourceNamespaceLoad,
"Disables the load of the saved resource namespace. Default: 0 (off)",
0);
#define RT_TOMBSTONE_ID 1
@ -72,6 +75,18 @@ typedef struct {
rt_rwlock lock;
} rt_resource_namespace;
#pragma pack(push, 1)
typedef struct {
rt_hash64 checksum;
uint32_t num_entries;
} rt_namespace_file_header;
typedef struct {
rt_resource_id id;
rt_resource_ref ref;
} rt_namespace_file_entry;
#pragma pack(pop)
/* ~~~ Utilities ~~~ */
static size_t GetResourceDataSize(const rt_resource *resource) {
@ -107,25 +122,6 @@ static void CopyResourceData(const rt_resource *resource, void *dest) {
}
}
#if 0
static rt_resource_ref *GetResourceRefPtr(rt_resource_id id) {
rt_resource_ref *ref = NULL;
rtLockRead(&_namespace.lock);
size_t ns_size = (size_t)rt_ResourceNamespaceSize.i;
for (size_t j = 0; j < ns_size; ++j) {
size_t at = (id + j) % ns_size;
if (_namespace.ids[at] == RT_INVALID_RESOURCE_ID) {
break;
} else if (_namespace.ids[at] == id) {
ref = &_namespace.refs[at];
break;
}
}
rtUnlockRead(&_namespace.lock);
return ref;
}
#endif
/* Fills the passed write struct with the necessary information to save the resource to a file */
static bool PrepareResourceFlushToFile(rt_resource_id id,
const rt_resource *resource,
@ -441,6 +437,79 @@ static void ShutdownNamespace(void) {
memset(&_namespace, 0, sizeof(_namespace));
}
static void LoadNamespace(void) {
char path[260];
rtSPrint(path, RT_ARRAY_COUNT(path), "%s/namespace.bin", rt_ResourceDirectory.s);
rt_file_id fid = rtAddFile(path);
size_t file_size = rtGetFileSize(path);
if (file_size == 0) {
rtLog("RESMGR", "Unable to determine size of %s", path);
return;
}
rt_temp_arena temp = rtGetTemporaryArena(NULL, 0);
if (!temp.arena) {
rtLog("RESMGR", "Unable to get temporary arena for loading the namespace.");
return;
}
void *dest = rtArenaPush(temp.arena, file_size);
if (!dest) {
rtReturnTemporaryArena(temp);
rtLog("RESMGR", "Unable to allocate temporary space for loading the namespace.");
return;
}
rt_aio_state state = rtSubmitSingleLoadSync(
(rt_file_load){.file = fid, .num_bytes = file_size, .offset = 0, .dest = dest});
if (state == RT_AIO_STATE_FINISHED) {
const rt_namespace_file_header *header = dest;
const rt_namespace_file_entry *entries = (const rt_namespace_file_entry *)(header + 1);
if ((header->num_entries * sizeof(rt_namespace_file_entry)) <=
(file_size - sizeof(*header))) {
rt_hash64 entries_hash =
rtHashBytes(entries, sizeof(rt_namespace_file_entry) * header->num_entries);
if (entries_hash == header->checksum) {
size_t ns_size = (size_t)rt_ResourceNamespaceSize.i;
for (uint32_t i = 0; i < header->num_entries; ++i) {
bool inserted = false;
for (size_t j = 0; j < ns_size; ++j) {
size_t at = (entries[i].id + j) % ns_size;
if (_namespace.ids[at] == RT_INVALID_RESOURCE_ID) {
inserted = true;
_namespace.ids[at] = entries[i].id;
_namespace.refs[at] = entries[i].ref;
break;
} else if (_namespace.ids[at] == entries[i].id) {
rtReportError(
"RESMGR",
"Resource ID (%llx) collision detected in namespace file %s",
entries[i].id,
path);
rtReturnTemporaryArena(temp);
return;
}
}
if (!inserted) {
rtReportError("RESMGR",
"Failed to insert namespace entry %llx",
entries[i].id);
break;
}
}
} else {
rtLog("RESMGR", "Checksum mismatch in %s", path);
}
} else {
rtLog("RESMGR", "Number of entries in %s is inconsistent with the file size.", path);
}
} else {
rtLog("RESMGR", "Failed to load %s.", path);
}
rtReturnTemporaryArena(temp);
}
static rt_resource_ref GetResourceRef(rt_resource_id id) {
rt_resource_ref ref = {.file = RT_INVALID_FILE_ID};
rtLockRead(&_namespace.lock);
@ -472,6 +541,9 @@ rt_result InitResourceManager(void) {
ShutdownResourceCache();
return res;
}
if (!rt_DisableResourceNamespaceLoad.i) {
LoadNamespace();
}
return RT_SUCCESS;
}
@ -736,7 +808,7 @@ RT_DLLEXPORT rt_result rtCreateResources(uint32_t count,
"Resource ID collision occured with resource %s.\nID: %llx",
names[i],
id);
result = RT_INVALID_FILE_ID;
result = RT_INVALID_VALUE;
goto out;
}
}
@ -858,3 +930,61 @@ RT_DLLEXPORT void rDebugLogResource(rt_resource_id id, const rt_resource *resour
rtLog("RESMGR", " unknown data at: %llx", (uintptr_t)resource->data);
}
}
RT_DLLEXPORT void rtSaveResourceNamespace(void) {
rt_temp_arena temp = rtGetTemporaryArena(NULL, 0);
rtLockRead(&_namespace.lock);
uint32_t entry_count = 0;
for (size_t i = 0; i < (size_t)rt_ResourceNamespaceSize.i; ++i) {
if (_namespace.ids[i] != RT_INVALID_RESOURCE_ID)
++entry_count;
}
size_t buffer_size =
sizeof(rt_namespace_file_header) + entry_count * sizeof(rt_namespace_file_entry);
void *buffer = rtArenaPush(temp.arena, buffer_size);
if (!buffer) {
rtReportError(
"RESMGR",
"Failed to allocate temporary buffer (%zu bytes) for writing the namespace file");
goto out;
}
rt_namespace_file_header *header = buffer;
rt_namespace_file_entry *entries = (rt_namespace_file_entry *)(header + 1);
size_t at = 0;
for (size_t i = 0; i < (size_t)rt_ResourceNamespaceSize.i; ++i) {
if (_namespace.ids[i] != RT_INVALID_RESOURCE_ID) {
entries[at].id = _namespace.ids[i];
entries[at].ref = _namespace.refs[i];
++at;
}
}
RT_ASSERT(at == entry_count, "");
header->num_entries = entry_count;
header->checksum = rtHashBytes(entries, entry_count * sizeof(rt_namespace_file_entry));
char path[260];
rtSPrint(path, RT_ARRAY_COUNT(path), "%s/namespace.bin", rt_ResourceDirectory.s);
rt_write_batch write = {.num_writes = 1};
write.writes[0] = (rt_file_write){
.file = rtAddFile(path),
.buffer = buffer,
.num_bytes = buffer_size,
.offset = 0,
};
rt_aio_handle handle;
if (rtSubmitWriteBatch(&write, &handle) != RT_SUCCESS) {
rtReportError("RESMGR", "Failed to submit the write for %s", path);
goto out;
}
rt_aio_state state = rtWaitForAIOCompletion(handle);
rtReleaseAIO(handle);
if (state != RT_AIO_STATE_FINISHED)
rtReportError("RESMGR", "Write to %s failed: %u", path, state);
out:
rtUnlockRead(&_namespace.lock);
rtReturnTemporaryArena(temp);
}

2
src/runtime/resources.h
View File

@ -77,6 +77,8 @@ RT_DLLEXPORT size_t rtGetResourceSize(rt_resource_id id);
/* Logs information about a resource. Useful for debugging */
RT_DLLEXPORT void rDebugLogResource(rt_resource_id id, const rt_resource *resource);
RT_DLLEXPORT void rtSaveResourceNamespace(void);
/* Registers resources with the resource manager, making them available to the system.
*
* The runtime will create a standalone file for each resource in the resource directory.