kevin/rtengine
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

load runtime asset dependency data

Browse Source 
removed init/shutdown functions from headers.
This commit is contained in:
Kevin Trogant 2023-12-21 00:19:48 +01:00
parent 213638009b
commit 0c89d63716
18 changed files with 378 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
meson.build
View File

@ -51,6 +51,7 @@ runtime_lib = library('vyrt',
# Project Sources
'src/runtime/aio.h',
'src/runtime/app.h',
'src/runtime/asset_dependencies.h',
'src/runtime/assets.h',
'src/runtime/buffer_manager.h',
'src/runtime/config.h',
@ -65,6 +66,7 @@ runtime_lib = library('vyrt',
'src/runtime/aio.c',
'src/runtime/app.c',
'src/runtime/asset_dependencies.c',
'src/runtime/buffer_manager.c',
'src/runtime/config.c',
'src/runtime/error_report.c',

28
src/runtime/aio.c
View File

@ -1,5 +1,6 @@
#include "aio.h"
#include "threading.h"
#include "config.h"
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
@ -102,7 +103,12 @@ win32CompletionRoutine(DWORD error_code, DWORD num_bytes_transfered, LPOVERLAPPE
}
#endif
VY_DLLEXPORT vy_result vyInitAIO(unsigned int max_concurrent_operations) {
VY_CVAR_I(rt_MaxConcurrentAsyncIO,
"Maximum number of concurrent async. I/O operations. Default: 1024",
1024);
vy_result InitAIO(void) {
unsigned int max_concurrent_operations = rt_MaxConcurrentAsyncIO.i;
_ringbuffer.guard = vyCreateMutex();
if (!_ringbuffer.guard) {
return VY_AIO_OUT_OF_MEMORY;
@ -119,7 +125,7 @@ VY_DLLEXPORT vy_result vyInitAIO(unsigned int max_concurrent_operations) {
return VY_SUCCESS;
}
VY_DLLEXPORT void vyShutdownAIO(void) {
void ShutdownAIO(void) {
vyDestroyMutex(_ringbuffer.guard);
free(_ringbuffer.storage);
_ringbuffer.capacity = 0;
@ -154,7 +160,6 @@ VY_DLLEXPORT vy_result vySubmitLoadBatch(const vy_load_batch *batch, vy_aio_hand
.InternalHigh = 0,
.Offset = (DWORD)(batch->loads[i].offset & MAXDWORD),
.OffsetHigh = (DWORD)(batch->loads[i].offset >> 32),
.Pointer = NULL,
};
WCHAR wpath[MAX_PATH];
@ -255,4 +260,21 @@ VY_DLLEXPORT vy_aio_state vyWaitForAIOCompletion(vy_aio_handle handle) {
#endif
} while (state == VY_AIO_STATE_PENDING);
return state;
}
VY_DLLEXPORT vy_result vySubmitSingleLoad(vy_file_load load, vy_aio_handle *handle) {
vy_load_batch batch;
batch.loads[0] = load;
batch.num_loads = 1;
return vySubmitLoadBatch(&batch, handle);
}
VY_DLLEXPORT vy_aio_state vySubmitSingleLoadSync(vy_file_load load) {
vy_aio_handle handle;
if (vySubmitSingleLoad(load, &handle) != VY_SUCCESS)
return VY_AIO_STATE_FAILED;
vy_aio_state state = vyWaitForAIOCompletion(handle);
vyReleaseAIO(handle);
return state;
}

11
src/runtime/aio.h
View File

@ -47,14 +47,12 @@ typedef enum {
VY_AIO_STATE_FAILED,
} vy_aio_state;
VY_DLLEXPORT vy_result vyInitAIO(unsigned int max_concurrent_operations);
VY_DLLEXPORT void vyShutdownAIO(void);
VY_DLLEXPORT vy_result vySubmitLoadBatch(const vy_load_batch *batch, vy_aio_handle *handles);
VY_DLLEXPORT volatile vy_aio_state vyGetAIOState(vy_aio_handle handle);
/* Blocks until the given operation is no longer pending.
* Returns the state that caused the wait to end. The handle is still valid after this function returned. */
VY_DLLEXPORT vy_aio_state vyWaitForAIOCompletion(vy_aio_handle handle);
/* Releases the internal storage for the operation.
@ -62,5 +60,10 @@ VY_DLLEXPORT vy_aio_state vyWaitForAIOCompletion(vy_aio_handle handle);
*/
VY_DLLEXPORT void vyReleaseAIO(vy_aio_handle handle);
VY_DLLEXPORT vy_result vySubmitSingleLoad(vy_file_load load, vy_aio_handle *handle);
/* Convenience wrapper for a single synchronous file load.
* Returns the state that caused the wait for completion to return. */
VY_DLLEXPORT vy_aio_state vySubmitSingleLoadSync(vy_file_load load);
#endif

137
src/runtime/asset_dependencies.c Normal file
View File

@ -0,0 +1,137 @@
#define VY_DEFINE_DEPENDENCY_FILE_STRUCTURES
#include "asset_dependencies.h"
#include "aio.h"
#include "buffer_manager.h"
#include <assert.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
typedef struct {
uint32_t begin;
uint32_t count;
} vy_dep_list;
typedef struct {
vy_uid *uids;
vy_dep_list *lists;
uint32_t capacity;
} vy_dep_map;
static vy_dep_map _map;
static vy_uid *_list_mem;
vy_result LoadAssetDependencies(void) {
vy_dependency_file_header header;
vy_file_id fid = vyAddFile("assets/deps.bin");
if (vySubmitSingleLoadSync((vy_file_load){.dest = &header,
.num_bytes = sizeof(header),
.offset = 0,
.file = fid}) != VY_AIO_STATE_FINISHED) {
vyReportError("core", "Failed to load deps.bin");
return VY_UNKNOWN_ERROR;
}
void *buffer = vyAllocBuffer(header.data_size);
if (vySubmitSingleLoadSync((vy_file_load){.dest = buffer,
.num_bytes = header.data_size,
.offset = sizeof(header),
.file = fid}) != VY_AIO_STATE_FINISHED) {
vyReportError("core", "Failed to load deps.bin");
return VY_UNKNOWN_ERROR;
}
/* We know the exact number of list entries */
uint64_t total_list_entries =
(header.data_size - header.num_lists * sizeof(vy_dependency_file_list_header)) /
sizeof(vy_uid);
_list_mem = malloc(total_list_entries * sizeof(vy_uid));
if (!_list_mem) {
vyReleaseBuffer(buffer, header.data_size);
vyReportError("core", "Failed to allocate dependency list storage.");
return VY_UNKNOWN_ERROR;
}
_map.capacity = vyNextPowerOfTwo32(header.num_lists);
_map.uids = calloc(_map.capacity, sizeof(vy_uid));
if (!_map.uids) {
free(_list_mem);
vyReleaseBuffer(buffer, header.data_size);
vyReportError("core", "Failed to allocate dependency list storage.");
return VY_UNKNOWN_ERROR;
}
_map.lists = calloc(_map.capacity, sizeof(vy_dep_list));
if (!_map.uids) {
free(_list_mem);
free(_map.uids);
vyReleaseBuffer(buffer, header.data_size);
vyReportError("core", "Failed to allocate dependency list storage.");
return VY_UNKNOWN_ERROR;
}
uint32_t storage_at = 0;
vy_dependency_file_list_header *list = buffer;
for (uint32_t i = 0; i < header.num_lists; ++i) {
const vy_uid *entries = (vy_uid *)(list + 1);
/* Validate the checksum */
XXH64_hash_t file_hash = XXH64_hashFromCanonical(&list->checksum);
XXH64_hash_t calc_hash = XXH3_64bits(entries, sizeof(vy_uid) * list->num_entries);
if (file_hash != calc_hash) {
vyReportError("core", "Checksum mismatch in list %u", i);
vyReleaseBuffer(buffer, header.data_size);
return VY_UNKNOWN_ERROR;
}
/* Store the list */
memcpy(_list_mem + storage_at, entries, sizeof(vy_uid) * list->num_entries);
bool inserted = false;
for (uint32_t j = 0; j < _map.capacity; ++j) {
uint32_t slot = (list->uid + j) % _map.capacity;
if (_map.uids[slot] == VY_INVALID_UID) {
_map.uids[slot] = list->uid;
_map.lists[slot].begin = storage_at;
_map.lists[slot].count = list->num_entries;
inserted = true;
break;
}
}
storage_at += list->num_entries;
assert(inserted);
assert(storage_at <= total_list_entries);
list = (vy_dependency_file_list_header *)(entries + list->num_entries);
}
vyReleaseBuffer(buffer, header.data_size);
return VY_SUCCESS;
}
void ReleaseAssetDependencies(void) {
free(_list_mem);
free(_map.uids);
free(_map.lists);
}
VY_DLLEXPORT vy_asset_dependency_list vyGetAssetDependencies(vy_uid asset) {
vy_asset_dependency_list result = {
.dependencies = NULL,
.count = 0,
};
for (uint32_t i = 0; i < _map.capacity; ++i) {
uint32_t slot = (asset + i) % _map.capacity;
if (_map.uids[slot] == asset) {
result.dependencies = &_list_mem[_map.lists[slot].begin];
result.count = _map.lists[slot].count;
break;
} else if (_map.uids[slot] == VY_INVALID_UID) {
break;
}
}
return result;
}

33
src/runtime/asset_dependencies.h Normal file
View File

@ -0,0 +1,33 @@
#ifndef VY_ASSET_DEPENDENCIES_H
#define VY_ASSET_DEPENDENCIES_H
#include "assets.h"
#ifdef VY_DEFINE_DEPENDENCY_FILE_STRUCTURES
#include "xxhash/xxhash.h"
#pragma pack(push, 1)
typedef struct {
uint64_t data_size;
uint32_t num_lists;
} vy_dependency_file_header;
typedef struct {
vy_uid uid;
uint32_t num_entries;
XXH64_canonical_t checksum;
} vy_dependency_file_list_header;
#pragma pack(pop)
#endif
typedef struct {
const vy_uid *dependencies;
uint32_t count;
} vy_asset_dependency_list;
VY_DLLEXPORT vy_asset_dependency_list vyGetAssetDependencies(vy_uid asset);
#endif

4
src/runtime/buffer_manager.c
View File

@ -57,7 +57,7 @@ VY_CVAR_SZ(rt_BufferManagerMemory,
"Total number of bytes allocated for the buffer manager. Default: 1GB",
VY_GB(1));
VY_DLLEXPORT vy_result vyInitBufferManager(void) {
vy_result InitBufferManager(void) {
if ((rt_BufferManagerMemory.sz % NUM_BLOCK_SIZES) != 0)
vyLog("BUFFERMGR",
"Configured memory amount is not dividable by number of block "
@ -106,7 +106,7 @@ VY_DLLEXPORT vy_result vyInitBufferManager(void) {
return VY_SUCCESS;
}
VY_DLLEXPORT void vyShutdownBufferManager(void) {
void ShutdownBufferManager(void) {
for (unsigned int i = 0; i < NUM_BLOCK_SIZES; ++i) {
vyDestroyMutex(_regions[i].guard);
free(_regions[i].memory);

4
src/runtime/buffer_manager.h
View File

@ -8,10 +8,6 @@ enum {
VY_BUFFER_MGR_MUTEX_CREATION_FAILED,
};
VY_DLLEXPORT vy_result vyInitBufferManager(void);
VY_DLLEXPORT void vyShutdownBufferManager(void);
VY_DLLEXPORT void *vyAllocBuffer(size_t size);
VY_DLLEXPORT void vyReleaseBuffer(const void *begin, size_t size);

12
src/runtime/file_tab.c
View File

@ -1,5 +1,6 @@
#include "file_tab.h"
#include "threading.h"
#include "config.h"
#include <xxhash/xxhash.h>
@ -16,7 +17,10 @@ typedef struct {
static vy_file_tab _file_tab;
vy_result vyInitFileTab(unsigned int max_files) {
VY_CVAR_I(rt_FileTabCapacity, "Maximum number of filetab entries. Default: 1024", 1024);
vy_result InitFileTab(void) {
unsigned int max_files = (unsigned int)rt_FileTabCapacity.i;
_file_tab.ids = calloc(max_files, sizeof(vy_file_id));
if (!_file_tab.ids)
return 1;
@ -33,21 +37,21 @@ vy_result vyInitFileTab(unsigned int max_files) {
return VY_SUCCESS;
}
void vyShutdownFileTab(void) {
void ShutdownFileTab(void) {
free(_file_tab.ids);
free(_file_tab.names);
free(_file_tab.name_offsets);
vyDestroyMutex(_file_tab.mutex);
}
vy_file_id vyGetFileId(const char *path) {
VY_DLLEXPORT vy_file_id vyGetFileId(const char *path) {
vy_text_span span;
span.start = path;
span.length = (unsigned int)strlen(path);
return vyGetFileIdFromSpan(span);
}
vy_file_id vyGetFileIdFromSpan(vy_text_span path) {
VY_DLLEXPORT vy_file_id vyGetFileIdFromSpan(vy_text_span path) {
/* Randomly choosen, aka finger smash keyboard */
XXH64_hash_t seed = 15340978;
vy_file_id fid = (vy_file_id)XXH3_64bits_withSeed(path.start, path.length, seed);

4
src/runtime/file_tab.h
View File

@ -9,10 +9,6 @@
typedef uint64_t vy_file_id;
#define VY_INVALID_FILE_ID 0
VY_DLLEXPORT vy_result vyInitFileTab(unsigned int max_files);
VY_DLLEXPORT void vyShutdownFileTab(void);
VY_DLLEXPORT vy_file_id vyGetFileId(const char *path);
VY_DLLEXPORT vy_file_id vyGetFileIdFromSpan(vy_text_span path);

46
src/runtime/init.c
View File

@ -10,47 +10,69 @@ extern vy_cvar rt_Fullscreen;
extern vy_cvar rt_WindowWidth;
extern vy_cvar rt_WindowHeight;
extern vy_cvar rt_BufferManagerMemory;
extern vy_cvar rt_FileTabCapacity;
extern vy_cvar rt_MaxConcurrentAsyncIO;
void __RegisterRuntimeCVars(void) {
void RegisterRuntimeCVars(void) {
vyRegisterCVAR(&rt_Renderer);
vyRegisterCVAR(&rt_Fullscreen);
vyRegisterCVAR(&rt_WindowWidth);
vyRegisterCVAR(&rt_WindowHeight);
vyRegisterCVAR(&rt_BufferManagerMemory);
vyRegisterCVAR(&rt_FileTabCapacity);
vyRegisterCVAR(&rt_MaxConcurrentAsyncIO);
}
extern void __RTSetMainThreadId(void);
extern void SetMainThreadId(void);
extern vy_result InitBufferManager(void);
extern void ShutdownBufferManager(void);
extern vy_result InitFileTab(void);
extern void ShutdownFileTab(void);
extern vy_result InitAIO(void);
extern void ShutdownAIO(void);
extern vy_result LoadUIDTable(void);
extern void ReleaseUIDTable(void);
extern vy_result LoadAssetDependencies(void);
extern void ReleaseAssetDependencies(void);
VY_DLLEXPORT vy_result vyInitRuntime(void) {
__RTSetMainThreadId();
__RegisterRuntimeCVars();
SetMainThreadId();
RegisterRuntimeCVars();
vy_result res;
if ((res = vyInitBufferManager()) != VY_SUCCESS) {
if ((res = InitBufferManager()) != VY_SUCCESS) {
vyReportError("BUFFERMGR", "Init failed.");
return res;
}
if ((res = vyInitFileTab(1024)) != VY_SUCCESS) {
if ((res = InitFileTab()) != VY_SUCCESS) {
vyReportError("FTAB", "Init failed.");
return res;
}
if ((res = vyInitAIO(0)) != VY_SUCCESS) {
if ((res = InitAIO()) != VY_SUCCESS) {
vyReportError("AIO", "Init failed.");
return res;
}
if ((res = vyLoadUIDTable()) != VY_SUCCESS) {
if ((res = LoadUIDTable()) != VY_SUCCESS) {
vyLog("CORE", "LoadUIDTable returned result: %u", res);
}
if ((res = LoadAssetDependencies()) != VY_SUCCESS) {
vyReportError("ASSETDEP", "Init failed.");
return res;
}
return VY_SUCCESS;
}
VY_DLLEXPORT void vyShutdownRuntime(void) {
vyReleaseUIDTable();
vyShutdownAIO();
vyShutdownFileTab();
vyShutdownBufferManager();
ReleaseAssetDependencies();
ReleaseUIDTable();
ShutdownAIO();
ShutdownFileTab();
ShutdownBufferManager();
}

15
src/runtime/runtime.h
View File

@ -87,6 +87,21 @@ static VY_INLINE void vySetRelptr(vy_relptr *ptr, void *target) {
}
}
/* Bitfiddling functions */
/* Portable solution, On x64 using clzl is probably faster. */
static VY_INLINE uint32_t vyNextPowerOfTwo32(uint32_t v) {
v--;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
v++;
return v;
}
/* Runtime init. Initializes basic systems.
* You need to call this, even if you build a CLI only app. */
VY_DLLEXPORT vy_result vyInitRuntime(void);

2
src/runtime/threading_thread.c
View File

@ -28,7 +28,7 @@ static INIT_ONCE _guard_init = INIT_ONCE_STATIC_INIT;
static vy_thread_id _main_thread_id;
/* Called by the runtime during setup */
extern void __RTSetMainThreadId(void) {
extern void SetMainThreadId(void) {
_main_thread_id = (vy_thread_id)GetCurrentThreadId();
}

20
src/runtime/uidtab.c
View File

@ -17,23 +17,11 @@ typedef struct {
static vy_uidtab _tab;
/* Portable solution, On x64 using clzl is probably faster. */
static uint32_t NextPowerOfTwo(uint32_t v) {
v--;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
v++;
return v;
}
VY_DLLEXPORT vy_result vyLoadUIDTable(void) {
vy_result LoadUIDTable(void) {
/* We use stdio here, because we cannot load any asset in parallel to this.
* This is because the uidtab is what tells us which assets exist.
*/
FILE *f = fopen("uidtab.bin", "rb");
FILE *f = fopen("assets/uidtab.bin", "rb");
if (!f)
return VY_LOAD_FAILED;
@ -52,7 +40,7 @@ VY_DLLEXPORT vy_result vyLoadUIDTable(void) {
}
*/
_tab.slots = NextPowerOfTwo(header.num_entries * 2);
_tab.slots = vyNextPowerOfTwo32(header.num_entries * 2);
void *mem = malloc((sizeof(vy_uid) + sizeof(vy_uid_data)) * _tab.slots);
if (!mem) {
fclose(f);
@ -114,7 +102,7 @@ VY_DLLEXPORT vy_result vyLoadUIDTable(void) {
return VY_SUCCESS;
}
VY_DLLEXPORT void vyReleaseUIDTable(void) {
void ReleaseUIDTable(void) {
free(_tab.uids);
_tab.slots = 0;
}

2
src/runtime/uidtab.h
View File

@ -31,8 +31,6 @@ typedef struct {
uint64_t size;
} vy_uid_data;
VY_DLLEXPORT vy_result vyLoadUIDTable(void);
VY_DLLEXPORT void vyReleaseUIDTable(void);
VY_DLLEXPORT const vy_uid_data *vyGetUIDData(vy_uid uid);
#endif

6
src/tools/assetc/assetc.c
View File

@ -102,9 +102,6 @@ int main(int argc, char **argv) {
vyInitPackages();
vySetWorkingDirectory(g_assetc_options.root_directory);
/* Explictily reload uidtab, because we changed the working directory here */
vyLoadUIDTable();
if (vyLoadAssetMeta() != VY_SUCCESS) {
return 1;
}
@ -136,6 +133,9 @@ int main(int argc, char **argv) {
if (vyWriteUIDTab() != VY_SUCCESS) {
return 1;
}
if (vySaveAssetDependencies() != VY_SUCCESS) {
return 1;
}
vyShutdownRuntime();

101
src/tools/assetc/dependency_tracking.c
View File

@ -1,13 +1,16 @@
#include "dependency_tracking.h"
#define VY_DEFINE_DEPENDENCY_FILE_STRUCTURES
#include "runtime/assets.h"
#include "runtime/threading.h"
#include "runtime/asset_dependencies.h"
#include <stdbool.h>
#include <limits.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
/* Track a list of dependencies per asset.
@ -125,3 +128,101 @@ vy_result vyAddAssetDependency(vy_uid dependent, vy_uid dependency) {
res = InsertIntoList(dependency, dependent, 1);
return res;
}
vy_result vySaveAssetDependencies(void) {
assert(vyIsMainThread());
vy_dependency_file_header header = {.num_lists = 0, .data_size = 0};
for (size_t i = 0; i < MAP_SIZE; ++i) {
if (_uids[i] != VY_INVALID_UID) {
if (!_lists[i].dependencies)
continue;
header.num_lists += 1;
/* Determine the list size */
vy_dep_list_bucket *bucket = &_buckets[_lists[i].dependencies];
uint32_t total_list_size = bucket->count;
while (bucket->next != END_OF_LIST) {
bucket = &_buckets[bucket->next];
total_list_size += bucket->count;
}
header.data_size += total_list_size * sizeof(vy_uid) + sizeof(vy_dependency_file_list_header);
}
}
FILE *f = fopen("deps.bin", "wb");
if (!f) {
vyReportError("ASSETC", "Failed to open 'deps.bin' for writing.");
return VY_UNKNOWN_ERROR;
}
if (fwrite(&header, sizeof(header), 1, f) != 1) {
vyReportError("ASSETC", "Failed to write to 'deps.bin'.");
fclose(f);
return VY_UNKNOWN_ERROR;
}
void *buffer = NULL;
size_t buffer_size = 0;
for (size_t i = 0; i < MAP_SIZE; ++i) {
if (_uids[i] != VY_INVALID_UID) {
if (!_lists[i].dependencies)
continue;
/* Determine the list size */
vy_dep_list_bucket *bucket = &_buckets[_lists[i].dependencies];
uint32_t total_list_size = bucket->count;
while (bucket->next != END_OF_LIST) {
bucket = &_buckets[bucket->next];
total_list_size += bucket->count;
}
/* Allocate */
size_t required_size =
total_list_size * sizeof(vy_uid) + sizeof(vy_dependency_file_list_header);
if (required_size > buffer_size) {
void *t = realloc(buffer, required_size);
if (!t) {
free(buffer);
fclose(f);
return VY_UNKNOWN_ERROR;
}
buffer = t;
buffer_size = required_size;
}
/* Fill header */
vy_dependency_file_list_header *list_header = buffer;
vy_uid *list = (vy_uid *)(list_header + 1);
list_header->uid = _uids[i];
list_header->num_entries = total_list_size;
/* Copy the list */
uint32_t at = 0;
bucket = &_buckets[_lists[i].dependencies];
do {
memcpy(&list[at], bucket->entries, sizeof(vy_uid) * bucket->count);
at += bucket->count;
bucket = &_buckets[bucket->next];
} while (bucket != &_buckets[END_OF_LIST]);
XXH64_hash_t hash = XXH3_64bits(list, sizeof(vy_uid) * total_list_size);
XXH64_canonicalFromHash(&list_header->checksum, hash);
if (fwrite(buffer, required_size, 1, f) != 1) {
vyReportError("ASSETC", "Failed to write to 'deps.bin'.");
fclose(f);
free(buffer);
return VY_UNKNOWN_ERROR;
}
}
}
fclose(f);
free(buffer);
return VY_SUCCESS;
}

2
src/tools/assetc/dependency_tracking.h
View File

@ -9,4 +9,6 @@ vy_result vyInitDependencyTracking(void);
vy_result vyAddAssetDependency(vy_uid dependent, vy_uid dependency);
vy_result vySaveAssetDependencies(void);
#endif

4
src/tools/assetc/discovery.c
View File

@ -33,6 +33,10 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
else if (memcmp(&name[name_len - 4], ".bin", 4) == 0)
return VY_SUCCESS;
}
if (name[0] == '.') {
return VY_SUCCESS;
}
/* Check if we know that file */
if (data->path_end > 0) {
data->path_scratch[data->path_end] = '/';