kevin/rtengine
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

assetc complete "lifetime"

Browse Source 
- Add unchanged assets to packages
- Allows re-runs without messing up the data
- Inefficient: unchanged assets need to be loaded & kept in memory
This commit is contained in:
Kevin Trogant 2023-12-22 00:25:27 +01:00
parent 0c89d63716
commit 5a4177c4dc
17 changed files with 182 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -2,3 +2,6 @@
/subprojects/*
!/subprojects/*.wrap
# assetc directories
/actemp/*
/data/*

4
assets/shader/cell.pipeline
View File

@ -1,10 +1,10 @@
optimization speed;
vertex {
vk shader/cell_vert.glsl;
vk assets/shader/cell_vert.glsl;
}
fragment {
vk shader/cell_frag.glsl;
vk assets/shader/cell_frag.glsl;
}
texture_bindings {

3
meson.build
View File

@ -59,6 +59,7 @@ runtime_lib = library('vyrt',
'src/runtime/file_tab.h',
'src/runtime/gfx.h',
'src/runtime/jobs.h',
'src/runtime/packages.h',
'src/runtime/renderer_api.h',
'src/runtime/runtime.h',
'src/runtime/threading.h',
@ -67,6 +68,7 @@ runtime_lib = library('vyrt',
'src/runtime/aio.c',
'src/runtime/app.c',
'src/runtime/asset_dependencies.c',
'src/runtime/asset_loading.c',
'src/runtime/buffer_manager.c',
'src/runtime/config.c',
'src/runtime/error_report.c',
@ -74,6 +76,7 @@ runtime_lib = library('vyrt',
'src/runtime/gfx_main.c',
'src/runtime/init.c',
'src/runtime/jobs.c',
'src/runtime/packages.c',
'src/runtime/text.c',
'src/runtime/threading_cond.c',
'src/runtime/threading_mutex.c',

2
src/runtime/asset_dependencies.c
View File

@ -25,7 +25,7 @@ static vy_uid *_list_mem;
vy_result LoadAssetDependencies(void) {
vy_dependency_file_header header;
vy_file_id fid = vyAddFile("assets/deps.bin");
vy_file_id fid = vyAddFile("data/deps.bin");
if (vySubmitSingleLoadSync((vy_file_load){.dest = &header,
.num_bytes = sizeof(header),

62
src/runtime/asset_loading.c Normal file
View File

@ -0,0 +1,62 @@
#include "assets.h"
#include "uidtab.h"
#include "aio.h"
#include "buffer_manager.h"
#define VY_DEFINE_PACKAGE_FILE_STRUCTURES
#include "packages.h"
#include "lz4/lz4.h"
VY_DLLEXPORT vy_result vyLoadAssetDirect(vy_uid uid, void **p_buffer, size_t *p_size) {
const vy_uid_data *data = vyGetUIDData(uid);
if (!data)
return VY_UNKNOWN_ASSET;
void *compressed_buffer = vyAllocBuffer(data->size);
if (!compressed_buffer) {
return VY_BUFFER_ALLOC_FAILED;
}
if (vySubmitSingleLoadSync((vy_file_load) {
.file = data->pkg_file,
.offset = data->offset,
.num_bytes = data->size,
.dest = compressed_buffer,
}) != VY_AIO_STATE_FINISHED) {
vyReleaseBuffer(compressed_buffer, data->size);
return VY_LOAD_FAILED;
}
const vy_package_asset_header *header = compressed_buffer;
size_t compressed_size = (data->size) - sizeof(*header);
XXH64_hash_t calculated_hash = XXH3_64bits((header + 1), compressed_size);
XXH64_hash_t file_hash = XXH64_hashFromCanonical(&header->checksum);
if (calculated_hash != file_hash) {
vyLog("core", "Checksum mismatch for asset %u", uid);
vyReleaseBuffer(compressed_buffer, data->size);
return VY_LOAD_FAILED;
}
size_t size = (size_t)header->decompressed_size;
void *decompressed_buffer = vyAllocBuffer(size);
if (!decompressed_buffer) {
vyReleaseBuffer(compressed_buffer, data->size);
return VY_BUFFER_ALLOC_FAILED;
}
if (LZ4_decompress_safe((const char *)(header + 1),
(char *)decompressed_buffer,
(int)compressed_size,
(int)size) < 0) {
vyReleaseBuffer(compressed_buffer, data->size);
return VY_LOAD_FAILED;
}
vyReleaseBuffer(compressed_buffer, data->size);
*p_buffer = decompressed_buffer;
*p_size = size;
return VY_SUCCESS;
}

5
src/runtime/init.c
View File

@ -36,6 +36,7 @@ extern vy_result LoadUIDTable(void);
extern void ReleaseUIDTable(void);
extern vy_result LoadAssetDependencies(void);
extern void ReleaseAssetDependencies(void);
extern vy_result LoadPackageNames(void);
VY_DLLEXPORT vy_result vyInitRuntime(void) {
SetMainThreadId();
@ -66,6 +67,10 @@ VY_DLLEXPORT vy_result vyInitRuntime(void) {
return res;
}
if ((res = LoadPackageNames()) != VY_SUCCESS) {
vyLog("CORE", "LoadPackageNames returned result: %u", res);
}
return VY_SUCCESS;
}

20
src/runtime/packages.c Normal file
View File

@ -0,0 +1,20 @@
#include <stdio.h>
#include "file_tab.h"
vy_result LoadPackageNames(void) {
FILE *f = fopen("data/packages.txt", "r");
if (!f) {
return VY_UNKNOWN_ERROR;
}
while (!feof(f)) {
char line[256];
fscanf(f, "%255s\n", line);
line[255] = '\0';
vyAddFile(line);
}
fclose(f);
return VY_SUCCESS;
}

18
src/runtime/packages.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef VY_PACKAGES_H
#define VY_PACKAGES_H
#ifdef VY_DEFINE_PACKAGE_FILE_STRUCTURES
#include <stdint.h>
#include "xxhash/xxhash.h"
#pragma pack(push, 1)
typedef struct {
XXH64_canonical_t checksum;
uint32_t decompressed_size;
} vy_package_asset_header;
#pragma pack(pop)
#endif
#endif

2
src/runtime/uidtab.c
View File

@ -21,7 +21,7 @@ vy_result LoadUIDTable(void) {
/* We use stdio here, because we cannot load any asset in parallel to this.
* This is because the uidtab is what tells us which assets exist.
*/
FILE *f = fopen("assets/uidtab.bin", "rb");
FILE *f = fopen("data/uidtab.bin", "rb");
if (!f)
return VY_LOAD_FAILED;

8
src/tools/assetc/assetc.c
View File

@ -100,7 +100,7 @@ int main(int argc, char **argv) {
return 1;
}
vyInitPackages();
vySetWorkingDirectory(g_assetc_options.root_directory);
if (vyLoadAssetMeta() != VY_SUCCESS) {
return 1;
@ -119,6 +119,12 @@ int main(int argc, char **argv) {
return 1;
}
/* Create necessary directories */
vySetWorkingDirectory(g_assetc_options.root_directory);
vyCreateDirectory("assets");
vyCreateDirectory("actemp");
vyCreateDirectory("data");
/* "Mainloop" */
vyDiscoverAssets();

10
src/tools/assetc/assetmeta.c
View File

@ -1,5 +1,6 @@
#include "assetmeta.h"
#include "utils.h"
#include "options.h"
#include "runtime/threading.h"
#include "runtime/aio.h"
@ -7,6 +8,7 @@
#include <xxhash/xxhash.h>
#include <stdio.h>
#include <string.h>
#define MAP_SIZE 2048
typedef struct {
@ -40,7 +42,7 @@ vy_result vyLoadAssetMeta(void) {
_guard = vyCreateMutex();
/* Load the meta file */
size_t fsz = vyGetFileSize("meta.bin");
size_t fsz = vyGetFileSize("actemp/meta.bin");
if (fsz == 0) {
vyLog("ASSETC", "Metadata file 'meta.bin' not found. All assets will be processed.");
return VY_SUCCESS;
@ -53,8 +55,8 @@ vy_result vyLoadAssetMeta(void) {
}
vy_load_batch load;
load.loads[0].dest = buffer;
load.loads[0].file = vyAddFile("meta.bin");
load.loads[0].dest = buffer;
load.loads[0].file = vyAddFile("actemp/meta.bin");
load.loads[0].num_bytes = fsz;
load.loads[0].offset = 0;
load.num_loads = 1;
@ -128,7 +130,7 @@ vy_result vySaveAssetMeta(void) {
XXH64_canonicalFromHash(&header.checksum, hash);
header._reserved = 0;
FILE *f = fopen("meta.bin", "wb");
FILE *f = fopen("actemp/meta.bin", "wb");
if (!f) {
vyReportError("ASSETC", "Failed to open 'meta.bin'");
vyReleaseBuffer(entries, sizeof(vy_assetmeta_entry) * header.num_entries);

4
src/tools/assetc/dependency_tracking.c
View File

@ -150,8 +150,8 @@ vy_result vySaveAssetDependencies(void) {
header.data_size += total_list_size * sizeof(vy_uid) + sizeof(vy_dependency_file_list_header);
}
}
FILE *f = fopen("deps.bin", "wb");
FILE *f = fopen("data/deps.bin", "wb");
if (!f) {
vyReportError("ASSETC", "Failed to open 'deps.bin' for writing.");
return VY_UNKNOWN_ERROR;

25
src/tools/assetc/discovery.c
View File

@ -33,6 +33,8 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
else if (memcmp(&name[name_len - 4], ".bin", 4) == 0)
return VY_SUCCESS;
}
if (strcmp(name, "packages.txt") == 0)
return VY_SUCCESS;
if (name[0] == '.') {
return VY_SUCCESS;
}
@ -68,12 +70,17 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
/* We need to load the processed data and add it to the package */
vy_uid uid = vyLookupUID(fid);
const vy_uid_data *asset_data = vyGetUIDData(uid);
if (!data) {
vyLog("ASSETC", "No data available for asset %s (%u)", data->path_scratch, uid);
if (uid == VY_INVALID_UID) {
vyLog("ASSETC", "Failed to lookup UID of known asset %s", data->path_scratch);
return VY_UNKNOWN_ERROR;
}
void *buffer;
size_t size;
if (vyLoadAssetDirect(uid, &buffer, &size) != VY_SUCCESS) {
vyLog("ASSETC", "Failed to load asset %s", data->path_scratch);
return VY_UNKNOWN_ASSET;
}
vyAddAssetToPackage(settings.package, uid, buffer, size);
}
} else {
/* Process it */
@ -109,10 +116,10 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
}
void vyDiscoverAssets(void) {
vy_discovery_data data;
data.path_scratch[0] = '.';
data.path_scratch[1] = '\0';
data.path_end = 0;
vyIterateDirectory(".", &data, DirectoryHandler);
vy_discovery_data data;
memcpy(data.path_scratch, "assets", sizeof("assets"));
data.path_end = sizeof("assets") - 1;
vyIterateDirectory("assets", &data, DirectoryHandler);
}

32
src/tools/assetc/packages.c
View File

@ -1,9 +1,11 @@
#include "packages.h"
#include "processing.h"
#define VY_DEFINE_PACKAGE_FILE_STRUCTURES
#include "runtime/threading.h"
#include "runtime/assets.h"
#include "runtime/file_tab.h"
#include "runtime/packages.h"
#include <assert.h>
#include <limits.h>
@ -40,14 +42,14 @@ unsigned int vyAddPackageFile(vy_text_span name) {
vyLockMutex(_guard);
for (unsigned int i = 0; i < _package_count; ++i) {
if (vyCompareSpanToString(name, _packages[i].name) == 0) {
if (vyCompareSpanToString(name, _packages[i].name + 5) == 0) {
vyUnlockMutex(_guard);
return i;
}
}
/* Create a new package */
_packages[_package_count].name = malloc(name.length + 1);
_packages[_package_count].name = malloc(name.length + 1 + 5);
if (!_packages[_package_count].name) {
vyReportError("ASSETC",
"Failed to allocate storage for new package %*.s",
@ -56,8 +58,9 @@ unsigned int vyAddPackageFile(vy_text_span name) {
vyUnlockMutex(_guard);
return UINT_MAX;
}
memcpy(_packages[_package_count].name, name.start, name.length);
_packages[_package_count].name[name.length] = '\0';
memcpy(_packages[_package_count].name, "data/", 5);
memcpy(_packages[_package_count].name + 5, name.start, name.length);
_packages[_package_count].name[name.length + 5] = '\0';
unsigned int index = _package_count++;
@ -119,13 +122,6 @@ void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffe
AddAssetToPackageImpl(package, uid, buffer, size, false);
}
#pragma pack(push, 1)
typedef struct {
XXH64_canonical_t checksum;
uint32_t decompressed_size;
} vy_package_asset_header;
#pragma pack(pop)
static vy_result SavePackage(vy_package *pkg) {
if (pkg->num_entries == 0) {
vyLog("ASSETC", "Package %s has no entries.", pkg->name);
@ -226,6 +222,20 @@ static vy_result SavePackage(vy_package *pkg) {
vy_result vySavePackages(void) {
assert(vyIsMainThread());
/* Save a .txt file with one line per package.
* Enables us to re-init the file-tab in future runs. */
FILE *f = fopen("data/packages.txt", "w");
if (!f) {
vyReportError("ASSETC", "Failed to write to 'packages.txt'");
return VY_UNKNOWN_ERROR;
}
for (unsigned int i = 0; i < _package_count; ++i) {
if (_packages[i].num_entries == 0)
continue;
fprintf(f, "%s\n", _packages[i].name);
}
fclose(f);
for (unsigned int i = 0; i < _package_count; ++i) {
vy_result res = SavePackage(&_packages[i]);
if (res != VY_SUCCESS)

2
src/tools/assetc/uidtable.c
View File

@ -47,7 +47,7 @@ vy_result vyWriteUIDTab(void) {
XXH64_canonicalFromHash(&header.checksum, checksum);
header.num_entries = (uint32_t)_entry_count;
FILE *f = fopen("uidtab.bin", "wb");
FILE *f = fopen("data/uidtab.bin", "wb");
if (!f) {
vyReportError("ASSETC", "Failed to open 'uidtab.bin' for writing.");
return VY_UNKNOWN_ERROR;

12
src/tools/assetc/utils.c
View File

@ -86,4 +86,16 @@ vy_result vyIterateDirectory(const char *path, void *user, vy_iterate_directory_
FindClose(h);
return res;
#endif
}
vy_result vyCreateDirectory(const char *path) {
#ifdef _WIN32
WCHAR wpath[MAX_PATH];
vy_result res = vyUTF8ToWStr(path, wpath, MAX_PATH);
if (res != VY_SUCCESS)
return res;
if (!CreateDirectoryW(wpath, NULL))
res = VY_UNKNOWN_ERROR;
return res;
#endif
}

2
src/tools/assetc/utils.h
View File

@ -22,4 +22,6 @@ typedef vy_result vy_iterate_directory_cb_func(const char *name, vyIterateDirEle
vy_result vyIterateDirectory(const char *path, void *user, vy_iterate_directory_cb_func iterate_cb);
vy_result vyCreateDirectory(const char *path);
#endif