diff --git a/.gitignore b/.gitignore index 0900d85..36b5b30 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ /subprojects/* !/subprojects/*.wrap +# assetc directories +/actemp/* +/data/* \ No newline at end of file diff --git a/assets/shader/cell.pipeline b/assets/shader/cell.pipeline index 1d9d122..79353e1 100644 --- a/assets/shader/cell.pipeline +++ b/assets/shader/cell.pipeline @@ -1,10 +1,10 @@ optimization speed; vertex { - vk shader/cell_vert.glsl; + vk assets/shader/cell_vert.glsl; } fragment { - vk shader/cell_frag.glsl; + vk assets/shader/cell_frag.glsl; } texture_bindings { diff --git a/meson.build b/meson.build index f089ae4..8f8294c 100644 --- a/meson.build +++ b/meson.build @@ -59,6 +59,7 @@ runtime_lib = library('vyrt', 'src/runtime/file_tab.h', 'src/runtime/gfx.h', 'src/runtime/jobs.h', + 'src/runtime/packages.h', 'src/runtime/renderer_api.h', 'src/runtime/runtime.h', 'src/runtime/threading.h', @@ -67,6 +68,7 @@ runtime_lib = library('vyrt', 'src/runtime/aio.c', 'src/runtime/app.c', 'src/runtime/asset_dependencies.c', + 'src/runtime/asset_loading.c', 'src/runtime/buffer_manager.c', 'src/runtime/config.c', 'src/runtime/error_report.c', @@ -74,6 +76,7 @@ runtime_lib = library('vyrt', 'src/runtime/gfx_main.c', 'src/runtime/init.c', 'src/runtime/jobs.c', + 'src/runtime/packages.c', 'src/runtime/text.c', 'src/runtime/threading_cond.c', 'src/runtime/threading_mutex.c', diff --git a/src/runtime/asset_dependencies.c b/src/runtime/asset_dependencies.c index 58bd341..91db567 100644 --- a/src/runtime/asset_dependencies.c +++ b/src/runtime/asset_dependencies.c @@ -25,7 +25,7 @@ static vy_uid *_list_mem; vy_result LoadAssetDependencies(void) { vy_dependency_file_header header; - vy_file_id fid = vyAddFile("assets/deps.bin"); + vy_file_id fid = vyAddFile("data/deps.bin"); if (vySubmitSingleLoadSync((vy_file_load){.dest = &header, .num_bytes = sizeof(header), diff --git a/src/runtime/asset_loading.c b/src/runtime/asset_loading.c new file mode 100644 index 0000000..6d2e157 --- /dev/null +++ b/src/runtime/asset_loading.c @@ -0,0 +1,62 @@ +#include "assets.h" +#include "uidtab.h" +#include "aio.h" +#include "buffer_manager.h" + +#define VY_DEFINE_PACKAGE_FILE_STRUCTURES +#include "packages.h" + +#include "lz4/lz4.h" + +VY_DLLEXPORT vy_result vyLoadAssetDirect(vy_uid uid, void **p_buffer, size_t *p_size) { + const vy_uid_data *data = vyGetUIDData(uid); + if (!data) + return VY_UNKNOWN_ASSET; + + void *compressed_buffer = vyAllocBuffer(data->size); + if (!compressed_buffer) { + return VY_BUFFER_ALLOC_FAILED; + } + + if (vySubmitSingleLoadSync((vy_file_load) { + .file = data->pkg_file, + .offset = data->offset, + .num_bytes = data->size, + .dest = compressed_buffer, + }) != VY_AIO_STATE_FINISHED) { + vyReleaseBuffer(compressed_buffer, data->size); + return VY_LOAD_FAILED; + } + + const vy_package_asset_header *header = compressed_buffer; + + size_t compressed_size = (data->size) - sizeof(*header); + XXH64_hash_t calculated_hash = XXH3_64bits((header + 1), compressed_size); + XXH64_hash_t file_hash = XXH64_hashFromCanonical(&header->checksum); + if (calculated_hash != file_hash) { + vyLog("core", "Checksum mismatch for asset %u", uid); + vyReleaseBuffer(compressed_buffer, data->size); + return VY_LOAD_FAILED; + } + + size_t size = (size_t)header->decompressed_size; + void *decompressed_buffer = vyAllocBuffer(size); + if (!decompressed_buffer) { + vyReleaseBuffer(compressed_buffer, data->size); + return VY_BUFFER_ALLOC_FAILED; + } + + if (LZ4_decompress_safe((const char *)(header + 1), + (char *)decompressed_buffer, + (int)compressed_size, + (int)size) < 0) { + vyReleaseBuffer(compressed_buffer, data->size); + return VY_LOAD_FAILED; + } + + vyReleaseBuffer(compressed_buffer, data->size); + *p_buffer = decompressed_buffer; + *p_size = size; + + return VY_SUCCESS; +} \ No newline at end of file diff --git a/src/runtime/init.c b/src/runtime/init.c index 8b170f0..c9682af 100644 --- a/src/runtime/init.c +++ b/src/runtime/init.c @@ -36,6 +36,7 @@ extern vy_result LoadUIDTable(void); extern void ReleaseUIDTable(void); extern vy_result LoadAssetDependencies(void); extern void ReleaseAssetDependencies(void); +extern vy_result LoadPackageNames(void); VY_DLLEXPORT vy_result vyInitRuntime(void) { SetMainThreadId(); @@ -66,6 +67,10 @@ VY_DLLEXPORT vy_result vyInitRuntime(void) { return res; } + if ((res = LoadPackageNames()) != VY_SUCCESS) { + vyLog("CORE", "LoadPackageNames returned result: %u", res); + } + return VY_SUCCESS; } diff --git a/src/runtime/packages.c b/src/runtime/packages.c new file mode 100644 index 0000000..6bdd087 --- /dev/null +++ b/src/runtime/packages.c @@ -0,0 +1,20 @@ +#include +#include "file_tab.h" + +vy_result LoadPackageNames(void) { + FILE *f = fopen("data/packages.txt", "r"); + if (!f) { + return VY_UNKNOWN_ERROR; + } + + while (!feof(f)) { + char line[256]; + fscanf(f, "%255s\n", line); + line[255] = '\0'; + vyAddFile(line); + } + + fclose(f); + + return VY_SUCCESS; +} \ No newline at end of file diff --git a/src/runtime/packages.h b/src/runtime/packages.h new file mode 100644 index 0000000..f504984 --- /dev/null +++ b/src/runtime/packages.h @@ -0,0 +1,18 @@ +#ifndef VY_PACKAGES_H +#define VY_PACKAGES_H + +#ifdef VY_DEFINE_PACKAGE_FILE_STRUCTURES + +#include +#include "xxhash/xxhash.h" + +#pragma pack(push, 1) +typedef struct { + XXH64_canonical_t checksum; + uint32_t decompressed_size; +} vy_package_asset_header; +#pragma pack(pop) + +#endif + +#endif diff --git a/src/runtime/uidtab.c b/src/runtime/uidtab.c index 7bc7325..a81a3b3 100644 --- a/src/runtime/uidtab.c +++ b/src/runtime/uidtab.c @@ -21,7 +21,7 @@ vy_result LoadUIDTable(void) { /* We use stdio here, because we cannot load any asset in parallel to this. * This is because the uidtab is what tells us which assets exist. */ - FILE *f = fopen("assets/uidtab.bin", "rb"); + FILE *f = fopen("data/uidtab.bin", "rb"); if (!f) return VY_LOAD_FAILED; diff --git a/src/tools/assetc/assetc.c b/src/tools/assetc/assetc.c index 7bbdc60..5f5363b 100644 --- a/src/tools/assetc/assetc.c +++ b/src/tools/assetc/assetc.c @@ -100,7 +100,7 @@ int main(int argc, char **argv) { return 1; } vyInitPackages(); - vySetWorkingDirectory(g_assetc_options.root_directory); + if (vyLoadAssetMeta() != VY_SUCCESS) { return 1; @@ -119,6 +119,12 @@ int main(int argc, char **argv) { return 1; } + /* Create necessary directories */ + vySetWorkingDirectory(g_assetc_options.root_directory); + vyCreateDirectory("assets"); + vyCreateDirectory("actemp"); + vyCreateDirectory("data"); + /* "Mainloop" */ vyDiscoverAssets(); diff --git a/src/tools/assetc/assetmeta.c b/src/tools/assetc/assetmeta.c index 1ebab13..db96fcd 100644 --- a/src/tools/assetc/assetmeta.c +++ b/src/tools/assetc/assetmeta.c @@ -1,5 +1,6 @@ #include "assetmeta.h" #include "utils.h" +#include "options.h" #include "runtime/threading.h" #include "runtime/aio.h" @@ -7,6 +8,7 @@ #include #include +#include #define MAP_SIZE 2048 typedef struct { @@ -40,7 +42,7 @@ vy_result vyLoadAssetMeta(void) { _guard = vyCreateMutex(); /* Load the meta file */ - size_t fsz = vyGetFileSize("meta.bin"); + size_t fsz = vyGetFileSize("actemp/meta.bin"); if (fsz == 0) { vyLog("ASSETC", "Metadata file 'meta.bin' not found. All assets will be processed."); return VY_SUCCESS; @@ -53,8 +55,8 @@ vy_result vyLoadAssetMeta(void) { } vy_load_batch load; - load.loads[0].dest = buffer; - load.loads[0].file = vyAddFile("meta.bin"); + load.loads[0].dest = buffer; + load.loads[0].file = vyAddFile("actemp/meta.bin"); load.loads[0].num_bytes = fsz; load.loads[0].offset = 0; load.num_loads = 1; @@ -128,7 +130,7 @@ vy_result vySaveAssetMeta(void) { XXH64_canonicalFromHash(&header.checksum, hash); header._reserved = 0; - FILE *f = fopen("meta.bin", "wb"); + FILE *f = fopen("actemp/meta.bin", "wb"); if (!f) { vyReportError("ASSETC", "Failed to open 'meta.bin'"); vyReleaseBuffer(entries, sizeof(vy_assetmeta_entry) * header.num_entries); diff --git a/src/tools/assetc/dependency_tracking.c b/src/tools/assetc/dependency_tracking.c index 5fc7e3d..3d39773 100644 --- a/src/tools/assetc/dependency_tracking.c +++ b/src/tools/assetc/dependency_tracking.c @@ -150,8 +150,8 @@ vy_result vySaveAssetDependencies(void) { header.data_size += total_list_size * sizeof(vy_uid) + sizeof(vy_dependency_file_list_header); } } - - FILE *f = fopen("deps.bin", "wb"); + + FILE *f = fopen("data/deps.bin", "wb"); if (!f) { vyReportError("ASSETC", "Failed to open 'deps.bin' for writing."); return VY_UNKNOWN_ERROR; diff --git a/src/tools/assetc/discovery.c b/src/tools/assetc/discovery.c index d3b234e..d6aa7e1 100644 --- a/src/tools/assetc/discovery.c +++ b/src/tools/assetc/discovery.c @@ -33,6 +33,8 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type else if (memcmp(&name[name_len - 4], ".bin", 4) == 0) return VY_SUCCESS; } + if (strcmp(name, "packages.txt") == 0) + return VY_SUCCESS; if (name[0] == '.') { return VY_SUCCESS; } @@ -68,12 +70,17 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type /* We need to load the processed data and add it to the package */ vy_uid uid = vyLookupUID(fid); - const vy_uid_data *asset_data = vyGetUIDData(uid); - if (!data) { - vyLog("ASSETC", "No data available for asset %s (%u)", data->path_scratch, uid); + if (uid == VY_INVALID_UID) { + vyLog("ASSETC", "Failed to lookup UID of known asset %s", data->path_scratch); return VY_UNKNOWN_ERROR; } - + void *buffer; + size_t size; + if (vyLoadAssetDirect(uid, &buffer, &size) != VY_SUCCESS) { + vyLog("ASSETC", "Failed to load asset %s", data->path_scratch); + return VY_UNKNOWN_ASSET; + } + vyAddAssetToPackage(settings.package, uid, buffer, size); } } else { /* Process it */ @@ -109,10 +116,10 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type } void vyDiscoverAssets(void) { - vy_discovery_data data; - data.path_scratch[0] = '.'; - data.path_scratch[1] = '\0'; - data.path_end = 0; - vyIterateDirectory(".", &data, DirectoryHandler); + vy_discovery_data data; + memcpy(data.path_scratch, "assets", sizeof("assets")); + data.path_end = sizeof("assets") - 1; + + vyIterateDirectory("assets", &data, DirectoryHandler); } \ No newline at end of file diff --git a/src/tools/assetc/packages.c b/src/tools/assetc/packages.c index 1aa28ac..26bbf92 100644 --- a/src/tools/assetc/packages.c +++ b/src/tools/assetc/packages.c @@ -1,9 +1,11 @@ #include "packages.h" #include "processing.h" +#define VY_DEFINE_PACKAGE_FILE_STRUCTURES #include "runtime/threading.h" #include "runtime/assets.h" #include "runtime/file_tab.h" +#include "runtime/packages.h" #include #include @@ -40,14 +42,14 @@ unsigned int vyAddPackageFile(vy_text_span name) { vyLockMutex(_guard); for (unsigned int i = 0; i < _package_count; ++i) { - if (vyCompareSpanToString(name, _packages[i].name) == 0) { + if (vyCompareSpanToString(name, _packages[i].name + 5) == 0) { vyUnlockMutex(_guard); return i; } } /* Create a new package */ - _packages[_package_count].name = malloc(name.length + 1); + _packages[_package_count].name = malloc(name.length + 1 + 5); if (!_packages[_package_count].name) { vyReportError("ASSETC", "Failed to allocate storage for new package %*.s", @@ -56,8 +58,9 @@ unsigned int vyAddPackageFile(vy_text_span name) { vyUnlockMutex(_guard); return UINT_MAX; } - memcpy(_packages[_package_count].name, name.start, name.length); - _packages[_package_count].name[name.length] = '\0'; + memcpy(_packages[_package_count].name, "data/", 5); + memcpy(_packages[_package_count].name + 5, name.start, name.length); + _packages[_package_count].name[name.length + 5] = '\0'; unsigned int index = _package_count++; @@ -119,13 +122,6 @@ void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffe AddAssetToPackageImpl(package, uid, buffer, size, false); } -#pragma pack(push, 1) -typedef struct { - XXH64_canonical_t checksum; - uint32_t decompressed_size; -} vy_package_asset_header; -#pragma pack(pop) - static vy_result SavePackage(vy_package *pkg) { if (pkg->num_entries == 0) { vyLog("ASSETC", "Package %s has no entries.", pkg->name); @@ -226,6 +222,20 @@ static vy_result SavePackage(vy_package *pkg) { vy_result vySavePackages(void) { assert(vyIsMainThread()); + /* Save a .txt file with one line per package. + * Enables us to re-init the file-tab in future runs. */ + FILE *f = fopen("data/packages.txt", "w"); + if (!f) { + vyReportError("ASSETC", "Failed to write to 'packages.txt'"); + return VY_UNKNOWN_ERROR; + } + for (unsigned int i = 0; i < _package_count; ++i) { + if (_packages[i].num_entries == 0) + continue; + fprintf(f, "%s\n", _packages[i].name); + } + fclose(f); + for (unsigned int i = 0; i < _package_count; ++i) { vy_result res = SavePackage(&_packages[i]); if (res != VY_SUCCESS) diff --git a/src/tools/assetc/uidtable.c b/src/tools/assetc/uidtable.c index 20f169b..4ac2ddb 100644 --- a/src/tools/assetc/uidtable.c +++ b/src/tools/assetc/uidtable.c @@ -47,7 +47,7 @@ vy_result vyWriteUIDTab(void) { XXH64_canonicalFromHash(&header.checksum, checksum); header.num_entries = (uint32_t)_entry_count; - FILE *f = fopen("uidtab.bin", "wb"); + FILE *f = fopen("data/uidtab.bin", "wb"); if (!f) { vyReportError("ASSETC", "Failed to open 'uidtab.bin' for writing."); return VY_UNKNOWN_ERROR; diff --git a/src/tools/assetc/utils.c b/src/tools/assetc/utils.c index d9502bb..004943e 100644 --- a/src/tools/assetc/utils.c +++ b/src/tools/assetc/utils.c @@ -86,4 +86,16 @@ vy_result vyIterateDirectory(const char *path, void *user, vy_iterate_directory_ FindClose(h); return res; #endif +} + +vy_result vyCreateDirectory(const char *path) { +#ifdef _WIN32 + WCHAR wpath[MAX_PATH]; + vy_result res = vyUTF8ToWStr(path, wpath, MAX_PATH); + if (res != VY_SUCCESS) + return res; + if (!CreateDirectoryW(wpath, NULL)) + res = VY_UNKNOWN_ERROR; + return res; +#endif } \ No newline at end of file diff --git a/src/tools/assetc/utils.h b/src/tools/assetc/utils.h index 8b3b79d..1a52920 100644 --- a/src/tools/assetc/utils.h +++ b/src/tools/assetc/utils.h @@ -22,4 +22,6 @@ typedef vy_result vy_iterate_directory_cb_func(const char *name, vyIterateDirEle vy_result vyIterateDirectory(const char *path, void *user, vy_iterate_directory_cb_func iterate_cb); +vy_result vyCreateDirectory(const char *path); + #endif