From b80d0d1bf945e37cc935942d508a08f340f84c60 Mon Sep 17 00:00:00 2001 From: Kevin Trogant Date: Sun, 4 Feb 2024 17:28:32 +0100 Subject: [PATCH] Remove unnecessary files --- src/runtime/Source.cpp | 0 src/runtime/packages.c | 20 -- src/runtime/packages.h | 18 -- src/tools/assetc/assetc.c | 149 --------- src/tools/assetc/assetmeta.c | 222 ------------- src/tools/assetc/assetmeta.h | 28 -- src/tools/assetc/assetsettings.c | 144 --------- src/tools/assetc/assetsettings.h | 21 -- src/tools/assetc/compiled.h | 13 - src/tools/assetc/dependency_tracking.c | 228 ------------- src/tools/assetc/dependency_tracking.h | 14 - src/tools/assetc/description_parser.c | 252 --------------- src/tools/assetc/description_parser.h | 55 ---- src/tools/assetc/discovery.c | 119 ------- src/tools/assetc/options.h | 28 -- src/tools/assetc/packages.c | 274 ---------------- src/tools/assetc/packages.h | 16 - src/tools/assetc/pipeline_processor.c | 423 ------------------------- src/tools/assetc/processing.h | 46 --- src/tools/assetc/processing_flags.h | 20 -- src/tools/assetc/processor.c | 381 ---------------------- src/tools/assetc/shader_processor.c | 123 ------- src/tools/assetc/uidmap.c | 0 src/tools/assetc/uidtable.c | 69 ---- src/tools/assetc/utils.c | 101 ------ src/tools/assetc/utils.h | 27 -- 26 files changed, 2791 deletions(-) delete mode 100644 src/runtime/Source.cpp delete mode 100644 src/runtime/packages.c delete mode 100644 src/runtime/packages.h delete mode 100644 src/tools/assetc/assetc.c delete mode 100644 src/tools/assetc/assetmeta.c delete mode 100644 src/tools/assetc/assetmeta.h delete mode 100644 src/tools/assetc/assetsettings.c delete mode 100644 src/tools/assetc/assetsettings.h delete mode 100644 src/tools/assetc/compiled.h delete mode 100644 src/tools/assetc/dependency_tracking.c delete mode 100644 src/tools/assetc/dependency_tracking.h delete mode 100644 src/tools/assetc/description_parser.c delete mode 100644 src/tools/assetc/description_parser.h delete mode 100644 src/tools/assetc/discovery.c delete mode 100644 src/tools/assetc/options.h delete mode 100644 src/tools/assetc/packages.c delete mode 100644 src/tools/assetc/packages.h delete mode 100644 src/tools/assetc/pipeline_processor.c delete mode 100644 src/tools/assetc/processing.h delete mode 100644 src/tools/assetc/processing_flags.h delete mode 100644 src/tools/assetc/processor.c delete mode 100644 src/tools/assetc/shader_processor.c delete mode 100644 src/tools/assetc/uidmap.c delete mode 100644 src/tools/assetc/uidtable.c delete mode 100644 src/tools/assetc/utils.c delete mode 100644 src/tools/assetc/utils.h diff --git a/src/runtime/Source.cpp b/src/runtime/Source.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/src/runtime/packages.c b/src/runtime/packages.c deleted file mode 100644 index 33332cc..0000000 --- a/src/runtime/packages.c +++ /dev/null @@ -1,20 +0,0 @@ -#include -#include "file_tab.h" - -rt_result LoadPackageNames(void) { - FILE *f = fopen("data/packages.txt", "r"); - if (!f) { - return RT_UNKNOWN_ERROR; - } - - while (!feof(f)) { - char line[256]; - fscanf(f, "%255s\n", line); - line[255] = '\0'; - rtAddFile(line); - } - - fclose(f); - - return RT_SUCCESS; -} \ No newline at end of file diff --git a/src/runtime/packages.h b/src/runtime/packages.h deleted file mode 100644 index ec5ed2f..0000000 --- a/src/runtime/packages.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef RT_PACKAGES_H -#define RT_PACKAGES_H - -#ifdef RT_DEFINE_PACKAGE_FILE_STRUCTURES - -#include -#include "xxhash/xxhash.h" - -#pragma pack(push, 1) -typedef struct { - XXH64_canonical_t checksum; - uint32_t decompressed_size; -} rt_package_asset_header; -#pragma pack(pop) - -#endif - -#endif diff --git a/src/tools/assetc/assetc.c b/src/tools/assetc/assetc.c deleted file mode 100644 index aab9605..0000000 --- a/src/tools/assetc/assetc.c +++ /dev/null @@ -1,149 +0,0 @@ -#include "assetmeta.h" -#include "processing.h" -#include "utils.h" -#include "packages.h" -#include "dependency_tracking.h" - -#define RT_ASSETC_DONT_DEFINE_OPTIONS_GLOBAL -#include "options.h" - -#include "runtime/aio.h" -#include "runtime/buffer_manager.h" -#include "runtime/uidtab.h" - -#include -#include -#include - -extern rt_result rtProcessPipelineFile(rt_file_id file, - void *buffer, - size_t size, - uint32_t flags, - rt_processor_output *output); - -extern rt_result rtProcessShaderFile(rt_file_id file, - void *buffer, - size_t size, - uint32_t flags, - rt_processor_output *output); - -extern void rtDiscoverAssets(void); - -rt_assetc_options g_assetc_options = { - .root_directory = ".", - .optimization = RT_ASSET_OPTIMIZATION_NONE, - .renderer_backend = RT_RENDERER_BACKEND_CODE_VK, -}; - -static bool ParseCommandLineArgs(int argc, char **argv) { - bool have_root_directory = false; - for (int i = 1; i < argc; ++i) { - if (i < argc - 1) { - const char *val = argv[i + 1]; - bool matched = false; - - if (strcmp(argv[i], "-r") == 0 || strcmp(argv[i], "--renderer") == 0) { - if (strcmp(val, "vk") == 0) { - g_assetc_options.renderer_backend = RT_RENDERER_BACKEND_CODE_VK; - } else { - rtReportError("ASSETC", - "Invalid render backend %s. Valid options are: vk", - val); - return false; - } - matched = true; - } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--optimization") == 0) { - if (strcmp(val, "none") == 0) { - g_assetc_options.optimization = RT_ASSET_OPTIMIZATION_NONE; - } else if (strcmp(val, "space") == 0) { - g_assetc_options.optimization = RT_ASSET_OPTIMIZATION_SPACE; - } else if (strcmp(val, "performance") == 0) { - g_assetc_options.optimization = RT_ASSET_OPTIMIZATION_PERFORMANCE; - } else { - rtReportError("ASSETC", - "Invalid optimization level %s. Valid options are: none, space, " - "performance", - val); - return false; - } - matched = true; - } else if (argv[i][0] == '-') { - rtReportError("ASSETC", "Invalid command line argument %s", argv[i]); - return false; - } - - if (matched) { - /* Skip the value */ - ++i; - continue; - } - } - - if (!have_root_directory) { - g_assetc_options.root_directory = argv[i]; - } else { - /* Maybe have secondary directories later? */ - rtReportError("ASSETC", - "More than one root directory passed as command line " - "argument."); - return false; - } - } - return true; -} - -int main(int argc, char **argv) { - rtInitRuntime(); - - /* Init assetc */ - if (!ParseCommandLineArgs(argc, argv)) { - return 1; - } - rtInitPackages(); - - - if (rtLoadAssetMeta() != RT_SUCCESS) { - return 1; - } - - if (rtInitDependencyTracking() != RT_SUCCESS) { - return 1; - } - - if (rtAddAssetProcessor(".pipeline", rtProcessPipelineFile) != RT_SUCCESS) - return 1; - if (rtAddAssetProcessor(".glsl", rtProcessShaderFile) != RT_SUCCESS) - return 1; - - if (rtStartProcessing() != RT_SUCCESS) { - return 1; - } - - /* Create necessary directories */ - rtSetWorkingDirectory(g_assetc_options.root_directory); - rtCreateDirectory("assets"); - rtCreateDirectory("actemp"); - rtCreateDirectory("data"); - - /* "Mainloop" */ - rtDiscoverAssets(); - - rtWaitUntilProcessingIsFinished(); - rtStopProcessing(); - - /* Write result */ - rtSaveAssetMeta(); - if (rtSavePackages() != RT_SUCCESS) { - return 1; - } - if (rtWriteUIDTab() != RT_SUCCESS) { - return 1; - } - if (rtSaveAssetDependencies() != RT_SUCCESS) { - return 1; - } - - rtShutdownRuntime(); - - return 0; -} \ No newline at end of file diff --git a/src/tools/assetc/assetmeta.c b/src/tools/assetc/assetmeta.c deleted file mode 100644 index c03c7c2..0000000 --- a/src/tools/assetc/assetmeta.c +++ /dev/null @@ -1,222 +0,0 @@ -#include "assetmeta.h" -#include "utils.h" -#include "options.h" - -#include "runtime/threading.h" -#include "runtime/aio.h" -#include "runtime/buffer_manager.h" - -#include -#include -#include - -#define MAP_SIZE 2048 -typedef struct { - rt_file_id fids[MAP_SIZE]; - rt_uid uids[MAP_SIZE]; - rt_assetmeta meta[MAP_SIZE]; - - unsigned int used_slots; -} rt_uid_map; - -#pragma pack(push, 1) -typedef struct { - XXH64_canonical_t checksum; - uint32_t num_entries; - uint32_t _reserved; -} rt_assetmeta_header; -#pragma pack(pop) - -#pragma pack(push, 1) -typedef struct { - rt_uid uid; - rt_file_id source_file; - rt_assetmeta meta; -} rt_assetmeta_entry; -#pragma pack(pop) - -static rt_uid_map _map; -static rt_mutex *_guard; - -rt_result rtLoadAssetMeta(void) { - _guard = rtCreateMutex(); - - /* Load the meta file */ - size_t fsz = rtGetFileSize("actemp/meta.bin"); - if (fsz == 0) { - rtLog("ASSETC", "Metadata file 'meta.bin' not found. All assets will be processed."); - return RT_SUCCESS; - } - - void *buffer = rtAllocBuffer(fsz); - if (!buffer) { - rtReportError("ASSETC", "Failed to allocate buffer for holding asset metadata."); - return 1; - } - - rt_load_batch load; - load.loads[0].dest = buffer; - load.loads[0].file = rtAddFile("actemp/meta.bin"); - load.loads[0].num_bytes = fsz; - load.loads[0].offset = 0; - load.num_loads = 1; - - rt_aio_handle handle; - if (rtSubmitLoadBatch(&load, &handle) != RT_SUCCESS) { - rtReportError("ASSETC", "Failed to submit load of 'meta.bin'."); - rtReleaseBuffer(buffer, fsz); - return 1; - } - if (rtWaitForAIOCompletion(handle) != RT_AIO_STATE_FINISHED) { - rtReportError("ASSETC", "Failed to load 'meta.bin'."); - rtReleaseBuffer(buffer, fsz); - rtReleaseAIO(handle); - return 1; - } - rtReleaseAIO(handle); - - const rt_assetmeta_header *header = buffer; - const rt_assetmeta_entry *entries = (rt_assetmeta_entry *)(header + 1); - - if ((sizeof(rt_assetmeta_entry) * header->num_entries + sizeof(*header)) > fsz) { - rtReportError("ASSETC", "Header of 'meta.bin' is corrupted: Mismatched num_entries and filesize."); - rtReleaseBuffer(buffer, fsz); - return 1; - } - XXH64_hash_t hash = XXH3_64bits(entries, sizeof(rt_assetmeta_entry) * header->num_entries); - XXH64_hash_t header_hash = XXH64_hashFromCanonical(&header->checksum); - if (hash != header_hash) { - rtReportError("ASSETC", - "Metadata file 'meta.bin' is corrupted: Wrong checksum."); - rtReleaseBuffer(buffer, fsz); - return 1; - } - - for (uint32_t i = 0; i < header->num_entries; ++i) { - /* Load here to avoid unaligned pointer */ - rt_assetmeta meta = entries[i].meta; - rtAddUIDMapping(entries[i].source_file, entries[i].uid, &meta); - } - - rtReleaseBuffer(buffer, fsz); - return 0; -} - -rt_result rtSaveAssetMeta(void) { - rt_assetmeta_header header = {0}; - - /* Count number of entries */ - for (size_t i = 0; i < MAP_SIZE; ++i) { - if (_map.fids[i] != RT_INVALID_FILE_ID) - header.num_entries += 1; - } - - rt_assetmeta_entry *entries = rtAllocBuffer(sizeof(rt_assetmeta_entry) * header.num_entries); - if (!entries) - return RT_UNKNOWN_ERROR; - - /* Store entries */ - size_t j = 0; - for (size_t i = 0; i < MAP_SIZE; ++i) { - if (_map.fids[i] != RT_INVALID_FILE_ID) { - entries[j].source_file = _map.fids[i]; - entries[j].uid = _map.uids[i]; - entries[j].meta = _map.meta[i]; - ++j; - } - } - - XXH64_hash_t hash = XXH3_64bits(entries, sizeof(rt_assetmeta_entry) * header.num_entries); - XXH64_canonicalFromHash(&header.checksum, hash); - header._reserved = 0; - - FILE *f = fopen("actemp/meta.bin", "wb"); - if (!f) { - rtReportError("ASSETC", "Failed to open 'meta.bin'"); - rtReleaseBuffer(entries, sizeof(rt_assetmeta_entry) * header.num_entries); - return RT_UNKNOWN_ERROR; - } - if (fwrite(&header, sizeof(header), 1, f) != 1) { - rtReportError("ASSETC", "Failed to write 'meta.bin'"); - rtReleaseBuffer(entries, sizeof(rt_assetmeta_entry) * header.num_entries); - return RT_UNKNOWN_ERROR; - } - if (fwrite(entries, sizeof(rt_assetmeta_entry), header.num_entries, f) != header.num_entries) { - rtReportError("ASSETC", "Failed to write 'meta.bin'"); - rtReleaseBuffer(entries, sizeof(rt_assetmeta_entry) * header.num_entries); - return RT_UNKNOWN_ERROR; - } - - fclose(f); - - return RT_SUCCESS; -} - -rt_uid rtLookupUID(rt_file_id fid) { - rtLockMutex(_guard); - unsigned int i = 0; - rt_uid result = RT_INVALID_UID; - while (i < MAP_SIZE) { - unsigned int slot = (fid + i) % MAP_SIZE; - if (_map.fids[slot] == fid) { - result = _map.uids[slot]; - break; - } else if (_map.fids[slot] == RT_INVALID_FILE_ID) { - break; - } - ++i; - } - rtUnlockMutex(_guard); - return result; -} - -bool rtGetAssetMeta(rt_file_id source_file, rt_assetmeta *meta) { - rtLockMutex(_guard); - unsigned int i = 0; - bool result = false; - while (i < MAP_SIZE) { - unsigned int slot = (source_file + i) % MAP_SIZE; - if (_map.fids[slot] == source_file) { - *meta = _map.meta[slot]; - result = true; - break; - } else if (_map.fids[slot] == RT_INVALID_FILE_ID) { - break; - } - ++i; - } - rtUnlockMutex(_guard); - return result; -} - -void rtAddUIDMapping(rt_file_id fid, rt_uid uid, const rt_assetmeta *meta) { - rtLockMutex(_guard); - float fill_rate = (float)_map.used_slots / MAP_SIZE; - if (fill_rate >= .5f) { - rtLog("ASSETC", "UID map is above 50% filled."); - } - unsigned int i = 0; - while (i < MAP_SIZE) { - unsigned int slot = (fid + i) % MAP_SIZE; - if (_map.fids[slot] == RT_INVALID_FILE_ID) { - _map.fids[slot] = fid; - _map.uids[slot] = uid; - if (meta) { - _map.meta[slot] = *meta; - } else { - _map.meta[slot].last_changed = 0; - _map.meta[slot].compiled_ts = 0; - _map.meta[slot].processing_flags = 0; - } - ++_map.used_slots; - break; - } else if (_map.fids[slot] == fid) { - break; - } - ++i; - } - if (i == MAP_SIZE) { - rtReportError("ASSETC", "Failed to insert entry into UID map."); - } - rtUnlockMutex(_guard); -} \ No newline at end of file diff --git a/src/tools/assetc/assetmeta.h b/src/tools/assetc/assetmeta.h deleted file mode 100644 index b6402f6..0000000 --- a/src/tools/assetc/assetmeta.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef RT_ASSETC_ASSETMETA_H -#define RT_ASSETC_ASSETMETA_H - -#include "runtime/file_tab.h" -#include "runtime/assets.h" - -#include - -/* Metadata for assets used only by assetc */ -typedef struct { - uint64_t last_changed; - uint64_t compiled_ts; - uint32_t processing_flags; -} rt_assetmeta; - -rt_result rtLoadAssetMeta(void); - -rt_result rtSaveAssetMeta(void); - -/* The UID map associates processed files with generated asset uids. */ -void rtAddUIDMapping(rt_file_id fid, rt_uid uid, const rt_assetmeta *meta); - -/* Returns true if the asset is found. false otherwise */ -bool rtGetAssetMeta(rt_file_id source_file, rt_assetmeta *meta); - -rt_uid rtLookupUID(rt_file_id fid); - -#endif diff --git a/src/tools/assetc/assetsettings.c b/src/tools/assetc/assetsettings.c deleted file mode 100644 index c6032a0..0000000 --- a/src/tools/assetc/assetsettings.c +++ /dev/null @@ -1,144 +0,0 @@ -#include "assetsettings.h" -#include "description_parser.h" -#include "packages.h" -#include "utils.h" - -#include "runtime/aio.h" -#include "runtime/buffer_manager.h" - -#include -#include -#include - -rt_result rtParseAssetSettings(const char *text, - size_t length, - const char *file_path, - rt_asset_settings *settings) { - unsigned int root_list; - rt_parse_state state; - rt_result res = rtParseDescription(text, length, file_path, &root_list, &state); - if (res != RT_SUCCESS) { - rtReportError("ASSETC", "Failed to parse asset settings: %s", file_path); - return res; - } - - /* Default settings */ - settings->package = 0; - settings->processing_flags = 0; - settings->reprocess_on_dependency_change = false; - - const rt_parsed_stmt *package_stmt = rtFindStatement(&state, root_list, "package"); - if (package_stmt) { - if (package_stmt->form != RT_STMT_FORM_VALUE) { - rtReportError("ASSETC", - "Expected a package name as the value of 'package' in %s.", - file_path); - res = RT_UNKNOWN_ERROR; - goto out; - } - settings->package = rtAddPackageFile(package_stmt->value); - } - - const rt_parsed_stmt *flags_stmt = rtFindStatement(&state, root_list, "processing_flags"); - if (flags_stmt) { - if (flags_stmt->form != RT_STMT_FORM_VALUE) { - rtReportError("ASSETC", - "Expected a hexadecimal number as the value of 'processing_flags' in %s.", - file_path); - res = RT_UNKNOWN_ERROR; - goto out; - } - - sscanf(flags_stmt->value.start, "%x", &settings->processing_flags); - } - - const rt_parsed_stmt *reprocess_stmt = - rtFindStatement(&state, root_list, "reprocess_on_dependency_change"); - if (reprocess_stmt) { - if (reprocess_stmt->form != RT_STMT_FORM_VALUE) { - rtReportError("ASSETC", - "Expected either 'true' or 'false' as the value of 'reprocess_on_dependency_change' in %s.", - file_path); - res = RT_UNKNOWN_ERROR; - goto out; - } - if (rtCompareSpanToString(reprocess_stmt->value, "true") == 0) - settings->reprocess_on_dependency_change = true; - else if (rtCompareSpanToString(reprocess_stmt->value, "false") == 0) - settings->reprocess_on_dependency_change = false; - else { - rtReportError("ASSETC", - "Expected either 'true' or 'false' as the value of " - "'reprocess_on_dependency_change' in %s.", - file_path); - res = RT_UNKNOWN_ERROR; - goto out; - } - } - -out: - rtReleaseParseState(&state); - return res; -} - -rt_result rtLoadAssetSettings(const char *asset_path, rt_asset_settings *settings) { - size_t path_len = strlen(asset_path); - - char *as_path = malloc(path_len + 3); - if (!as_path) { - return RT_UNKNOWN_ERROR; - } - memcpy(as_path, asset_path, path_len); - - size_t ext_len = 0; - while (ext_len < path_len) { - if (as_path[path_len - ext_len] == '.') - break; - ++ext_len; - } - - strcpy(&as_path[path_len - ext_len], ".as"); - size_t as_size = rtGetFileSize(as_path); - if (as_size == 0) { - rtReportError("ASSETC", "Failed to retrieve size of setting file %s", as_path); - free(as_path); - return RT_UNKNOWN_ERROR; - } - - void *as_buffer = rtAllocBuffer(as_size); - - rt_load_batch as_load; - as_load.loads[0].file = rtAddFile(as_path); - as_load.loads[0].num_bytes = as_size; - as_load.loads[0].dest = as_buffer; - if (!as_load.loads[0].dest) { - rtReportError("ASSETC", "Failed to allocate buffer for setting file %s", as_path); - free(as_path); - return RT_UNKNOWN_ERROR; - } - as_load.loads[0].offset = 0; - as_load.num_loads = 1; - - rt_aio_handle as_handle; - if (rtSubmitLoadBatch(&as_load, &as_handle) != RT_SUCCESS) { - rtReportError("ASSETC", "Failed to submit load of setting file %s", as_path); - free(as_path); - return RT_UNKNOWN_ERROR; - } - - if (rtWaitForAIOCompletion(as_handle) != RT_AIO_STATE_FINISHED) { - rtReportError("ASSETC", "Failed to load setting file %s", as_path); - free(as_path); - return RT_UNKNOWN_ERROR; - } - - rtReleaseAIO(as_handle); - - if (rtParseAssetSettings(as_buffer, as_size, as_path, settings) != RT_SUCCESS) { - free(as_path); - return RT_UNKNOWN_ERROR; - } - - free(as_path); - return RT_SUCCESS; -} \ No newline at end of file diff --git a/src/tools/assetc/assetsettings.h b/src/tools/assetc/assetsettings.h deleted file mode 100644 index f8cdc85..0000000 --- a/src/tools/assetc/assetsettings.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef RT_ASSETC_ASSETSETTINGS_H -#define RT_ASSETC_ASSETSETTINGS_H - -#include "runtime/runtime.h" -#include - -typedef struct { - unsigned int package; - uint32_t processing_flags; - bool reprocess_on_dependency_change; -} rt_asset_settings; - - -rt_result rtParseAssetSettings(const char *text, - size_t length, - const char *file_path, - rt_asset_settings *settings); - -rt_result rtLoadAssetSettings(const char *asset_path, rt_asset_settings *settings); - -#endif diff --git a/src/tools/assetc/compiled.h b/src/tools/assetc/compiled.h deleted file mode 100644 index e5e9d7e..0000000 --- a/src/tools/assetc/compiled.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef RT_ASSETC_COMPILED_H -#define RT_ASSETC_COMPILED_H - -#include "runtime/runtime.h" -#include "runtime/assets.h" - -rt_uid rtGetUID(const char *name); - -void rtStoreOutput(rt_uid uid, const void *data, size_t size); - -rt_result rtWriteCompiledFiles(void); - -#endif diff --git a/src/tools/assetc/dependency_tracking.c b/src/tools/assetc/dependency_tracking.c deleted file mode 100644 index d794782..0000000 --- a/src/tools/assetc/dependency_tracking.c +++ /dev/null @@ -1,228 +0,0 @@ -#include "dependency_tracking.h" - -#define RT_DEFINE_DEPENDENCY_FILE_STRUCTURES -#include "runtime/assets.h" -#include "runtime/threading.h" -#include "runtime/asset_dependencies.h" - -#include -#include -#include -#include -#include -#include - - -/* Track a list of dependencies per asset. - * For the runtime, we only care about dependent -> dependency (so for example material -> texture), - * but for assetc we also care about the opposite direction: - * If an asset has the reprocess_on_dependency_change setting enabled, - * we need to queue a reprocessing, even if the asset itself did not change. - */ - -#define END_OF_LIST 0 - -/* 64 byte cache line - 8 (next index + count) / 4 (u32) = 14 */ -#define BUCKET_ENTRY_COUNT 14 -typedef struct rt_dep_list_bucket_s { - uint32_t next; - uint32_t count; - rt_uid entries[BUCKET_ENTRY_COUNT]; -} rt_dep_list_bucket; - -typedef union { - /* Indices of the first buckets */ - struct { - uint32_t dependencies; - uint32_t dependents; - }; - uint32_t lists[2]; -} rt_dep_list; - -static rt_mutex *_guard; - -static rt_dep_list_bucket *_buckets; -static uint32_t _bucket_count; -static uint32_t _bucket_capacity; - - -#define MAP_SIZE 2048 -static rt_uid _uids[MAP_SIZE]; -static rt_dep_list _lists[MAP_SIZE]; - -rt_result rtInitDependencyTracking(void) { - _guard = rtCreateMutex(); - if (!_guard) - return RT_UNKNOWN_ERROR; - - _buckets = malloc(sizeof(rt_dep_list_bucket) * 256); - if (!_buckets) - return RT_UNKNOWN_ERROR; - _bucket_capacity = 256; - _bucket_count = 1; - - return RT_SUCCESS; -} - -static uint32_t AllocNewBucket(void) { - if (_bucket_count == _bucket_capacity) { - void *t = realloc(_buckets, (size_t)_bucket_capacity * 2 * sizeof(rt_dep_list_bucket)); - if (!t) - return 0; - _buckets = t; - _bucket_capacity *= 2; - } - memset(&_buckets[_bucket_count], 0, sizeof(rt_dep_list_bucket)); - return _bucket_count++; -} - -static rt_result InsertIntoList(rt_uid list_asset, rt_uid uid, int list_index) { - rtLockMutex(_guard); - bool inserted = false; - for (uint32_t i = 0; i < MAP_SIZE; i++) { - uint32_t at = (list_asset + i) % MAP_SIZE; - if (_uids[at] == list_asset || _uids[at] == 0) { - _uids[at] = list_asset; - - /* Alloc a new list, if necessary */ - if (_lists[at].lists[list_index] == 0) { - _lists[at].lists[list_index] = AllocNewBucket(); - if (!_lists[at].lists[list_index]) { - rtUnlockMutex(_guard); - return RT_UNKNOWN_ERROR; - } - } - - /* Advance to the end of the list */ - rt_dep_list_bucket *bucket = &_buckets[_lists[at].lists[list_index]]; - while (bucket->next != END_OF_LIST) { - bucket = &_buckets[bucket->next]; - } - - /* Grow the list, if necessary */ - if (bucket->count == BUCKET_ENTRY_COUNT) { - bucket->next = AllocNewBucket(); - if (!bucket->next) { - rtUnlockMutex(_guard); - return RT_UNKNOWN_ERROR; - } - bucket = &_buckets[bucket->next]; - } - - assert(bucket->count < BUCKET_ENTRY_COUNT); - bucket->entries[bucket->count++] = uid; - - inserted = true; - break; - } - } - rtUnlockMutex(_guard); - assert(inserted); - return RT_SUCCESS; -} - -rt_result rtAddAssetDependency(rt_uid dependent, rt_uid dependency) { - rt_result res = InsertIntoList(dependent, dependency, 0); - if (res != RT_SUCCESS) - return res; - res = InsertIntoList(dependency, dependent, 1); - return res; -} - -rt_result rtSaveAssetDependencies(void) { - assert(rtIsMainThread()); - rt_dependency_file_header header = {.num_lists = 0, .data_size = 0}; - - for (size_t i = 0; i < MAP_SIZE; ++i) { - if (_uids[i] != RT_INVALID_UID) { - if (!_lists[i].dependencies) - continue; - header.num_lists += 1; - - /* Determine the list size */ - rt_dep_list_bucket *bucket = &_buckets[_lists[i].dependencies]; - uint32_t total_list_size = bucket->count; - while (bucket->next != END_OF_LIST) { - bucket = &_buckets[bucket->next]; - total_list_size += bucket->count; - } - - header.data_size += total_list_size * sizeof(rt_uid) + sizeof(rt_dependency_file_list_header); - } - } - - FILE *f = fopen("data/deps.bin", "wb"); - if (!f) { - rtReportError("ASSETC", "Failed to open 'deps.bin' for writing."); - return RT_UNKNOWN_ERROR; - } - - - if (fwrite(&header, sizeof(header), 1, f) != 1) { - rtReportError("ASSETC", "Failed to write to 'deps.bin'."); - fclose(f); - return RT_UNKNOWN_ERROR; - } - - void *buffer = NULL; - size_t buffer_size = 0; - - for (size_t i = 0; i < MAP_SIZE; ++i) { - if (_uids[i] != RT_INVALID_UID) { - if (!_lists[i].dependencies) - continue; - - /* Determine the list size */ - rt_dep_list_bucket *bucket = &_buckets[_lists[i].dependencies]; - uint32_t total_list_size = bucket->count; - while (bucket->next != END_OF_LIST) { - bucket = &_buckets[bucket->next]; - total_list_size += bucket->count; - } - - /* Allocate */ - size_t required_size = - total_list_size * sizeof(rt_uid) + sizeof(rt_dependency_file_list_header); - if (required_size > buffer_size) { - void *t = realloc(buffer, required_size); - if (!t) { - free(buffer); - fclose(f); - return RT_UNKNOWN_ERROR; - } - buffer = t; - buffer_size = required_size; - } - - /* Fill header */ - rt_dependency_file_list_header *list_header = buffer; - rt_uid *list = (rt_uid *)(list_header + 1); - list_header->uid = _uids[i]; - list_header->num_entries = total_list_size; - - /* Copy the list */ - uint32_t at = 0; - bucket = &_buckets[_lists[i].dependencies]; - do { - memcpy(&list[at], bucket->entries, sizeof(rt_uid) * bucket->count); - at += bucket->count; - bucket = &_buckets[bucket->next]; - } while (bucket != &_buckets[END_OF_LIST]); - - XXH64_hash_t hash = XXH3_64bits(list, sizeof(rt_uid) * total_list_size); - XXH64_canonicalFromHash(&list_header->checksum, hash); - - if (fwrite(buffer, required_size, 1, f) != 1) { - rtReportError("ASSETC", "Failed to write to 'deps.bin'."); - fclose(f); - free(buffer); - return RT_UNKNOWN_ERROR; - } - } - } - - fclose(f); - free(buffer); - - return RT_SUCCESS; -} \ No newline at end of file diff --git a/src/tools/assetc/dependency_tracking.h b/src/tools/assetc/dependency_tracking.h deleted file mode 100644 index f1f5765..0000000 --- a/src/tools/assetc/dependency_tracking.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef RT_ASSETC_DEPENDENCY_TRACKING_H -#define RT_ASSETC_DEPENDENCY_TRACKING_H - -#include "runtime/runtime.h" -#include "runtime/assets.h" - - -rt_result rtInitDependencyTracking(void); - -rt_result rtAddAssetDependency(rt_uid dependent, rt_uid dependency); - -rt_result rtSaveAssetDependencies(void); - -#endif diff --git a/src/tools/assetc/description_parser.c b/src/tools/assetc/description_parser.c deleted file mode 100644 index c8eecc5..0000000 --- a/src/tools/assetc/description_parser.c +++ /dev/null @@ -1,252 +0,0 @@ -#include "description_parser.h" - -#include "runtime/runtime.h" - -#include -#include -#include - -extern int memcmp(const void *s1, const void *s2, size_t n); - -static bool IsAllowedChar(char c) { - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || - (c == '.') || (c == '_') || (c == '/'); -} - -static bool IsWhitespace(char c) { - return c == ' ' || c == '\t' || c == '\n' || c == '\r'; -} - -static void SkipWhitespace(rt_parse_state *state) { - while (state->at < state->length && IsWhitespace(state->text[state->at])) { - if (state->text[state->at] == '\n') - ++state->line; - ++state->at; - } -} - -static bool ParseAttribute(rt_parse_state *state, rt_text_span *_name) { - rt_text_span name; - name.start = &state->text[state->at]; - name.length = 0; - while (state->at < state->length && !IsWhitespace(state->text[state->at])) { - if (IsAllowedChar(state->text[state->at])) { - ++state->at; - ++name.length; - } else { - rtReportError("GFX", - "%s:%d Unexpected character %c", - state->file, - state->line, - state->text[state->at]); - return false; - } - } - *_name = name; - return true; -} - -static bool ParseValue(rt_parse_state *state, rt_text_span *_value) { - rt_text_span value; - value.start = &state->text[state->at]; - value.length = 0; - while (state->at < state->length && !IsWhitespace(state->text[state->at]) && - state->text[state->at] != ';') { - if (IsAllowedChar(state->text[state->at])) { - ++state->at; - ++value.length; - } else { - rtReportError("GFX", - "%s:%d Unexpected character %c", - state->file, - state->line, - state->text[state->at]); - return false; - } - } - *_value = value; - return true; -} - -#define BLOCK_BEGIN "BEGIN" -#define BLOCK_BEGIN_LENGTH 5 -#define BLOCK_END "END" -#define BLOCK_END_LENGTH 3 - -RT_INLINE static bool IsBlockBegin(rt_parse_state *state) { - return (state->length - state->at >= BLOCK_BEGIN_LENGTH) && - (memcmp(&state->text[state->at], BLOCK_BEGIN, BLOCK_BEGIN_LENGTH) == 0); -} - -RT_INLINE static bool IsBlockEnd(rt_parse_state *state) { - return (state->length - state->at >= BLOCK_END_LENGTH) && - (memcmp(&state->text[state->at], BLOCK_END, BLOCK_END_LENGTH) == 0); -} - -static bool ParseBlock(rt_parse_state *state, rt_text_span *p_value) { - rt_text_span value; - value.start = &state->text[state->at]; - value.length = 0; - while (state->at < state->length) { - if (state->text[state->at] == BLOCK_END[0] && IsBlockEnd(state)) { - *p_value = value; - return true; - } - ++value.length; - ++state->at; - } - /* did not find END */ - return false; -} - -static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index); - -static bool ParseStmt(rt_parse_state *state, unsigned int *stmt_index) { - rt_parsed_stmt stmt; - stmt.next = UINT_MAX; /* end of list */ - - SkipWhitespace(state); - if (!ParseAttribute(state, &stmt.attribute)) - return false; - SkipWhitespace(state); - - if (state->at == state->length) { - rtReportError("GFX", "%s:%d Expected either a value or '{'", state->file, state->line); - return false; - } - - if (state->text[state->at] == '{') { - /* Consume '{' */ - ++state->at; - - stmt.form = RT_STMT_FORM_LIST; - if (!ParseStmtList(state, &stmt.list_index)) - return false; - - /* Consume '}' */ - if (state->at < state->length && state->text[state->at] == '}') - ++state->at; - } else if (IsBlockBegin(state)) { - /* Consume BEGIN */ - state->at += BLOCK_BEGIN_LENGTH; - - stmt.form = RT_STMT_FORM_BLOCK; - if (!ParseBlock(state, &stmt.block)) - return false; - - /* Consume END */ - state->at += BLOCK_END_LENGTH; - } else { - stmt.form = RT_STMT_FORM_VALUE; - if (!ParseValue(state, &stmt.value)) - return false; - - /* Consume ';' */ - if (state->at < state->length && state->text[state->at] == ';') - ++state->at; - } - - SkipWhitespace(state); - - /* Add statement to array */ - if (state->statement_count == state->statement_capacity) { - unsigned int cap = (state->statement_capacity > 0) ? state->statement_capacity * 2 : 64; - rt_parsed_stmt *temp = realloc(state->statements, sizeof(rt_parsed_stmt) * cap); - if (!temp) { - rtReportError("GFX", "While parsing %s: Out of memory\n", state->file); - return false; - } - state->statements = temp; - state->statement_capacity = cap; - } - state->statements[state->statement_count] = stmt; - *stmt_index = state->statement_count++; - - return true; -} - -static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index) { - rt_parsed_stmt_list list; - list.first = UINT_MAX; - list.count = 0; - - unsigned int last = UINT_MAX; - - while (state->at < state->length && state->text[state->at] != '}') { - unsigned int stmt; - if (!ParseStmt(state, &stmt)) - return false; - - if (last != UINT_MAX) - state->statements[last].next = stmt; - else - list.first = stmt; - last = stmt; - - ++list.count; - } - - /* Add list to array */ - if (state->statement_list_count == state->statement_list_capacity) { - unsigned int cap = - (state->statement_list_capacity > 0) ? state->statement_list_capacity * 2 : 64; - rt_parsed_stmt_list *temp = - realloc(state->statement_lists, sizeof(rt_parsed_stmt_list) * cap); - if (!temp) { - rtReportError("GFX", "While parsing %s: Out of memory\n", state->file); - return false; - } - state->statement_lists = temp; - state->statement_list_capacity = cap; - } - state->statement_lists[state->statement_list_count] = list; - *list_index = state->statement_list_count++; - return true; -} - -const rt_parsed_stmt * rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute) { - if (list_index >= state->statement_list_count) - return NULL; - const rt_parsed_stmt_list *list = &state->statement_lists[list_index]; - unsigned int stmt_index = list->first; - for (unsigned int i = 0; i < list->count; ++i) { - const rt_parsed_stmt *stmt = &state->statements[stmt_index]; - if (rtCompareSpanToString(stmt->attribute, attribute) == 0) - return stmt; - stmt_index = stmt->next; - } - return NULL; -} - - -rt_result rtParseDescription(const char *text, - size_t length, - const char *file_path, - unsigned int *_root_list, - rt_parse_state *_state) { - - rt_parse_state state = {.text = text, - .at = 0, - .length = length, - .line = 1, - .file = file_path, - .statements = NULL, - .statement_lists = NULL, - .statement_capacity = 0, - .statement_list_capacity = 0}; - - unsigned int root_list = 0; - if (!ParseStmtList(&state, &root_list)) { - return 1; - } - *_root_list = root_list; - *_state = state; - - return RT_SUCCESS; -} - - -void rtReleaseParseState(rt_parse_state *state) { - free(state->statements); - free(state->statement_lists); -} \ No newline at end of file diff --git a/src/tools/assetc/description_parser.h b/src/tools/assetc/description_parser.h deleted file mode 100644 index dffcfe8..0000000 --- a/src/tools/assetc/description_parser.h +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef RT_ASSETC_DESCRIPTION_PARSER_H -#define RT_ASSETC_DESCRIPTION_PARSER_H - -#include "runtime/runtime.h" - -typedef enum { - RT_STMT_FORM_VALUE, - RT_STMT_FORM_LIST, - RT_STMT_FORM_BLOCK, -} rt_stmt_form; - -typedef struct { - unsigned int first; - unsigned int count; -} rt_parsed_stmt_list; - -typedef struct { - rt_stmt_form form; - rt_text_span attribute; - union { - rt_text_span value; - rt_text_span block; - unsigned int list_index; - }; - /* For lists */ - unsigned int next; -} rt_parsed_stmt; - -typedef struct { - const char *file; - const char *text; - size_t at; - size_t length; - int line; - - rt_parsed_stmt *statements; - unsigned int statement_count; - unsigned int statement_capacity; - - rt_parsed_stmt_list *statement_lists; - unsigned int statement_list_count; - unsigned int statement_list_capacity; -} rt_parse_state; - -rt_result rtParseDescription(const char *text, - size_t length, - const char *file_path, - unsigned int *root_list, - rt_parse_state *state); - -const rt_parsed_stmt *rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute); - -void rtReleaseParseState(rt_parse_state *state); - -#endif diff --git a/src/tools/assetc/discovery.c b/src/tools/assetc/discovery.c deleted file mode 100644 index 8437d97..0000000 --- a/src/tools/assetc/discovery.c +++ /dev/null @@ -1,119 +0,0 @@ -#include "utils.h" -#include "assetmeta.h" -#include "processing.h" -#include "assetsettings.h" -#include "packages.h" - -#include - -#include "runtime/uidtab.h" - -typedef struct { - char path_scratch[1024]; - unsigned int path_end; -} rt_discovery_data; - -static rt_result LoadCompressedAsset(rt_uid uid, void **buffer, size_t size) { -} - -static rt_result DirectoryHandler(const char *name, rtIterateDirElementType type, void *user) { - rt_discovery_data *data = user; - - size_t name_len = strlen(name); - - if (type == RT_DIR_ELEMENT_TYPE_FILE) { - /* Skip files we don't want to process */ - if (name_len >= 3) { - if (memcmp(&name[name_len - 3], ".as", 3) == 0) - return RT_SUCCESS; - } - if (name_len >= 4) { - if (memcmp(&name[name_len - 4], ".pkg", 4) == 0) - return RT_SUCCESS; - else if (memcmp(&name[name_len - 4], ".bin", 4) == 0) - return RT_SUCCESS; - } - if (strcmp(name, "packages.txt") == 0) - return RT_SUCCESS; - if (name[0] == '.') { - return RT_SUCCESS; - } - - /* Check if we know that file */ - if (data->path_end > 0) { - data->path_scratch[data->path_end] = '/'; - memcpy(data->path_scratch + data->path_end + 1, name, name_len); - data->path_scratch[data->path_end + 1 + name_len] = '\0'; - } else { - memcpy(data->path_scratch, name, name_len); - data->path_scratch[name_len] = '\0'; - } - - rt_file_id fid = rtAddFile(data->path_scratch); - if (rtLookupUID(fid) != RT_INVALID_UID) { - rt_assetmeta meta = {0}; - if (!rtGetAssetMeta(fid, &meta) || (meta.last_changed >= meta.compiled_ts)) { - /* The file (may have) changed */ - rt_result res = rtAddFileToProcessingQueue(fid, meta.processing_flags); - if (res != RT_SUCCESS) - return res; - } - else { - /* The file is unchanged, we just need to add it to the output again. */ - rtLog("ASSETC", "File %s is unchanged.", data->path_scratch); - - rt_asset_settings settings = {0}; - if (rtLoadAssetSettings(data->path_scratch, &settings) != RT_SUCCESS) { - rtLog("ASSETC", "Failed to load settings for %s", data->path_scratch); - return RT_UNKNOWN_ERROR; - } - - /* We need to load the processed data and add it to the package */ - rt_uid uid = rtLookupUID(fid); - if (uid == RT_INVALID_UID) { - rtLog("ASSETC", "Failed to lookup UID of known asset %s", data->path_scratch); - return RT_UNKNOWN_ERROR; - } - rtAddUnchangedAssetToPackage(settings.package, uid); - } - } else { - /* Process it */ - rt_asset_settings settings = {0}; - if (rtLoadAssetSettings(data->path_scratch, &settings) != RT_SUCCESS) { - rtLog("ASSETC", "Failed to load settings for %s", data->path_scratch); - } - - rt_result res = rtAddFileToProcessingQueue(fid, settings.processing_flags); - if (res != RT_SUCCESS) - return res; - } - } else if (type == RT_DIR_ELEMENT_TYPE_DIRECTORY) { - if (strcmp(name, ".") == 0) - return RT_SUCCESS; - if (strcmp(name, "..") == 0) - return RT_SUCCESS; - - unsigned int path_end_before = data->path_end; - if (data->path_end > 0) - data->path_scratch[data->path_end++] = '/'; - memcpy(data->path_scratch + data->path_end, name, name_len); - data->path_scratch[data->path_end + name_len] = '\0'; - data->path_end += (unsigned int)name_len; - - rt_result res = rtIterateDirectory(data->path_scratch, user, DirectoryHandler); - if (res != RT_SUCCESS) - return res; - - data->path_end = path_end_before; - } - return RT_SUCCESS; -} - -void rtDiscoverAssets(void) { - - rt_discovery_data data; - memcpy(data.path_scratch, "assets", sizeof("assets")); - data.path_end = sizeof("assets") - 1; - - rtIterateDirectory("assets", &data, DirectoryHandler); -} \ No newline at end of file diff --git a/src/tools/assetc/options.h b/src/tools/assetc/options.h deleted file mode 100644 index 1d6f3a6..0000000 --- a/src/tools/assetc/options.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef RT_ASSETC_OPTIONS_H -#define RT_ASSETC_OPTIONS_H - -#include "runtime/assets.h" - -typedef enum { - /* No optimization */ - RT_ASSET_OPTIMIZATION_NONE, - - /* Create small assets */ - RT_ASSET_OPTIMIZATION_SPACE, - - /* Create assets for fast execution */ - RT_ASSET_OPTIMIZATION_PERFORMANCE, -} rt_asset_optimization_level; - -/* Options parsed from command line arguments */ -typedef struct { - const char *root_directory; - rt_renderer_backend_code renderer_backend; - rt_asset_optimization_level optimization; -} rt_assetc_options; - -#ifndef RT_ASSETC_DONT_DEFINE_OPTIONS_GLOBAL -extern rt_assetc_options g_assetc_options; -#endif - -#endif diff --git a/src/tools/assetc/packages.c b/src/tools/assetc/packages.c deleted file mode 100644 index 75fb5b0..0000000 --- a/src/tools/assetc/packages.c +++ /dev/null @@ -1,274 +0,0 @@ -#include "packages.h" -#include "processing.h" -#include "utils.h" - -#define RT_DEFINE_PACKAGE_FILE_STRUCTURES -#include "runtime/threading.h" -#include "runtime/assets.h" -#include "runtime/file_tab.h" -#include "runtime/packages.h" - -#include -#include -#include -#include -#include -#include - -#include "xxhash/xxhash.h" -#include "lz4/lz4.h" - -typedef struct { - rt_uid uid; - size_t disk_size; -} rt_package_entry; - -typedef struct { - char *name; - unsigned int num_entries; - unsigned int entry_capacity; - rt_package_entry *entries; -} rt_package; - -#define MAX_PACKAGES 1024 - -rt_package _packages[MAX_PACKAGES]; -unsigned int _package_count = 0; - -rt_mutex *_guard; - -unsigned int rtAddPackageFile(rt_text_span name) { - rtLockMutex(_guard); - - for (unsigned int i = 0; i < _package_count; ++i) { - if (rtCompareSpanToString(name, _packages[i].name + 5) == 0) { - rtUnlockMutex(_guard); - return i; - } - } - - /* Create a new package */ - _packages[_package_count].name = malloc(name.length + 1 + 5); - if (!_packages[_package_count].name) { - rtReportError("ASSETC", - "Failed to allocate storage for new package %*.s", - name.length, - name.start); - rtUnlockMutex(_guard); - return UINT_MAX; - } - memcpy(_packages[_package_count].name, "data/", 5); - memcpy(_packages[_package_count].name + 5, name.start, name.length); - _packages[_package_count].name[name.length + 5] = '\0'; - - unsigned int index = _package_count++; - - rtUnlockMutex(_guard); - return index; -} - -void rtInitPackages(void) { - _guard = rtCreateMutex(); - /* Create the default package (0) */ - rtAddPackageFile((rt_text_span){.start = "default.pkg", .length = 12}); -} - -static void AddAssetToPackageImpl(unsigned int package, - rt_uid uid, - void *buffer, - size_t size, - bool needs_compression) { - rtLockMutex(_guard); - if (package >= _package_count) { - rtReportError("ASSETC", "Trying to add an asset to a non-existing package."); - rtUnlockMutex(_guard); - return; - } - - rt_package *pkg = &_packages[package]; - - for (unsigned int i = 0; i < pkg->num_entries; ++i) { - if (pkg->entries[i].uid == uid) { - rtUnlockMutex(_guard); - return; - } - } - - if (pkg->num_entries == pkg->entry_capacity) { - unsigned int new_cap = (pkg->entry_capacity > 0) ? 2 * pkg->entry_capacity : 256; - rt_package_entry *n = realloc(pkg->entries, new_cap * sizeof(rt_package_entry)); - if (!n) { - rtReportError("ASSETC", "Failed to grow storage for package %u.", package); - return; - } - pkg->entry_capacity = new_cap; - pkg->entries = n; - } - - char tmp_path[256]; - snprintf(tmp_path, 256, "actemp/%u.bin", uid); - if (needs_compression) { - FILE *tmp_f = fopen(tmp_path, "wb"); - if (!tmp_f) { - rtUnlockMutex(_guard); - return; - } - - int required_size = LZ4_compressBound((int)size); - void *compressed_buffer = malloc(required_size); - if (!compressed_buffer) { - fclose(tmp_f); - rtUnlockMutex(_guard); - return; - } - - int compressed_bytes = - LZ4_compress_default(buffer, compressed_buffer, (int)size, required_size); - if (compressed_bytes == 0) { - free(compressed_buffer); - fclose(tmp_f); - rtReportError("ASSETC", "Failed to compress asset %x of package %s", uid, pkg->name); - return; - } - - rt_package_asset_header header; - XXH64_hash_t checksum = XXH3_64bits_withSeed(buffer, (size_t)compressed_bytes, 0); - XXH64_canonicalFromHash(&header.checksum, checksum); - header.decompressed_size = (uint32_t)size; - - if (fwrite(&header, sizeof(header), 1, tmp_f) != 1) { - rtReportError("ASSETC", "Failed to write to actemp/%u.bin", uid); - free(compressed_buffer); - fclose(tmp_f); - return; - } - if (fwrite(buffer, compressed_bytes, 1, tmp_f) != 1) { - rtReportError("ASSETC", "Failed to write to actemp/%u.bin", uid); - free(compressed_buffer); - fclose(tmp_f); - return; - } - fclose(tmp_f); - - pkg->entries[pkg->num_entries].disk_size = - (size_t)compressed_bytes + sizeof(rt_package_asset_header); - } else { - pkg->entries[pkg->num_entries].disk_size = rtGetFileSize(tmp_path); - if (pkg->entries[pkg->num_entries].disk_size == 0) { - rtReportError("ASSETC", "Failed to determine size of actemp/%u.bin", uid); - } - } - - pkg->entries[pkg->num_entries].uid = uid; - ++pkg->num_entries; - - - rtUnlockMutex(_guard); -} - -void rtAddAssetToPackage(unsigned int package, rt_uid uid, void *buffer, size_t size) { - AddAssetToPackageImpl(package, uid, buffer, size, true); -} - -void rtAddUnchangedAssetToPackage(unsigned int package, rt_uid uid) { - AddAssetToPackageImpl(package, uid, NULL, 0, false); -} - -static rt_result SavePackage(rt_package *pkg) { - if (pkg->num_entries == 0) { - rtLog("ASSETC", "Package %s has no entries.", pkg->name); - return RT_SUCCESS; - } - - size_t current_buffer_size = 0; - void *buffer = NULL; - - size_t offset_in_file = 0; - - rt_file_id package_fid = rtAddFile(pkg->name); - - FILE *f = fopen(pkg->name, "wb"); - if (!f) { - rtReportError("ASSETC", "Failed to open %s for writing.", pkg->name); - return RT_UNKNOWN_ERROR; - } - - for (unsigned int i = 0; i < pkg->num_entries; ++i) { - char tmp_path[256]; - snprintf(tmp_path, 256, "actemp/%u.bin", pkg->entries[i].uid); - FILE *tmp_f = fopen(tmp_path, "rb"); - if (!tmp_f) { - rtReportError("ASSETC", "Failed to open %s for reading.", tmp_path); - fclose(f); - free(buffer); - return RT_UNKNOWN_ERROR; - } - - if (current_buffer_size < pkg->entries[i].disk_size) { - void *tmp = realloc(buffer, pkg->entries[i].disk_size); - if (!tmp) { - rtReportError("ASSETC", "Failed to allocate buffer (%zu bytes) for reading %s.", pkg->entries[i].disk_size, tmp_path); - fclose(f); - fclose(tmp_f); - free(buffer); - return RT_UNKNOWN_ERROR; - } - buffer = tmp; - current_buffer_size = pkg->entries[i].disk_size; - } - - if (fread(buffer, pkg->entries[i].disk_size, 1, tmp_f) != 1) { - rtReportError("ASSETC", "Failed to read %s.", tmp_path); - fclose(f); - fclose(tmp_f); - free(buffer); - return RT_UNKNOWN_ERROR; - } - - if (fwrite(buffer, pkg->entries[i].disk_size, 1, f) != 1) { - rtReportError("ASSETC", "Failed to write (%zu bytes) to %s.", pkg->entries[i].disk_size, pkg->name); - fclose(f); - fclose(tmp_f); - free(buffer); - return RT_UNKNOWN_ERROR; - } - - rtAddUIDTabEntry(package_fid, - pkg->entries[i].uid, - offset_in_file, - pkg->entries[i].disk_size); - offset_in_file += pkg->entries[i].disk_size; - - fclose(tmp_f); - } - - free(buffer); - fclose(f); - - return RT_SUCCESS; -} - -rt_result rtSavePackages(void) { - assert(rtIsMainThread()); - - /* Save a .txt file with one line per package. - * Enables us to re-init the file-tab in future runs. */ - FILE *f = fopen("data/packages.txt", "w"); - if (!f) { - rtReportError("ASSETC", "Failed to write to 'packages.txt'"); - return RT_UNKNOWN_ERROR; - } - for (unsigned int i = 0; i < _package_count; ++i) { - if (_packages[i].num_entries == 0) - continue; - fprintf(f, "%s\n", _packages[i].name); - } - fclose(f); - - for (unsigned int i = 0; i < _package_count; ++i) { - rt_result res = SavePackage(&_packages[i]); - if (res != RT_SUCCESS) - return res; - } - return RT_SUCCESS; -} \ No newline at end of file diff --git a/src/tools/assetc/packages.h b/src/tools/assetc/packages.h deleted file mode 100644 index 0f67488..0000000 --- a/src/tools/assetc/packages.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef RT_ASSETC_PACKAGES_H -#define RT_ASSETC_PACKAGES_H - -#include "runtime/runtime.h" -#include "runtime/assets.h" - -void rtInitPackages(void); - -unsigned int rtAddPackageFile(rt_text_span name); - -void rtAddAssetToPackage(unsigned int package, rt_uid uid, void *buffer, size_t size); -void rtAddUnchangedAssetToPackage(unsigned int package, rt_uid uid); - -rt_result rtSavePackages(void); - -#endif diff --git a/src/tools/assetc/pipeline_processor.c b/src/tools/assetc/pipeline_processor.c deleted file mode 100644 index 2933420..0000000 --- a/src/tools/assetc/pipeline_processor.c +++ /dev/null @@ -1,423 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "runtime/aio.h" -#include "runtime/gfx.h" -#include "runtime/handles.h" -#include "runtime/runtime.h" -#include "runtime/buffer_manager.h" -#include "runtime/renderer_api.h" - -#include "assetmeta.h" -#include "description_parser.h" -#include "options.h" -#include "processing.h" -#include "utils.h" -#include "processing_flags.h" -#include "dependency_tracking.h" - -typedef struct { - rt_attribute_binding *uniform_bindings; - rt_attribute_binding *storage_bindings; - rt_attribute_binding *texture_bindings; - - rt_uid vertex_shader; - rt_uid fragment_shader; - rt_uid compute_shader; - - /* TODO Fixed function settings */ - - /* Sampler settings */ - - uint16_t uniform_binding_count; - uint16_t storage_binding_count; - uint16_t texture_binding_count; -} rt_parsed_pipeline_data; - -static void -DbgPrintShaderFile(const rt_parse_state *state, unsigned int list_index, unsigned int indent) { - assert(list_index < state->statement_list_count); - const rt_parsed_stmt_list *list = &state->statement_lists[list_index]; - - unsigned int stmt_index = list->first; - for (unsigned int i = 0; i < list->count; ++i) { - const rt_parsed_stmt *stmt = &state->statements[stmt_index]; - for (unsigned int j = 0; j < indent; ++j) - printf(" "); - printf("%.*s: ", stmt->attribute.length, stmt->attribute.start); - if (stmt->form == RT_STMT_FORM_VALUE) { - printf("%.*s\n", stmt->value.length, stmt->value.start); - } else { - printf("{\n"); - DbgPrintShaderFile(state, stmt->list_index, indent + 2); - printf("}\n"); - } - stmt_index = stmt->next; - } - assert(stmt_index == UINT_MAX || stmt_index == 0); -} - -static bool ParseBindingIndex(rt_text_span span, unsigned int *index) { - if (span.length == 0) - return false; - int at = (int)span.length - 1; - unsigned int exp = 1; - unsigned int n = 0; - while (at >= 0) { - if (span.start[at] >= '0' && span.start[at] <= '9') { - unsigned int digit = (unsigned int)(span.start[at] - '0'); - n += digit * exp; - } else { - rtReportError("GFX", "Unexpected non-digit character in binding index"); - return false; - } - --at; - exp *= 10; - } - *index = n; - return true; -} - -static rt_attribute_value ParseBindingValue(rt_text_span span) { - if (rtCompareSpanToString(span, "MATERIAL_ALBEDO") == 0) { - return RT_ATTRIBUTE_VALUE_MATERIAL_ALBEDO; - } else if (rtCompareSpanToString(span, "MATERIAL_NORMAL") == 0) { - return RT_ATTRIBUTE_VALUE_MATERIAL_NORMAL; - } - rtReportError("GFX", "Unsupported binding value %*.s", span.length, span.start); - return RT_ATTRIBUTE_VALUE_UNDEFINED; -} - -static bool ParseBindings(rt_parse_state *state, - unsigned int root_list, - const char *name, - const char *file_path, - rt_attribute_binding **p_bindings, - uint16_t *p_binding_count) { - const rt_parsed_stmt *bindings = rtFindStatement(state, root_list, name); - if (bindings) { - if (bindings->form != RT_STMT_FORM_LIST) { - rtReportError("GFX", - "Expected list of bindings as the value of " - "\"%s\" in %s", - name, - file_path); - return false; - } - const rt_parsed_stmt_list *binding_list = &state->statement_lists[bindings->list_index]; - rt_attribute_binding *shader_bindings = - rtAllocBuffer(sizeof(rt_attribute_binding) * binding_list->count); - if (!bindings) { - rtReportError("GFX", "Out of memory"); - return false; - } - unsigned int binding_count = binding_list->count; - - unsigned int stmt_index = binding_list->first; - for (unsigned int i = 0; i < binding_list->count; ++i) { - const rt_parsed_stmt *stmt = &state->statements[stmt_index]; - if (!ParseBindingIndex(stmt->attribute, &shader_bindings[i].index)) { - rtReleaseBuffer(shader_bindings, - sizeof(rt_attribute_binding) * binding_list->count); - return false; - } - - shader_bindings[i].value = ParseBindingValue(stmt->value); - if (shader_bindings[i].value == RT_ATTRIBUTE_VALUE_UNDEFINED) { - rtReleaseBuffer(shader_bindings, - sizeof(rt_attribute_binding) * binding_list->count); - return false; - } - stmt_index = stmt->next; - } - - *p_bindings = shader_bindings; - *p_binding_count = (uint16_t)binding_count; - return true; - } else { - *p_bindings = NULL; - *p_binding_count = 0; - return true; - } -} - -static rt_result ParseShader(rt_parse_state *state, - unsigned int root_list, - const char *name, - const char *file_path, - uint32_t processing_flags, - rt_uid *p_shader_uid) { - const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, name); - if (stmt) { - if (stmt->form != RT_STMT_FORM_LIST) { - rtReportError("GFX", - "Expected a list as the value of " - "\"%s\" in %s", - name, - file_path); - return RT_PROCESSING_FAILED; - } - const rt_parsed_stmt_list *shader_list = &state->statement_lists[stmt->list_index]; - - unsigned int stmt_index = shader_list->first; - for (unsigned int i = 0; i < shader_list->count; ++i) { - const rt_parsed_stmt *shader = &state->statements[stmt_index]; - if (shader->form != RT_STMT_FORM_VALUE) { - rtReportError("GFX", - "Expected a list as the value of " - "\"%s.%*s\" in %s", - name, - (int)shader->attribute.length, - shader->attribute.start, - file_path); - return RT_PROCESSING_FAILED; - } - rt_renderer_backend_code backend = RT_INVALID_RENDERER_BACKEND_CODE; - if (rtCompareSpanToString(shader->attribute, "vk") == 0) { - backend = RT_RENDERER_BACKEND_CODE_VK; - } else { - rtReportError("GFX", - "Invalid renderer backend" - "\"%*s\" in %s of file %s", - (int)shader->attribute.length, - shader->attribute.start, - name, - file_path); - return RT_PROCESSING_FAILED; - } - - if (backend == g_assetc_options.renderer_backend) { - rt_file_id shader_file = rtAddFileFromSpan(shader->value); - rt_uid uid = rtLookupUID(shader_file); - if (uid == RT_INVALID_UID) { - /* Add the shader file to processing and wait until its done - */ - if (rtAddFileToProcessingQueue(shader_file, processing_flags) != RT_SUCCESS) - return RT_PROCESSING_FAILED; - return RT_PROCESSING_TRY_AGAIN; - } - *p_shader_uid = uid; - - /* Don't break, because that validates the file. (attributes are checked for invalid - * values). */ - } - stmt_index = shader->next; - } - return RT_SUCCESS; - } - return RT_PROCESSING_FAILED; -} - -static uint32_t -ParseOptimizationLevel(rt_parse_state *state, unsigned int root_list, const char *file_path) { - uint32_t optimization_level; - switch (g_assetc_options.optimization) { - case RT_ASSET_OPTIMIZATION_PERFORMANCE: - optimization_level = RT_SHADER_FLAG_OPTIMIZE_SPEED; - break; - case RT_ASSET_OPTIMIZATION_SPACE: - optimization_level = RT_SHADER_FLAG_OPTIMIZE_SIZE; - break; - default: - optimization_level = 0; - } - - const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, "optimization"); - - if (stmt) { - if (stmt->form != RT_STMT_FORM_VALUE) { - rtReportError("GFX", - "Expected a simple statement for" - "\"optimization\" in %s", - file_path); - return optimization_level; - } - - if (rtCompareSpanToString(stmt->value, "speed") == 0) { - optimization_level = RT_SHADER_FLAG_OPTIMIZE_SPEED; - } else if (rtCompareSpanToString(stmt->value, "size") == 0) { - optimization_level = RT_SHADER_FLAG_OPTIMIZE_SIZE; - } else if (rtCompareSpanToString(stmt->value, "none") == 0) { - optimization_level = 0; - } else { - rtReportError("GFX", - "Expected one of 'speed', 'size' and 'none' for \"optimization\" in %s", - file_path); - } - } - - return optimization_level; -} - -static rt_result -ParsePipelineFile(rt_file_id fid, const char *text, size_t length, rt_parsed_pipeline_data *pipeline) { - /* This is the grammar for pipeline files: - * ::= * - * ::= ( ( ';' ) | ( '{' '}' ) ) - * ::= [:alnum:]* - * :: = [:alnum:]* */ - const char *file_path = rtGetFilePath(fid); - rt_parse_state state; - unsigned int root_list; - rt_result result = rtParseDescription(text, length, file_path, &root_list, &state); - if (result != RT_SUCCESS) { - goto out; - } - - /* We allow the pipeline file to overwrite the optimization level */ - uint32_t optimization = ParseOptimizationLevel(&state, root_list, file_path); - - /* Process shader stages */ - if (ParseShader(&state, - root_list, - "vertex", - file_path, - RT_SHADER_FLAG_VERTEX | optimization, - &pipeline->vertex_shader) == RT_PROCESSING_TRY_AGAIN) { - result = RT_PROCESSING_TRY_AGAIN; - goto out; - } - if (ParseShader(&state, - root_list, - "fragment", - file_path, - RT_SHADER_FLAG_FRAGMENT | optimization, - &pipeline->fragment_shader) == RT_PROCESSING_TRY_AGAIN) { - result = RT_PROCESSING_TRY_AGAIN; - goto out; - } - if (ParseShader(&state, - root_list, - "compute", - file_path, - RT_SHADER_FLAG_COMPUTE | optimization, - &pipeline->compute_shader) == RT_PROCESSING_TRY_AGAIN) { - result = RT_PROCESSING_TRY_AGAIN; - goto out; - } - - /* Process bindings */ - pipeline->texture_bindings = NULL; - pipeline->texture_binding_count = 0; - pipeline->uniform_bindings = NULL; - pipeline->uniform_binding_count = 0; - pipeline->storage_bindings = NULL; - pipeline->storage_binding_count = 0; - - if (!ParseBindings(&state, - root_list, - "texture_bindings", - file_path, - &pipeline->texture_bindings, - &pipeline->texture_binding_count)) { - result = RT_AIO_STATE_FAILED; - goto out; - } - - if (!ParseBindings(&state, - root_list, - "uniform_bindings", - file_path, - &pipeline->uniform_bindings, - &pipeline->uniform_binding_count)) { - result = RT_AIO_STATE_FAILED; - goto out; - } - - if (!ParseBindings(&state, - root_list, - "storage_bindings", - file_path, - &pipeline->storage_bindings, - &pipeline->storage_binding_count)) { - result = RT_AIO_STATE_FAILED; - goto out; - } - -out: - free(state.statements); - free(state.statement_lists); - return result; -} - -rt_result rtProcessPipelineFile(rt_file_id file, - void *buffer, - size_t size, - uint32_t flags, - rt_processor_output *output) { - RT_UNUSED(flags); - rt_parsed_pipeline_data tmp; - memset(&tmp, 0, sizeof(tmp)); - - rt_result result = ParsePipelineFile(file, buffer, size, &tmp); - if (result == RT_SUCCESS) { - /* parsed_pipeline_data contains arrays of bindings. - * We need to convert these to a flat buffer that can be written to a file */ - - size_t outbuffer_size = - sizeof(rt_pipeline_info) + - sizeof(rt_attribute_binding) * - (tmp.storage_binding_count + tmp.texture_binding_count + tmp.uniform_binding_count); - void *out_buffer = rtAllocBuffer(outbuffer_size); - if (!buffer) { - return RT_PROCESSING_FAILED; - } - rt_pipeline_info *info = out_buffer; - info->vertex_shader = tmp.vertex_shader; - info->fragment_shader = tmp.fragment_shader; - info->compute_shader = tmp.compute_shader; - info->texture_binding_count = tmp.texture_binding_count; - info->uniform_binding_count = tmp.uniform_binding_count; - info->storage_binding_count = tmp.storage_binding_count; - - rt_attribute_binding *texture_bindings = NULL; - if (tmp.texture_binding_count > 0) { - texture_bindings = (rt_attribute_binding *)(info + 1); - memcpy(texture_bindings, - tmp.texture_bindings, - sizeof(rt_attribute_binding) * tmp.texture_binding_count); - } - rt_attribute_binding *uniform_bindings = NULL; - if (tmp.uniform_binding_count > 0) { - uniform_bindings = texture_bindings + tmp.texture_binding_count; - memcpy(uniform_bindings, - tmp.uniform_bindings, - sizeof(rt_attribute_binding) * tmp.uniform_binding_count); - } - rt_attribute_binding *storage_bindings = NULL; - if (tmp.storage_binding_count > 0) { - storage_bindings = uniform_bindings + tmp.uniform_binding_count; - memcpy(storage_bindings, - tmp.storage_bindings, - sizeof(rt_attribute_binding) * tmp.storage_binding_count); - } - rtSetRelptr(&info->texture_bindings, texture_bindings); - rtSetRelptr(&info->uniform_bindings, uniform_bindings); - rtSetRelptr(&info->storage_bindings, storage_bindings); - - rtReleaseBuffer(tmp.texture_bindings, - sizeof(rt_attribute_binding) * tmp.texture_binding_count); - rtReleaseBuffer(tmp.storage_bindings, - sizeof(rt_attribute_binding) * tmp.storage_binding_count); - rtReleaseBuffer(tmp.uniform_bindings, - sizeof(rt_attribute_binding) * tmp.uniform_binding_count); - - output->data = out_buffer; - output->size = outbuffer_size; - output->asset_uid = rtCalculateUID(rtGetFilePath(file)); - - /* Store dependencies to shaders */ - if (info->vertex_shader != RT_INVALID_UID) - result = rtAddAssetDependency(output->asset_uid, info->vertex_shader); - if (info->fragment_shader != RT_INVALID_UID) - result = rtAddAssetDependency(output->asset_uid, info->fragment_shader); - if (info->compute_shader != RT_INVALID_UID) - result = rtAddAssetDependency(output->asset_uid, info->compute_shader); - } - return result; -} \ No newline at end of file diff --git a/src/tools/assetc/processing.h b/src/tools/assetc/processing.h deleted file mode 100644 index 1e1b328..0000000 --- a/src/tools/assetc/processing.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef RT_ASSETC_PROCESSING_H -#define RT_ASSETC_PROCESSING_H - -#include "runtime/assets.h" -#include "runtime/file_tab.h" - -enum { - RT_PROCESSING_FAILED = RT_CUSTOM_ERROR_START, - /* Used if the processing depends on other files beeing processed first. */ - RT_PROCESSING_TRY_AGAIN, -}; - -typedef struct { - rt_file_id output_file; - -} rt_asset_options; - -typedef struct { - rt_uid asset_uid; - - /* Allocated via the rtAllocBuffer API */ - void *data; - size_t size; -} rt_processor_output; - -typedef rt_result rt_processor_fn(rt_file_id file, - void *buffer, - size_t size, - uint32_t flags, - rt_processor_output *output); - -rt_result rtAddAssetProcessor(const char *file_extension, rt_processor_fn fn); - -/* Flags are file type specific */ -rt_result rtAddFileToProcessingQueue(rt_file_id file, uint32_t flags); - -rt_result rtStartProcessing(void); -void rtStopProcessing(void); - -rt_uid rtCalculateUID(const char *name); -rt_result rtWriteUIDTab(void); -rt_result rtAddUIDTabEntry(rt_file_id package_fid, rt_uid uid, uint64_t offset, uint64_t size); - -void rtWaitUntilProcessingIsFinished(void); - -#endif diff --git a/src/tools/assetc/processing_flags.h b/src/tools/assetc/processing_flags.h deleted file mode 100644 index bb2941c..0000000 --- a/src/tools/assetc/processing_flags.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef RT_ASSETC_PROCESSING_FLAGS_H -#define RT_ASSETC_PROCESSING_FLAGS_H - -/* Shader processing flags */ -enum { - /* Type is encoded in the lower bits */ - RT_SHADER_FLAG_VERTEX = 0x01, - RT_SHADER_FLAG_FRAGMENT = 0x02, - RT_SHADER_FLAG_COMPUTE = 0x03, - - RT_SHADER_FLAG_TYPE_MASK = - RT_SHADER_FLAG_VERTEX | RT_SHADER_FLAG_FRAGMENT | RT_SHADER_FLAG_COMPUTE, - - RT_SHADER_FLAG_OPTIMIZE_SPEED = 0x04, - RT_SHADER_FLAG_OPTIMIZE_SIZE = 0x08, - RT_SHADER_FLAG_OPTIMIZE_MASK = RT_SHADER_FLAG_OPTIMIZE_SPEED | RT_SHADER_FLAG_OPTIMIZE_SIZE, -}; - - -#endif diff --git a/src/tools/assetc/processor.c b/src/tools/assetc/processor.c deleted file mode 100644 index a4f9ca3..0000000 --- a/src/tools/assetc/processor.c +++ /dev/null @@ -1,381 +0,0 @@ -#include "processing.h" -#include "utils.h" -#include "description_parser.h" -#include "packages.h" -#include "assetmeta.h" -#include "assetsettings.h" - -#include "runtime/aio.h" -#include "runtime/buffer_manager.h" -#include "runtime/file_tab.h" -#include "runtime/threading.h" - -#include -#include -#include -#include - - -typedef struct { - rt_file_id fid; - - uint32_t flags; - - /* How many times has this file been added? */ - unsigned int turn; -} rt_file_processing_queue_entry; - -#define QUEUE_LENGTH 1024 -typedef struct { - rt_file_processing_queue_entry entries[QUEUE_LENGTH]; - unsigned int head; - unsigned int tail; -} rt_file_processing_queue; - -static rt_file_processing_queue _queues[2]; -static rt_file_processing_queue *_processing_queue; -static rt_file_processing_queue *_retry_queue; - -static rt_mutex *_guard; -static bool _keep_running; - -/* A single file could have a lot of dependencies. */ -#define MAX_TURNS 100 - -#define MAX_PROCESSING_THREADS 16 - -#define FORCE_SINGLE_THREAD 1 - -static unsigned int _num_processing_threads = 0; -static rt_thread *_processing_threads[MAX_PROCESSING_THREADS]; - -static unsigned int _processing_thread_count = 0; - - -static rt_result rtAddFileToProcessingQueueImpl(rt_file_processing_queue *queue, - rt_file_id file, - uint32_t flags, - unsigned int turn) { - rt_result result = RT_SUCCESS; - - rtLog("ASSETC", "Adding %s to processing queue.", rtGetFilePath(file)); - - rt_file_processing_queue_entry entry = { - .fid = file, - .flags = flags, - .turn = turn, - }; - if ((queue->head + 1) % QUEUE_LENGTH != queue->tail) { - unsigned int slot = queue->head; - queue->entries[slot] = entry; - queue->head = (queue->head + 1) % QUEUE_LENGTH; - } else { - rtReportError("ASSETC", "The processing queue is full!"); - result = 1; - } - return result; -} - -rt_result rtAddFileToProcessingQueue(rt_file_id file, uint32_t flags) { - assert(_guard != NULL); - - rtLockMutex(_guard); - rt_result res = rtAddFileToProcessingQueueImpl(_processing_queue, file, flags, 1); - rtUnlockMutex(_guard); - return res; -} - -#define MAX_PROCESSORS 256 -static rt_processor_fn *_processor_fns[MAX_PROCESSORS]; -static const char *_processor_exts[MAX_PROCESSORS]; -static unsigned int _processor_count; - -rt_result rtAddAssetProcessor(const char *file_extension, rt_processor_fn fn) { - /* Should only be called from main thread */ - if (_processor_count == MAX_PROCESSORS) { - rtReportError("ASSETC", "Too many asset processor functions!"); - return 1; - } - _processor_fns[_processor_count] = fn; - _processor_exts[_processor_count] = file_extension; - ++_processor_count; - return RT_SUCCESS; -} - -static void PopAndSwapSubmittedData(unsigned int at, - unsigned int *count, - rt_file_processing_queue_entry *queue_entries, - rt_aio_handle *handles, - void **buffers, - size_t *sizes) { - if (at < *count - 1) { - queue_entries[at] = queue_entries[*count - 1]; - buffers[at] = buffers[*count - 1]; - handles[at] = handles[*count - 1]; - sizes[at] = sizes[*count - 1]; - } - *count = *count - 1; -} - -static rt_result ProcessLoadedFile(rt_file_processing_queue_entry entry, void *buffer, size_t size) { - /* Search for a matching processor function */ - const char *path = rtGetFilePath(entry.fid); - size_t path_len = strlen(path); - for (unsigned int i = 0; i < _processor_count; ++i) { - size_t ext_len = strlen(_processor_exts[i]); - if (ext_len > path_len) - continue; - - const char *path_end = &path[path_len - ext_len]; - if (memcmp(path_end, _processor_exts[i], ext_len) == 0) { - - /* Load the corresponding .as file. - * TODO: Using malloc here is probably relatively slow. - */ - rt_result res; - - rt_asset_settings settings; - if ((res = rtLoadAssetSettings(path, &settings)) != RT_SUCCESS) { - return res; - } - - /* Process the asset */ - rt_processor_output out; - res = _processor_fns[i](entry.fid, buffer, size, entry.flags, &out); - if (res == RT_SUCCESS) { - /* Add the output to the appropriate package file */ - rt_assetmeta meta; - meta.compiled_ts = rtGetCurrentTimestamp(); - meta.last_changed = rtGetFileModificationTimestamp(entry.fid); - meta.processing_flags = entry.flags; - rtAddUIDMapping(entry.fid, out.asset_uid, &meta); - - rtAddAssetToPackage(settings.package, out.asset_uid, out.data, out.size); - } else if (res == RT_PROCESSING_TRY_AGAIN) { - if (entry.turn < MAX_TURNS) { - rtLockMutex(_guard); - rtAddFileToProcessingQueueImpl(_retry_queue, entry.fid, entry.flags, entry.turn + 1); - rtUnlockMutex(_guard); - } else { - rtLog("ASSETC", - "File '%s' took too many turns to process: %u", - path, - entry.turn); - } - } else { - rtLog("ASSETC", "Failed to process file: %s (Result %u)", path, res); - } - return res; - } - } - rtLog("ASSETC", "No asset processor for file: %s", path); - return RT_UNKNOWN_ERROR; -} - -static void ProcessingThread(void *_param) { - RT_UNUSED(_param); - - rt_file_processing_queue_entry submitted_entries[RT_LOAD_BATCH_MAX_SIZE]; - rt_aio_handle submitted_handles[RT_LOAD_BATCH_MAX_SIZE]; - void *submitted_buffers[RT_LOAD_BATCH_MAX_SIZE]; - size_t submitted_sizes[RT_LOAD_BATCH_MAX_SIZE]; - unsigned int submitted_outstanding = 0; - - while (_keep_running) { - rt_load_batch load_batch; - rt_file_processing_queue_entry load_entries[RT_LOAD_BATCH_MAX_SIZE]; - void *load_buffers[RT_LOAD_BATCH_MAX_SIZE]; - size_t load_sizes[RT_LOAD_BATCH_MAX_SIZE]; - load_batch.num_loads = 0; - - bool got_entry = false; - do { - got_entry = false; - rt_file_processing_queue_entry entry = {0}; - rtLockMutex(_guard); - if (_processing_queue->head != _processing_queue->tail) { - entry = _processing_queue->entries[_processing_queue->tail]; - _processing_queue->tail = (_processing_queue->tail + 1) % QUEUE_LENGTH; - got_entry = true; - } else if (load_batch.num_loads == 0) { - /* Switch the queues -> Retry all the entries that returned RT_PROCESSING_TRY_AGAIN */ - if (_retry_queue->head != _retry_queue->tail) { - rt_file_processing_queue *tmp = _retry_queue; - _retry_queue = _processing_queue; - _processing_queue = tmp; - } - } - rtUnlockMutex(_guard); - - /* Retry, if we did not get an entry */ - if (!got_entry) - continue; - - const char *path = rtGetFilePath(entry.fid); - if (!path) { - rtLog("ASSETC", "Invalid file id: %#x", entry.fid); - continue; - } - - rtLog("ASSETC", "Processing %s", path); - - size_t fsz = rtGetFileSize(path); - void *dest = rtAllocBuffer(fsz); - if (!dest) { - rtLog("ASSETC", "Ran out of memory for loading the file: %s", path); - continue; - } - memset(dest, 0, fsz); - - load_sizes[load_batch.num_loads] = fsz; - load_buffers[load_batch.num_loads] = dest; - load_batch.loads[load_batch.num_loads].file = entry.fid; - load_batch.loads[load_batch.num_loads].dest = dest; - load_batch.loads[load_batch.num_loads].num_bytes = fsz; - load_batch.loads[load_batch.num_loads].offset = 0; - load_entries[load_batch.num_loads] = entry; - ++load_batch.num_loads; - } while (got_entry && load_batch.num_loads < RT_LOAD_BATCH_MAX_SIZE); - - rt_aio_handle load_handles[RT_LOAD_BATCH_MAX_SIZE]; - if (load_batch.num_loads > 0) { - rt_result submit_result = rtSubmitLoadBatch(&load_batch, load_handles); - if (submit_result != RT_SUCCESS) { - rtLog("ASSETC", "SubmitLoadBatch failed: %u", submit_result); - continue; - } - } - - /* Process the previously submitted loads */ - while (submitted_outstanding > 0) { - rtLockMutex(_guard); - _processing_thread_count += 1; - rtUnlockMutex(_guard); - - for (unsigned int i = 0; i < submitted_outstanding; ++i) { - rt_aio_state state = rtGetAIOState(submitted_handles[i]); - switch (state) { - case RT_AIO_STATE_PENDING: - continue; - case RT_AIO_STATE_FAILED: - rtLog("ASSETC", - "Loading file %s failed.", - rtGetFilePath(submitted_entries[i].fid)); - rtReleaseAIO(submitted_handles[i]); - rtReleaseBuffer(submitted_buffers[i], submitted_sizes[i]); - PopAndSwapSubmittedData(i, - &submitted_outstanding, - submitted_entries, - submitted_handles, - submitted_buffers, - submitted_sizes); - --i; - break; - case RT_AIO_STATE_INVALID: - rtLog("ASSETC", - "Got invalid AIO handle for file: %s", - rtGetFilePath(submitted_entries[i].fid)); - rtReleaseBuffer(submitted_buffers[i], submitted_sizes[i]); - PopAndSwapSubmittedData(i, - &submitted_outstanding, - submitted_entries, - submitted_handles, - submitted_buffers, - submitted_sizes); - --i; - break; - case RT_AIO_STATE_FINISHED: - ProcessLoadedFile(submitted_entries[i], - submitted_buffers[i], - submitted_sizes[i]); - rtReleaseAIO(submitted_handles[i]); - rtReleaseBuffer(submitted_buffers[i], submitted_sizes[i]); - PopAndSwapSubmittedData(i, - &submitted_outstanding, - submitted_entries, - submitted_handles, - submitted_buffers, - submitted_sizes); - --i; - } - } - - rtLockMutex(_guard); - _processing_thread_count -= 1; - rtUnlockMutex(_guard); - } - - /* Start new round */ - assert(sizeof(submitted_entries) == sizeof(load_entries)); - assert(sizeof(submitted_handles) == sizeof(load_handles)); - assert(sizeof(submitted_buffers) == sizeof(load_buffers)); - assert(sizeof(submitted_sizes) == sizeof(load_sizes)); - memcpy(submitted_entries, load_entries, sizeof(submitted_entries)); - memcpy(submitted_handles, load_handles, sizeof(submitted_handles)); - memcpy(submitted_buffers, load_buffers, sizeof(submitted_buffers)); - memcpy(submitted_sizes, load_sizes, sizeof(submitted_sizes)); - submitted_outstanding = load_batch.num_loads; - } -} - -rt_result rtStartProcessing(void) { - if (!_guard) - _guard = rtCreateMutex(); - -#if !FORCE_SINGLE_THREAD - _num_processing_threads = rtGetCPUCoreCount(); - if (_num_processing_threads > MAX_PROCESSING_THREADS) - _num_processing_threads = MAX_PROCESSING_THREADS; -#else - _num_processing_threads = 1; -#endif - - _processing_queue = &_queues[0]; - _retry_queue = &_queues[1]; - - _keep_running = true; - for (unsigned int i = 0; i < _num_processing_threads; ++i) { - _processing_threads[i] = rtSpawnThread(ProcessingThread, NULL, "Processor"); - if (!_processing_threads[i]) { - rtReportError("ASSETC", "Failed to spawn processing thread %u!", i); - _keep_running = false; - return 1; - } - } - return RT_SUCCESS; -} - -void rtStopProcessing(void) { - _keep_running = false; - for (unsigned int i = 0; i < _num_processing_threads; ++i) - rtJoinThread(_processing_threads[i]); -} - - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#else -#include -#endif - - -void rtWaitUntilProcessingIsFinished(void) { - unsigned int done_counter = 0; - while (done_counter < 3) { -#ifdef _WIN32 - Sleep(200); -#else - sleep(1); -#endif - rtLockMutex(_guard); - volatile bool done = _processing_queue->head == _processing_queue->tail && - _retry_queue->head == _retry_queue->tail && - _processing_thread_count == 0; - rtUnlockMutex(_guard); - if (done) - ++done_counter; - else - done_counter = 0; - } -} \ No newline at end of file diff --git a/src/tools/assetc/shader_processor.c b/src/tools/assetc/shader_processor.c deleted file mode 100644 index 1cf6f60..0000000 --- a/src/tools/assetc/shader_processor.c +++ /dev/null @@ -1,123 +0,0 @@ -#include "assetmeta.h" -#include "processing.h" -#include "processing_flags.h" -#include "utils.h" - -#include "runtime/buffer_manager.h" - -#include -#include - -static shaderc_include_result *ResolveInclude(void *user_data, - const char *requested_source, - int type, - const char *requesting_source, - size_t include_depth) { - shaderc_include_result *result = NULL; - - return result; -} - -static void ReleaseIncludeResult(void *user_data, shaderc_include_result *result) { -} - -rt_result rtProcessShaderFile(rt_file_id file, - void *buffer, - size_t size, - uint32_t flags, - rt_processor_output *output) { - /* If we determine that shader compilation takes too long, we can instead - * keep the compiler around */ - - shaderc_compiler_t compiler = shaderc_compiler_initialize(); - if (!compiler) { - rtLog("ASSETC", "Failed to initialize the shaderc compiler."); - return RT_PROCESSING_FAILED; - } - - const char *path = rtGetFilePath(file); - - shaderc_compile_options_t options = shaderc_compile_options_initialize(); - if (!options) { - rtLog("ASSETC", "Failed to initialize shader compile options."); - shaderc_compiler_release(compiler); - return RT_PROCESSING_FAILED; - } - - shaderc_compile_options_set_include_callbacks(options, - ResolveInclude, - ReleaseIncludeResult, - NULL); - - uint32_t optimize_from_flags = flags & RT_SHADER_FLAG_OPTIMIZE_MASK; - if (optimize_from_flags == RT_SHADER_FLAG_OPTIMIZE_SPEED) - shaderc_compile_options_set_optimization_level(options, - shaderc_optimization_level_performance); - else if (optimize_from_flags == RT_SHADER_FLAG_OPTIMIZE_SIZE) - shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_size); - else - shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_zero); - - uint32_t type_from_flags = flags & RT_SHADER_FLAG_TYPE_MASK; - shaderc_shader_kind shaderc_kind; - switch (type_from_flags) { - case RT_SHADER_FLAG_VERTEX: - shaderc_kind = shaderc_glsl_vertex_shader; - break; - case RT_SHADER_FLAG_FRAGMENT: - shaderc_kind = shaderc_glsl_fragment_shader; - break; - case RT_SHADER_FLAG_COMPUTE: - shaderc_kind = shaderc_glsl_compute_shader; - break; - default: - rtLog("ASSETC", "Invalid shader stage flag %u", type_from_flags); - shaderc_compile_options_release(options); - shaderc_compiler_release(compiler); - return RT_PROCESSING_FAILED; - } - - shaderc_compilation_result_t result = shaderc_compile_into_spv(compiler, - (const char *)buffer, - size, - shaderc_kind, - path, - "main", - options); - - - shaderc_compilation_status status = shaderc_result_get_compilation_status(result); - if (status != shaderc_compilation_status_success || shaderc_result_get_num_errors(result) > 0) { - - rtLog("ASSETC", - "Compilation of %s failed: %s", - path, - shaderc_result_get_error_message(result)); - - shaderc_result_release(result); - shaderc_compile_options_release(options); - shaderc_compiler_release(compiler); - return RT_PROCESSING_FAILED; - } - - size_t output_size = shaderc_result_get_length(result); - output->data = rtAllocBuffer(output_size); - if (!output->data) { - shaderc_result_release(result); - shaderc_compile_options_release(options); - shaderc_compiler_release(compiler); - rtLog("ASSETC", "Failed to allocate %zu bytes for shader compilation output.", output_size); - return RT_PROCESSING_FAILED; - } - const uint32_t *spv_bytes = (uint32_t *)shaderc_result_get_bytes(result); - memcpy(output->data, spv_bytes, output_size); - output->size = output_size; - - shaderc_result_release(result); - shaderc_compile_options_release(options); - shaderc_compiler_release(compiler); - - output->asset_uid = rtCalculateUID(path); - - return RT_SUCCESS; -} \ No newline at end of file diff --git a/src/tools/assetc/uidmap.c b/src/tools/assetc/uidmap.c deleted file mode 100644 index e69de29..0000000 diff --git a/src/tools/assetc/uidtable.c b/src/tools/assetc/uidtable.c deleted file mode 100644 index 3549d5e..0000000 --- a/src/tools/assetc/uidtable.c +++ /dev/null @@ -1,69 +0,0 @@ -#include "processing.h" -#include "utils.h" - -#define RT_DEFINE_UIDTAB_FILE_STRUCTURES -#include "runtime/threading.h" -#include "runtime/uidtab.h" - -#include -#include -#include -#include -#include - -static rt_uidtab_entry *_entries; -static size_t _entry_capacity; -static size_t _entry_count; - -rt_uid rtCalculateUID(const char *name) { - assert(sizeof(XXH32_hash_t) == sizeof(rt_uid)); - - size_t len = strlen(name); - return (rt_uid)XXH32(name, len, 0); -} - -rt_result rtAddUIDTabEntry(rt_file_id package_fid, rt_uid uid, uint64_t offset, uint64_t size) { - assert(rtIsMainThread()); - - if (_entry_count == _entry_capacity) { - size_t new_cap = (_entry_capacity > 0) ? _entry_capacity * 2 : 256; - void *t = realloc(_entries, sizeof(rt_uidtab_entry) * new_cap); - if (!t) - return RT_UNKNOWN_ERROR; - _entry_capacity = new_cap; - _entries = t; - } - _entries[_entry_count].file = package_fid; - _entries[_entry_count].offset = offset; - _entries[_entry_count].size = size; - _entries[_entry_count].uid = uid; - _entry_count++; - return RT_SUCCESS; -} - -rt_result rtWriteUIDTab(void) { - rt_uidtab_header header; - XXH64_hash_t checksum = XXH3_64bits(_entries, sizeof(rt_uidtab_entry) * _entry_count); - XXH64_canonicalFromHash(&header.checksum, checksum); - header.num_entries = (uint32_t)_entry_count; - - FILE *f = fopen("data/uidtab.bin", "wb"); - if (!f) { - rtReportError("ASSETC", "Failed to open 'uidtab.bin' for writing."); - return RT_UNKNOWN_ERROR; - } - - if (fwrite(&header, sizeof(header), 1, f) != 1) { - rtReportError("ASSETC", "Failed to write header to 'uidtab.bin'"); - fclose(f); - return RT_UNKNOWN_ERROR; - } - if (fwrite(_entries, sizeof(rt_uidtab_entry), _entry_count, f) != _entry_count) { - rtReportError("ASSETC", "Failed to write entries to 'uidtab.bin'"); - fclose(f); - return RT_UNKNOWN_ERROR; - } - - fclose(f); - return RT_SUCCESS; -} \ No newline at end of file diff --git a/src/tools/assetc/utils.c b/src/tools/assetc/utils.c deleted file mode 100644 index 72bb634..0000000 --- a/src/tools/assetc/utils.c +++ /dev/null @@ -1,101 +0,0 @@ -#include "utils.h" - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif - -size_t rtGetFileSize(const char *path) { -#ifdef _WIN32 - WCHAR wpath[MAX_PATH]; - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH); - WIN32_FILE_ATTRIBUTE_DATA attribs; - if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs)) - return 0; - return (size_t)attribs.nFileSizeHigh << 32 | (size_t)attribs.nFileSizeLow; -#endif -} - -void rtSetWorkingDirectory(const char *path) { -#ifdef _WIN32 - WCHAR wpath[MAX_PATH]; - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH); - SetCurrentDirectoryW(wpath); -#endif -} - -uint64_t rtGetCurrentTimestamp(void) { -#ifdef _WIN32 - FILETIME ft; - GetSystemTimeAsFileTime(&ft); - uint64_t ts = ft.dwLowDateTime; - ts |= (uint64_t)ft.dwHighDateTime << 32; - return ts; -#endif -} - -uint64_t rtGetFileModificationTimestamp(rt_file_id fid) { -#ifdef _WIN32 - const char *path = rtGetFilePath(fid); - if (!path) { - rtLog("ASSETC", "Tried to get modification timestamp of unknown file."); - return 0; - } - WCHAR wpath[MAX_PATH]; - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH); - WIN32_FILE_ATTRIBUTE_DATA attribs; - if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs)) - return 0; - uint64_t ts = attribs.ftLastWriteTime.dwLowDateTime; - ts |= (uint64_t)attribs.ftLastWriteTime.dwHighDateTime << 32; - return ts; -#endif -} - -rt_result rtIterateDirectory(const char *path, void *user, rt_iterate_directory_cb_func iterate_cb) { -#ifdef _WIN32 - rt_result res; - WCHAR wpath[MAX_PATH]; - char wildcard_path[MAX_PATH]; - strncpy(wildcard_path, path, MAX_PATH); - strncat(wildcard_path, "\\*", MAX_PATH - strlen(path)); - - res = rtUTF8ToWStr(wildcard_path, wpath, MAX_PATH); - if (res != RT_SUCCESS) - return res; - - WIN32_FIND_DATAW find; - HANDLE h = FindFirstFileW(wpath, &find); - if (h == INVALID_HANDLE_VALUE) - return RT_UNKNOWN_ERROR; - do { - char utf8_file[MAX_PATH]; - res = rtWStrToUTF8(find.cFileName, utf8_file, MAX_PATH); - if (res != RT_SUCCESS) - break; - rtIterateDirElementType type; - if (find.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - type = RT_DIR_ELEMENT_TYPE_DIRECTORY; - else - type = RT_DIR_ELEMENT_TYPE_FILE; - res = iterate_cb(utf8_file, type, user); - if (res != RT_SUCCESS) - break; - } while (FindNextFileW(h, &find) != 0); - - FindClose(h); - return res; -#endif -} - -rt_result rtCreateDirectory(const char *path) { -#ifdef _WIN32 - WCHAR wpath[MAX_PATH]; - rt_result res = rtUTF8ToWStr(path, wpath, MAX_PATH); - if (res != RT_SUCCESS) - return res; - if (!CreateDirectoryW(wpath, NULL)) - res = RT_UNKNOWN_ERROR; - return res; -#endif -} \ No newline at end of file diff --git a/src/tools/assetc/utils.h b/src/tools/assetc/utils.h deleted file mode 100644 index 9aacc7f..0000000 --- a/src/tools/assetc/utils.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef RT_ASSETC_UTILS_H -#define RT_ASSETC_UTILS_H - -#include - -#include "runtime/file_tab.h" - -size_t rtGetFileSize(const char *path); - -void rtSetWorkingDirectory(const char *path); - -uint64_t rtGetCurrentTimestamp(void); - -uint64_t rtGetFileModificationTimestamp(rt_file_id fid); - -typedef enum { - RT_DIR_ELEMENT_TYPE_FILE, - RT_DIR_ELEMENT_TYPE_DIRECTORY, -} rtIterateDirElementType; - -typedef rt_result rt_iterate_directory_cb_func(const char *name, rtIterateDirElementType type, void *user); - -rt_result rtIterateDirectory(const char *path, void *user, rt_iterate_directory_cb_func iterate_cb); - -rt_result rtCreateDirectory(const char *path); - -#endif