kevin/rtengine
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove unnecessary files

Browse Source
This commit is contained in:
Kevin Trogant 2024-02-04 17:28:32 +01:00
parent cfec746545
commit b80d0d1bf9
26 changed files with 0 additions and 2791 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
src/runtime/Source.cpp
View File

20
src/runtime/packages.c
View File

@ -1,20 +0,0 @@
#include <stdio.h>
#include "file_tab.h"
rt_result LoadPackageNames(void) {
FILE *f = fopen("data/packages.txt", "r");
if (!f) {
return RT_UNKNOWN_ERROR;
}
while (!feof(f)) {
char line[256];
fscanf(f, "%255s\n", line);
line[255] = '\0';
rtAddFile(line);
}
fclose(f);
return RT_SUCCESS;
}

18
src/runtime/packages.h
View File

@ -1,18 +0,0 @@
#ifndef RT_PACKAGES_H
#define RT_PACKAGES_H
#ifdef RT_DEFINE_PACKAGE_FILE_STRUCTURES
#include <stdint.h>
#include "xxhash/xxhash.h"
#pragma pack(push, 1)
typedef struct {
XXH64_canonical_t checksum;
uint32_t decompressed_size;
} rt_package_asset_header;
#pragma pack(pop)
#endif
#endif

149
src/tools/assetc/assetc.c
View File

@ -1,149 +0,0 @@
#include "assetmeta.h"
#include "processing.h"
#include "utils.h"
#include "packages.h"
#include "dependency_tracking.h"
#define RT_ASSETC_DONT_DEFINE_OPTIONS_GLOBAL
#include "options.h"
#include "runtime/aio.h"
#include "runtime/buffer_manager.h"
#include "runtime/uidtab.h"
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
extern rt_result rtProcessPipelineFile(rt_file_id file,
void *buffer,
size_t size,
uint32_t flags,
rt_processor_output *output);
extern rt_result rtProcessShaderFile(rt_file_id file,
void *buffer,
size_t size,
uint32_t flags,
rt_processor_output *output);
extern void rtDiscoverAssets(void);
rt_assetc_options g_assetc_options = {
.root_directory = ".",
.optimization = RT_ASSET_OPTIMIZATION_NONE,
.renderer_backend = RT_RENDERER_BACKEND_CODE_VK,
};
static bool ParseCommandLineArgs(int argc, char **argv) {
bool have_root_directory = false;
for (int i = 1; i < argc; ++i) {
if (i < argc - 1) {
const char *val = argv[i + 1];
bool matched = false;
if (strcmp(argv[i], "-r") == 0 || strcmp(argv[i], "--renderer") == 0) {
if (strcmp(val, "vk") == 0) {
g_assetc_options.renderer_backend = RT_RENDERER_BACKEND_CODE_VK;
} else {
rtReportError("ASSETC",
"Invalid render backend %s. Valid options are: vk",
val);
return false;
}
matched = true;
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--optimization") == 0) {
if (strcmp(val, "none") == 0) {
g_assetc_options.optimization = RT_ASSET_OPTIMIZATION_NONE;
} else if (strcmp(val, "space") == 0) {
g_assetc_options.optimization = RT_ASSET_OPTIMIZATION_SPACE;
} else if (strcmp(val, "performance") == 0) {
g_assetc_options.optimization = RT_ASSET_OPTIMIZATION_PERFORMANCE;
} else {
rtReportError("ASSETC",
"Invalid optimization level %s. Valid options are: none, space, "
"performance",
val);
return false;
}
matched = true;
} else if (argv[i][0] == '-') {
rtReportError("ASSETC", "Invalid command line argument %s", argv[i]);
return false;
}
if (matched) {
/* Skip the value */
++i;
continue;
}
}
if (!have_root_directory) {
g_assetc_options.root_directory = argv[i];
} else {
/* Maybe have secondary directories later? */
rtReportError("ASSETC",
"More than one root directory passed as command line "
"argument.");
return false;
}
}
return true;
}
int main(int argc, char **argv) {
rtInitRuntime();
/* Init assetc */
if (!ParseCommandLineArgs(argc, argv)) {
return 1;
}
rtInitPackages();
if (rtLoadAssetMeta() != RT_SUCCESS) {
return 1;
}
if (rtInitDependencyTracking() != RT_SUCCESS) {
return 1;
}
if (rtAddAssetProcessor(".pipeline", rtProcessPipelineFile) != RT_SUCCESS)
return 1;
if (rtAddAssetProcessor(".glsl", rtProcessShaderFile) != RT_SUCCESS)
return 1;
if (rtStartProcessing() != RT_SUCCESS) {
return 1;
}
/* Create necessary directories */
rtSetWorkingDirectory(g_assetc_options.root_directory);
rtCreateDirectory("assets");
rtCreateDirectory("actemp");
rtCreateDirectory("data");
/* "Mainloop" */
rtDiscoverAssets();
rtWaitUntilProcessingIsFinished();
rtStopProcessing();
/* Write result */
rtSaveAssetMeta();
if (rtSavePackages() != RT_SUCCESS) {
return 1;
}
if (rtWriteUIDTab() != RT_SUCCESS) {
return 1;
}
if (rtSaveAssetDependencies() != RT_SUCCESS) {
return 1;
}
rtShutdownRuntime();
return 0;
}

222
src/tools/assetc/assetmeta.c
View File

@ -1,222 +0,0 @@
#include "assetmeta.h"
#include "utils.h"
#include "options.h"
#include "runtime/threading.h"
#include "runtime/aio.h"
#include "runtime/buffer_manager.h"
#include <xxhash/xxhash.h>
#include <stdio.h>
#include <string.h>
#define MAP_SIZE 2048
typedef struct {
rt_file_id fids[MAP_SIZE];
rt_uid uids[MAP_SIZE];
rt_assetmeta meta[MAP_SIZE];
unsigned int used_slots;
} rt_uid_map;
#pragma pack(push, 1)
typedef struct {
XXH64_canonical_t checksum;
uint32_t num_entries;
uint32_t _reserved;
} rt_assetmeta_header;
#pragma pack(pop)
#pragma pack(push, 1)
typedef struct {
rt_uid uid;
rt_file_id source_file;
rt_assetmeta meta;
} rt_assetmeta_entry;
#pragma pack(pop)
static rt_uid_map _map;
static rt_mutex *_guard;
rt_result rtLoadAssetMeta(void) {
_guard = rtCreateMutex();
/* Load the meta file */
size_t fsz = rtGetFileSize("actemp/meta.bin");
if (fsz == 0) {
rtLog("ASSETC", "Metadata file 'meta.bin' not found. All assets will be processed.");
return RT_SUCCESS;
}
void *buffer = rtAllocBuffer(fsz);
if (!buffer) {
rtReportError("ASSETC", "Failed to allocate buffer for holding asset metadata.");
return 1;
}
rt_load_batch load;
load.loads[0].dest = buffer;
load.loads[0].file = rtAddFile("actemp/meta.bin");
load.loads[0].num_bytes = fsz;
load.loads[0].offset = 0;
load.num_loads = 1;
rt_aio_handle handle;
if (rtSubmitLoadBatch(&load, &handle) != RT_SUCCESS) {
rtReportError("ASSETC", "Failed to submit load of 'meta.bin'.");
rtReleaseBuffer(buffer, fsz);
return 1;
}
if (rtWaitForAIOCompletion(handle) != RT_AIO_STATE_FINISHED) {
rtReportError("ASSETC", "Failed to load 'meta.bin'.");
rtReleaseBuffer(buffer, fsz);
rtReleaseAIO(handle);
return 1;
}
rtReleaseAIO(handle);
const rt_assetmeta_header *header = buffer;
const rt_assetmeta_entry *entries = (rt_assetmeta_entry *)(header + 1);
if ((sizeof(rt_assetmeta_entry) * header->num_entries + sizeof(*header)) > fsz) {
rtReportError("ASSETC", "Header of 'meta.bin' is corrupted: Mismatched num_entries and filesize.");
rtReleaseBuffer(buffer, fsz);
return 1;
}
XXH64_hash_t hash = XXH3_64bits(entries, sizeof(rt_assetmeta_entry) * header->num_entries);
XXH64_hash_t header_hash = XXH64_hashFromCanonical(&header->checksum);
if (hash != header_hash) {
rtReportError("ASSETC",
"Metadata file 'meta.bin' is corrupted: Wrong checksum.");
rtReleaseBuffer(buffer, fsz);
return 1;
}
for (uint32_t i = 0; i < header->num_entries; ++i) {
/* Load here to avoid unaligned pointer */
rt_assetmeta meta = entries[i].meta;
rtAddUIDMapping(entries[i].source_file, entries[i].uid, &meta);
}
rtReleaseBuffer(buffer, fsz);
return 0;
}
rt_result rtSaveAssetMeta(void) {
rt_assetmeta_header header = {0};
/* Count number of entries */
for (size_t i = 0; i < MAP_SIZE; ++i) {
if (_map.fids[i] != RT_INVALID_FILE_ID)
header.num_entries += 1;
}
rt_assetmeta_entry *entries = rtAllocBuffer(sizeof(rt_assetmeta_entry) * header.num_entries);
if (!entries)
return RT_UNKNOWN_ERROR;
/* Store entries */
size_t j = 0;
for (size_t i = 0; i < MAP_SIZE; ++i) {
if (_map.fids[i] != RT_INVALID_FILE_ID) {
entries[j].source_file = _map.fids[i];
entries[j].uid = _map.uids[i];
entries[j].meta = _map.meta[i];
++j;
}
}
XXH64_hash_t hash = XXH3_64bits(entries, sizeof(rt_assetmeta_entry) * header.num_entries);
XXH64_canonicalFromHash(&header.checksum, hash);
header._reserved = 0;
FILE *f = fopen("actemp/meta.bin", "wb");
if (!f) {
rtReportError("ASSETC", "Failed to open 'meta.bin'");
rtReleaseBuffer(entries, sizeof(rt_assetmeta_entry) * header.num_entries);
return RT_UNKNOWN_ERROR;
}
if (fwrite(&header, sizeof(header), 1, f) != 1) {
rtReportError("ASSETC", "Failed to write 'meta.bin'");
rtReleaseBuffer(entries, sizeof(rt_assetmeta_entry) * header.num_entries);
return RT_UNKNOWN_ERROR;
}
if (fwrite(entries, sizeof(rt_assetmeta_entry), header.num_entries, f) != header.num_entries) {
rtReportError("ASSETC", "Failed to write 'meta.bin'");
rtReleaseBuffer(entries, sizeof(rt_assetmeta_entry) * header.num_entries);
return RT_UNKNOWN_ERROR;
}
fclose(f);
return RT_SUCCESS;
}
rt_uid rtLookupUID(rt_file_id fid) {
rtLockMutex(_guard);
unsigned int i = 0;
rt_uid result = RT_INVALID_UID;
while (i < MAP_SIZE) {
unsigned int slot = (fid + i) % MAP_SIZE;
if (_map.fids[slot] == fid) {
result = _map.uids[slot];
break;
} else if (_map.fids[slot] == RT_INVALID_FILE_ID) {
break;
}
++i;
}
rtUnlockMutex(_guard);
return result;
}
bool rtGetAssetMeta(rt_file_id source_file, rt_assetmeta *meta) {
rtLockMutex(_guard);
unsigned int i = 0;
bool result = false;
while (i < MAP_SIZE) {
unsigned int slot = (source_file + i) % MAP_SIZE;
if (_map.fids[slot] == source_file) {
*meta = _map.meta[slot];
result = true;
break;
} else if (_map.fids[slot] == RT_INVALID_FILE_ID) {
break;
}
++i;
}
rtUnlockMutex(_guard);
return result;
}
void rtAddUIDMapping(rt_file_id fid, rt_uid uid, const rt_assetmeta *meta) {
rtLockMutex(_guard);
float fill_rate = (float)_map.used_slots / MAP_SIZE;
if (fill_rate >= .5f) {
rtLog("ASSETC", "UID map is above 50% filled.");
}
unsigned int i = 0;
while (i < MAP_SIZE) {
unsigned int slot = (fid + i) % MAP_SIZE;
if (_map.fids[slot] == RT_INVALID_FILE_ID) {
_map.fids[slot] = fid;
_map.uids[slot] = uid;
if (meta) {
_map.meta[slot] = *meta;
} else {
_map.meta[slot].last_changed = 0;
_map.meta[slot].compiled_ts = 0;
_map.meta[slot].processing_flags = 0;
}
++_map.used_slots;
break;
} else if (_map.fids[slot] == fid) {
break;
}
++i;
}
if (i == MAP_SIZE) {
rtReportError("ASSETC", "Failed to insert entry into UID map.");
}
rtUnlockMutex(_guard);
}

28
src/tools/assetc/assetmeta.h
View File

@ -1,28 +0,0 @@
#ifndef RT_ASSETC_ASSETMETA_H
#define RT_ASSETC_ASSETMETA_H
#include "runtime/file_tab.h"
#include "runtime/assets.h"
#include <stdbool.h>
/* Metadata for assets used only by assetc */
typedef struct {
uint64_t last_changed;
uint64_t compiled_ts;
uint32_t processing_flags;
} rt_assetmeta;
rt_result rtLoadAssetMeta(void);
rt_result rtSaveAssetMeta(void);
/* The UID map associates processed files with generated asset uids. */
void rtAddUIDMapping(rt_file_id fid, rt_uid uid, const rt_assetmeta *meta);
/* Returns true if the asset is found. false otherwise */
bool rtGetAssetMeta(rt_file_id source_file, rt_assetmeta *meta);
rt_uid rtLookupUID(rt_file_id fid);
#endif

144
src/tools/assetc/assetsettings.c
View File

@ -1,144 +0,0 @@
#include "assetsettings.h"
#include "description_parser.h"
#include "packages.h"
#include "utils.h"
#include "runtime/aio.h"
#include "runtime/buffer_manager.h"
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
rt_result rtParseAssetSettings(const char *text,
size_t length,
const char *file_path,
rt_asset_settings *settings) {
unsigned int root_list;
rt_parse_state state;
rt_result res = rtParseDescription(text, length, file_path, &root_list, &state);
if (res != RT_SUCCESS) {
rtReportError("ASSETC", "Failed to parse asset settings: %s", file_path);
return res;
}
/* Default settings */
settings->package = 0;
settings->processing_flags = 0;
settings->reprocess_on_dependency_change = false;
const rt_parsed_stmt *package_stmt = rtFindStatement(&state, root_list, "package");
if (package_stmt) {
if (package_stmt->form != RT_STMT_FORM_VALUE) {
rtReportError("ASSETC",
"Expected a package name as the value of 'package' in %s.",
file_path);
res = RT_UNKNOWN_ERROR;
goto out;
}
settings->package = rtAddPackageFile(package_stmt->value);
}
const rt_parsed_stmt *flags_stmt = rtFindStatement(&state, root_list, "processing_flags");
if (flags_stmt) {
if (flags_stmt->form != RT_STMT_FORM_VALUE) {
rtReportError("ASSETC",
"Expected a hexadecimal number as the value of 'processing_flags' in %s.",
file_path);
res = RT_UNKNOWN_ERROR;
goto out;
}
sscanf(flags_stmt->value.start, "%x", &settings->processing_flags);
}
const rt_parsed_stmt *reprocess_stmt =
rtFindStatement(&state, root_list, "reprocess_on_dependency_change");
if (reprocess_stmt) {
if (reprocess_stmt->form != RT_STMT_FORM_VALUE) {
rtReportError("ASSETC",
"Expected either 'true' or 'false' as the value of 'reprocess_on_dependency_change' in %s.",
file_path);
res = RT_UNKNOWN_ERROR;
goto out;
}
if (rtCompareSpanToString(reprocess_stmt->value, "true") == 0)
settings->reprocess_on_dependency_change = true;
else if (rtCompareSpanToString(reprocess_stmt->value, "false") == 0)
settings->reprocess_on_dependency_change = false;
else {
rtReportError("ASSETC",
"Expected either 'true' or 'false' as the value of "
"'reprocess_on_dependency_change' in %s.",
file_path);
res = RT_UNKNOWN_ERROR;
goto out;
}
}
out:
rtReleaseParseState(&state);
return res;
}
rt_result rtLoadAssetSettings(const char *asset_path, rt_asset_settings *settings) {
size_t path_len = strlen(asset_path);
char *as_path = malloc(path_len + 3);
if (!as_path) {
return RT_UNKNOWN_ERROR;
}
memcpy(as_path, asset_path, path_len);
size_t ext_len = 0;
while (ext_len < path_len) {
if (as_path[path_len - ext_len] == '.')
break;
++ext_len;
}
strcpy(&as_path[path_len - ext_len], ".as");
size_t as_size = rtGetFileSize(as_path);
if (as_size == 0) {
rtReportError("ASSETC", "Failed to retrieve size of setting file %s", as_path);
free(as_path);
return RT_UNKNOWN_ERROR;
}
void *as_buffer = rtAllocBuffer(as_size);
rt_load_batch as_load;
as_load.loads[0].file = rtAddFile(as_path);
as_load.loads[0].num_bytes = as_size;
as_load.loads[0].dest = as_buffer;
if (!as_load.loads[0].dest) {
rtReportError("ASSETC", "Failed to allocate buffer for setting file %s", as_path);
free(as_path);
return RT_UNKNOWN_ERROR;
}
as_load.loads[0].offset = 0;
as_load.num_loads = 1;
rt_aio_handle as_handle;
if (rtSubmitLoadBatch(&as_load, &as_handle) != RT_SUCCESS) {
rtReportError("ASSETC", "Failed to submit load of setting file %s", as_path);
free(as_path);
return RT_UNKNOWN_ERROR;
}
if (rtWaitForAIOCompletion(as_handle) != RT_AIO_STATE_FINISHED) {
rtReportError("ASSETC", "Failed to load setting file %s", as_path);
free(as_path);
return RT_UNKNOWN_ERROR;
}
rtReleaseAIO(as_handle);
if (rtParseAssetSettings(as_buffer, as_size, as_path, settings) != RT_SUCCESS) {
free(as_path);
return RT_UNKNOWN_ERROR;
}
free(as_path);
return RT_SUCCESS;
}

21
src/tools/assetc/assetsettings.h
View File

@ -1,21 +0,0 @@
#ifndef RT_ASSETC_ASSETSETTINGS_H
#define RT_ASSETC_ASSETSETTINGS_H
#include "runtime/runtime.h"
#include <stdbool.h>
typedef struct {
unsigned int package;
uint32_t processing_flags;
bool reprocess_on_dependency_change;
} rt_asset_settings;
rt_result rtParseAssetSettings(const char *text,
size_t length,
const char *file_path,
rt_asset_settings *settings);
rt_result rtLoadAssetSettings(const char *asset_path, rt_asset_settings *settings);
#endif

13
src/tools/assetc/compiled.h
View File

@ -1,13 +0,0 @@
#ifndef RT_ASSETC_COMPILED_H
#define RT_ASSETC_COMPILED_H
#include "runtime/runtime.h"
#include "runtime/assets.h"
rt_uid rtGetUID(const char *name);
void rtStoreOutput(rt_uid uid, const void *data, size_t size);
rt_result rtWriteCompiledFiles(void);
#endif

228
src/tools/assetc/dependency_tracking.c
View File

@ -1,228 +0,0 @@
#include "dependency_tracking.h"
#define RT_DEFINE_DEPENDENCY_FILE_STRUCTURES
#include "runtime/assets.h"
#include "runtime/threading.h"
#include "runtime/asset_dependencies.h"
#include <stdbool.h>
#include <limits.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
/* Track a list of dependencies per asset.
* For the runtime, we only care about dependent -> dependency (so for example material -> texture),
* but for assetc we also care about the opposite direction:
* If an asset has the reprocess_on_dependency_change setting enabled,
* we need to queue a reprocessing, even if the asset itself did not change.
*/
#define END_OF_LIST 0
/* 64 byte cache line - 8 (next index + count) / 4 (u32) = 14 */
#define BUCKET_ENTRY_COUNT 14
typedef struct rt_dep_list_bucket_s {
uint32_t next;
uint32_t count;
rt_uid entries[BUCKET_ENTRY_COUNT];
} rt_dep_list_bucket;
typedef union {
/* Indices of the first buckets */
struct {
uint32_t dependencies;
uint32_t dependents;
};
uint32_t lists[2];
} rt_dep_list;
static rt_mutex *_guard;
static rt_dep_list_bucket *_buckets;
static uint32_t _bucket_count;
static uint32_t _bucket_capacity;
#define MAP_SIZE 2048
static rt_uid _uids[MAP_SIZE];
static rt_dep_list _lists[MAP_SIZE];
rt_result rtInitDependencyTracking(void) {
_guard = rtCreateMutex();
if (!_guard)
return RT_UNKNOWN_ERROR;
_buckets = malloc(sizeof(rt_dep_list_bucket) * 256);
if (!_buckets)
return RT_UNKNOWN_ERROR;
_bucket_capacity = 256;
_bucket_count = 1;
return RT_SUCCESS;
}
static uint32_t AllocNewBucket(void) {
if (_bucket_count == _bucket_capacity) {
void *t = realloc(_buckets, (size_t)_bucket_capacity * 2 * sizeof(rt_dep_list_bucket));
if (!t)
return 0;
_buckets = t;
_bucket_capacity *= 2;
}
memset(&_buckets[_bucket_count], 0, sizeof(rt_dep_list_bucket));
return _bucket_count++;
}
static rt_result InsertIntoList(rt_uid list_asset, rt_uid uid, int list_index) {
rtLockMutex(_guard);
bool inserted = false;
for (uint32_t i = 0; i < MAP_SIZE; i++) {
uint32_t at = (list_asset + i) % MAP_SIZE;
if (_uids[at] == list_asset || _uids[at] == 0) {
_uids[at] = list_asset;
/* Alloc a new list, if necessary */
if (_lists[at].lists[list_index] == 0) {
_lists[at].lists[list_index] = AllocNewBucket();
if (!_lists[at].lists[list_index]) {
rtUnlockMutex(_guard);
return RT_UNKNOWN_ERROR;
}
}
/* Advance to the end of the list */
rt_dep_list_bucket *bucket = &_buckets[_lists[at].lists[list_index]];
while (bucket->next != END_OF_LIST) {
bucket = &_buckets[bucket->next];
}
/* Grow the list, if necessary */
if (bucket->count == BUCKET_ENTRY_COUNT) {
bucket->next = AllocNewBucket();
if (!bucket->next) {
rtUnlockMutex(_guard);
return RT_UNKNOWN_ERROR;
}
bucket = &_buckets[bucket->next];
}
assert(bucket->count < BUCKET_ENTRY_COUNT);
bucket->entries[bucket->count++] = uid;
inserted = true;
break;
}
}
rtUnlockMutex(_guard);
assert(inserted);
return RT_SUCCESS;
}
rt_result rtAddAssetDependency(rt_uid dependent, rt_uid dependency) {
rt_result res = InsertIntoList(dependent, dependency, 0);
if (res != RT_SUCCESS)
return res;
res = InsertIntoList(dependency, dependent, 1);
return res;
}
rt_result rtSaveAssetDependencies(void) {
assert(rtIsMainThread());
rt_dependency_file_header header = {.num_lists = 0, .data_size = 0};
for (size_t i = 0; i < MAP_SIZE; ++i) {
if (_uids[i] != RT_INVALID_UID) {
if (!_lists[i].dependencies)
continue;
header.num_lists += 1;
/* Determine the list size */
rt_dep_list_bucket *bucket = &_buckets[_lists[i].dependencies];
uint32_t total_list_size = bucket->count;
while (bucket->next != END_OF_LIST) {
bucket = &_buckets[bucket->next];
total_list_size += bucket->count;
}
header.data_size += total_list_size * sizeof(rt_uid) + sizeof(rt_dependency_file_list_header);
}
}
FILE *f = fopen("data/deps.bin", "wb");
if (!f) {
rtReportError("ASSETC", "Failed to open 'deps.bin' for writing.");
return RT_UNKNOWN_ERROR;
}
if (fwrite(&header, sizeof(header), 1, f) != 1) {
rtReportError("ASSETC", "Failed to write to 'deps.bin'.");
fclose(f);
return RT_UNKNOWN_ERROR;
}
void *buffer = NULL;
size_t buffer_size = 0;
for (size_t i = 0; i < MAP_SIZE; ++i) {
if (_uids[i] != RT_INVALID_UID) {
if (!_lists[i].dependencies)
continue;
/* Determine the list size */
rt_dep_list_bucket *bucket = &_buckets[_lists[i].dependencies];
uint32_t total_list_size = bucket->count;
while (bucket->next != END_OF_LIST) {
bucket = &_buckets[bucket->next];
total_list_size += bucket->count;
}
/* Allocate */
size_t required_size =
total_list_size * sizeof(rt_uid) + sizeof(rt_dependency_file_list_header);
if (required_size > buffer_size) {
void *t = realloc(buffer, required_size);
if (!t) {
free(buffer);
fclose(f);
return RT_UNKNOWN_ERROR;
}
buffer = t;
buffer_size = required_size;
}
/* Fill header */
rt_dependency_file_list_header *list_header = buffer;
rt_uid *list = (rt_uid *)(list_header + 1);
list_header->uid = _uids[i];
list_header->num_entries = total_list_size;
/* Copy the list */
uint32_t at = 0;
bucket = &_buckets[_lists[i].dependencies];
do {
memcpy(&list[at], bucket->entries, sizeof(rt_uid) * bucket->count);
at += bucket->count;
bucket = &_buckets[bucket->next];
} while (bucket != &_buckets[END_OF_LIST]);
XXH64_hash_t hash = XXH3_64bits(list, sizeof(rt_uid) * total_list_size);
XXH64_canonicalFromHash(&list_header->checksum, hash);
if (fwrite(buffer, required_size, 1, f) != 1) {
rtReportError("ASSETC", "Failed to write to 'deps.bin'.");
fclose(f);
free(buffer);
return RT_UNKNOWN_ERROR;
}
}
}
fclose(f);
free(buffer);
return RT_SUCCESS;
}

14
src/tools/assetc/dependency_tracking.h
View File

@ -1,14 +0,0 @@
#ifndef RT_ASSETC_DEPENDENCY_TRACKING_H
#define RT_ASSETC_DEPENDENCY_TRACKING_H
#include "runtime/runtime.h"
#include "runtime/assets.h"
rt_result rtInitDependencyTracking(void);
rt_result rtAddAssetDependency(rt_uid dependent, rt_uid dependency);
rt_result rtSaveAssetDependencies(void);
#endif

252
src/tools/assetc/description_parser.c
View File

@ -1,252 +0,0 @@
#include "description_parser.h"
#include "runtime/runtime.h"
#include <stdbool.h>
#include <limits.h>
#include <stdlib.h>
extern int memcmp(const void *s1, const void *s2, size_t n);
static bool IsAllowedChar(char c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
(c == '.') || (c == '_') || (c == '/');
}
static bool IsWhitespace(char c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r';
}
static void SkipWhitespace(rt_parse_state *state) {
while (state->at < state->length && IsWhitespace(state->text[state->at])) {
if (state->text[state->at] == '\n')
++state->line;
++state->at;
}
}
static bool ParseAttribute(rt_parse_state *state, rt_text_span *_name) {
rt_text_span name;
name.start = &state->text[state->at];
name.length = 0;
while (state->at < state->length && !IsWhitespace(state->text[state->at])) {
if (IsAllowedChar(state->text[state->at])) {
++state->at;
++name.length;
} else {
rtReportError("GFX",
"%s:%d Unexpected character %c",
state->file,
state->line,
state->text[state->at]);
return false;
}
}
*_name = name;
return true;
}
static bool ParseValue(rt_parse_state *state, rt_text_span *_value) {
rt_text_span value;
value.start = &state->text[state->at];
value.length = 0;
while (state->at < state->length && !IsWhitespace(state->text[state->at]) &&
state->text[state->at] != ';') {
if (IsAllowedChar(state->text[state->at])) {
++state->at;
++value.length;
} else {
rtReportError("GFX",
"%s:%d Unexpected character %c",
state->file,
state->line,
state->text[state->at]);
return false;
}
}
*_value = value;
return true;
}
#define BLOCK_BEGIN "BEGIN"
#define BLOCK_BEGIN_LENGTH 5
#define BLOCK_END "END"
#define BLOCK_END_LENGTH 3
RT_INLINE static bool IsBlockBegin(rt_parse_state *state) {
return (state->length - state->at >= BLOCK_BEGIN_LENGTH) &&
(memcmp(&state->text[state->at], BLOCK_BEGIN, BLOCK_BEGIN_LENGTH) == 0);
}
RT_INLINE static bool IsBlockEnd(rt_parse_state *state) {
return (state->length - state->at >= BLOCK_END_LENGTH) &&
(memcmp(&state->text[state->at], BLOCK_END, BLOCK_END_LENGTH) == 0);
}
static bool ParseBlock(rt_parse_state *state, rt_text_span *p_value) {
rt_text_span value;
value.start = &state->text[state->at];
value.length = 0;
while (state->at < state->length) {
if (state->text[state->at] == BLOCK_END[0] && IsBlockEnd(state)) {
*p_value = value;
return true;
}
++value.length;
++state->at;
}
/* did not find END */
return false;
}
static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index);
static bool ParseStmt(rt_parse_state *state, unsigned int *stmt_index) {
rt_parsed_stmt stmt;
stmt.next = UINT_MAX; /* end of list */
SkipWhitespace(state);
if (!ParseAttribute(state, &stmt.attribute))
return false;
SkipWhitespace(state);
if (state->at == state->length) {
rtReportError("GFX", "%s:%d Expected either a value or '{'", state->file, state->line);
return false;
}
if (state->text[state->at] == '{') {
/* Consume '{' */
++state->at;
stmt.form = RT_STMT_FORM_LIST;
if (!ParseStmtList(state, &stmt.list_index))
return false;
/* Consume '}' */
if (state->at < state->length && state->text[state->at] == '}')
++state->at;
} else if (IsBlockBegin(state)) {
/* Consume BEGIN */
state->at += BLOCK_BEGIN_LENGTH;
stmt.form = RT_STMT_FORM_BLOCK;
if (!ParseBlock(state, &stmt.block))
return false;
/* Consume END */
state->at += BLOCK_END_LENGTH;
} else {
stmt.form = RT_STMT_FORM_VALUE;
if (!ParseValue(state, &stmt.value))
return false;
/* Consume ';' */
if (state->at < state->length && state->text[state->at] == ';')
++state->at;
}
SkipWhitespace(state);
/* Add statement to array */
if (state->statement_count == state->statement_capacity) {
unsigned int cap = (state->statement_capacity > 0) ? state->statement_capacity * 2 : 64;
rt_parsed_stmt *temp = realloc(state->statements, sizeof(rt_parsed_stmt) * cap);
if (!temp) {
rtReportError("GFX", "While parsing %s: Out of memory\n", state->file);
return false;
}
state->statements = temp;
state->statement_capacity = cap;
}
state->statements[state->statement_count] = stmt;
*stmt_index = state->statement_count++;
return true;
}
static bool ParseStmtList(rt_parse_state *state, unsigned int *list_index) {
rt_parsed_stmt_list list;
list.first = UINT_MAX;
list.count = 0;
unsigned int last = UINT_MAX;
while (state->at < state->length && state->text[state->at] != '}') {
unsigned int stmt;
if (!ParseStmt(state, &stmt))
return false;
if (last != UINT_MAX)
state->statements[last].next = stmt;
else
list.first = stmt;
last = stmt;
++list.count;
}
/* Add list to array */
if (state->statement_list_count == state->statement_list_capacity) {
unsigned int cap =
(state->statement_list_capacity > 0) ? state->statement_list_capacity * 2 : 64;
rt_parsed_stmt_list *temp =
realloc(state->statement_lists, sizeof(rt_parsed_stmt_list) * cap);
if (!temp) {
rtReportError("GFX", "While parsing %s: Out of memory\n", state->file);
return false;
}
state->statement_lists = temp;
state->statement_list_capacity = cap;
}
state->statement_lists[state->statement_list_count] = list;
*list_index = state->statement_list_count++;
return true;
}
const rt_parsed_stmt * rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute) {
if (list_index >= state->statement_list_count)
return NULL;
const rt_parsed_stmt_list *list = &state->statement_lists[list_index];
unsigned int stmt_index = list->first;
for (unsigned int i = 0; i < list->count; ++i) {
const rt_parsed_stmt *stmt = &state->statements[stmt_index];
if (rtCompareSpanToString(stmt->attribute, attribute) == 0)
return stmt;
stmt_index = stmt->next;
}
return NULL;
}
rt_result rtParseDescription(const char *text,
size_t length,
const char *file_path,
unsigned int *_root_list,
rt_parse_state *_state) {
rt_parse_state state = {.text = text,
.at = 0,
.length = length,
.line = 1,
.file = file_path,
.statements = NULL,
.statement_lists = NULL,
.statement_capacity = 0,
.statement_list_capacity = 0};
unsigned int root_list = 0;
if (!ParseStmtList(&state, &root_list)) {
return 1;
}
*_root_list = root_list;
*_state = state;
return RT_SUCCESS;
}
void rtReleaseParseState(rt_parse_state *state) {
free(state->statements);
free(state->statement_lists);
}

55
src/tools/assetc/description_parser.h
View File

@ -1,55 +0,0 @@
#ifndef RT_ASSETC_DESCRIPTION_PARSER_H
#define RT_ASSETC_DESCRIPTION_PARSER_H
#include "runtime/runtime.h"
typedef enum {
RT_STMT_FORM_VALUE,
RT_STMT_FORM_LIST,
RT_STMT_FORM_BLOCK,
} rt_stmt_form;
typedef struct {
unsigned int first;
unsigned int count;
} rt_parsed_stmt_list;
typedef struct {
rt_stmt_form form;
rt_text_span attribute;
union {
rt_text_span value;
rt_text_span block;
unsigned int list_index;
};
/* For lists */
unsigned int next;
} rt_parsed_stmt;
typedef struct {
const char *file;
const char *text;
size_t at;
size_t length;
int line;
rt_parsed_stmt *statements;
unsigned int statement_count;
unsigned int statement_capacity;
rt_parsed_stmt_list *statement_lists;
unsigned int statement_list_count;
unsigned int statement_list_capacity;
} rt_parse_state;
rt_result rtParseDescription(const char *text,
size_t length,
const char *file_path,
unsigned int *root_list,
rt_parse_state *state);
const rt_parsed_stmt *rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute);
void rtReleaseParseState(rt_parse_state *state);
#endif

119
src/tools/assetc/discovery.c
View File

@ -1,119 +0,0 @@
#include "utils.h"
#include "assetmeta.h"
#include "processing.h"
#include "assetsettings.h"
#include "packages.h"
#include <string.h>
#include "runtime/uidtab.h"
typedef struct {
char path_scratch[1024];
unsigned int path_end;
} rt_discovery_data;
static rt_result LoadCompressedAsset(rt_uid uid, void **buffer, size_t size) {
}
static rt_result DirectoryHandler(const char *name, rtIterateDirElementType type, void *user) {
rt_discovery_data *data = user;
size_t name_len = strlen(name);
if (type == RT_DIR_ELEMENT_TYPE_FILE) {
/* Skip files we don't want to process */
if (name_len >= 3) {
if (memcmp(&name[name_len - 3], ".as", 3) == 0)
return RT_SUCCESS;
}
if (name_len >= 4) {
if (memcmp(&name[name_len - 4], ".pkg", 4) == 0)
return RT_SUCCESS;
else if (memcmp(&name[name_len - 4], ".bin", 4) == 0)
return RT_SUCCESS;
}
if (strcmp(name, "packages.txt") == 0)
return RT_SUCCESS;
if (name[0] == '.') {
return RT_SUCCESS;
}
/* Check if we know that file */
if (data->path_end > 0) {
data->path_scratch[data->path_end] = '/';
memcpy(data->path_scratch + data->path_end + 1, name, name_len);
data->path_scratch[data->path_end + 1 + name_len] = '\0';
} else {
memcpy(data->path_scratch, name, name_len);
data->path_scratch[name_len] = '\0';
}
rt_file_id fid = rtAddFile(data->path_scratch);
if (rtLookupUID(fid) != RT_INVALID_UID) {
rt_assetmeta meta = {0};
if (!rtGetAssetMeta(fid, &meta) || (meta.last_changed >= meta.compiled_ts)) {
/* The file (may have) changed */
rt_result res = rtAddFileToProcessingQueue(fid, meta.processing_flags);
if (res != RT_SUCCESS)
return res;
}
else {
/* The file is unchanged, we just need to add it to the output again. */
rtLog("ASSETC", "File %s is unchanged.", data->path_scratch);
rt_asset_settings settings = {0};
if (rtLoadAssetSettings(data->path_scratch, &settings) != RT_SUCCESS) {
rtLog("ASSETC", "Failed to load settings for %s", data->path_scratch);
return RT_UNKNOWN_ERROR;
}
/* We need to load the processed data and add it to the package */
rt_uid uid = rtLookupUID(fid);
if (uid == RT_INVALID_UID) {
rtLog("ASSETC", "Failed to lookup UID of known asset %s", data->path_scratch);
return RT_UNKNOWN_ERROR;
}
rtAddUnchangedAssetToPackage(settings.package, uid);
}
} else {
/* Process it */
rt_asset_settings settings = {0};
if (rtLoadAssetSettings(data->path_scratch, &settings) != RT_SUCCESS) {
rtLog("ASSETC", "Failed to load settings for %s", data->path_scratch);
}
rt_result res = rtAddFileToProcessingQueue(fid, settings.processing_flags);
if (res != RT_SUCCESS)
return res;
}
} else if (type == RT_DIR_ELEMENT_TYPE_DIRECTORY) {
if (strcmp(name, ".") == 0)
return RT_SUCCESS;
if (strcmp(name, "..") == 0)
return RT_SUCCESS;
unsigned int path_end_before = data->path_end;
if (data->path_end > 0)
data->path_scratch[data->path_end++] = '/';
memcpy(data->path_scratch + data->path_end, name, name_len);
data->path_scratch[data->path_end + name_len] = '\0';
data->path_end += (unsigned int)name_len;
rt_result res = rtIterateDirectory(data->path_scratch, user, DirectoryHandler);
if (res != RT_SUCCESS)
return res;
data->path_end = path_end_before;
}
return RT_SUCCESS;
}
void rtDiscoverAssets(void) {
rt_discovery_data data;
memcpy(data.path_scratch, "assets", sizeof("assets"));
data.path_end = sizeof("assets") - 1;
rtIterateDirectory("assets", &data, DirectoryHandler);
}

28
src/tools/assetc/options.h
View File

@ -1,28 +0,0 @@
#ifndef RT_ASSETC_OPTIONS_H
#define RT_ASSETC_OPTIONS_H
#include "runtime/assets.h"
typedef enum {
/* No optimization */
RT_ASSET_OPTIMIZATION_NONE,
/* Create small assets */
RT_ASSET_OPTIMIZATION_SPACE,
/* Create assets for fast execution */
RT_ASSET_OPTIMIZATION_PERFORMANCE,
} rt_asset_optimization_level;
/* Options parsed from command line arguments */
typedef struct {
const char *root_directory;
rt_renderer_backend_code renderer_backend;
rt_asset_optimization_level optimization;
} rt_assetc_options;
#ifndef RT_ASSETC_DONT_DEFINE_OPTIONS_GLOBAL
extern rt_assetc_options g_assetc_options;
#endif
#endif

274
src/tools/assetc/packages.c
View File

@ -1,274 +0,0 @@
#include "packages.h"
#include "processing.h"
#include "utils.h"
#define RT_DEFINE_PACKAGE_FILE_STRUCTURES
#include "runtime/threading.h"
#include "runtime/assets.h"
#include "runtime/file_tab.h"
#include "runtime/packages.h"
#include <assert.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdbool.h>
#include "xxhash/xxhash.h"
#include "lz4/lz4.h"
typedef struct {
rt_uid uid;
size_t disk_size;
} rt_package_entry;
typedef struct {
char *name;
unsigned int num_entries;
unsigned int entry_capacity;
rt_package_entry *entries;
} rt_package;
#define MAX_PACKAGES 1024
rt_package _packages[MAX_PACKAGES];
unsigned int _package_count = 0;
rt_mutex *_guard;
unsigned int rtAddPackageFile(rt_text_span name) {
rtLockMutex(_guard);
for (unsigned int i = 0; i < _package_count; ++i) {
if (rtCompareSpanToString(name, _packages[i].name + 5) == 0) {
rtUnlockMutex(_guard);
return i;
}
}
/* Create a new package */
_packages[_package_count].name = malloc(name.length + 1 + 5);
if (!_packages[_package_count].name) {
rtReportError("ASSETC",
"Failed to allocate storage for new package %*.s",
name.length,
name.start);
rtUnlockMutex(_guard);
return UINT_MAX;
}
memcpy(_packages[_package_count].name, "data/", 5);
memcpy(_packages[_package_count].name + 5, name.start, name.length);
_packages[_package_count].name[name.length + 5] = '\0';
unsigned int index = _package_count++;
rtUnlockMutex(_guard);
return index;
}
void rtInitPackages(void) {
_guard = rtCreateMutex();
/* Create the default package (0) */
rtAddPackageFile((rt_text_span){.start = "default.pkg", .length = 12});
}
static void AddAssetToPackageImpl(unsigned int package,
rt_uid uid,
void *buffer,
size_t size,
bool needs_compression) {
rtLockMutex(_guard);
if (package >= _package_count) {
rtReportError("ASSETC", "Trying to add an asset to a non-existing package.");
rtUnlockMutex(_guard);
return;
}
rt_package *pkg = &_packages[package];
for (unsigned int i = 0; i < pkg->num_entries; ++i) {
if (pkg->entries[i].uid == uid) {
rtUnlockMutex(_guard);
return;
}
}
if (pkg->num_entries == pkg->entry_capacity) {
unsigned int new_cap = (pkg->entry_capacity > 0) ? 2 * pkg->entry_capacity : 256;
rt_package_entry *n = realloc(pkg->entries, new_cap * sizeof(rt_package_entry));
if (!n) {
rtReportError("ASSETC", "Failed to grow storage for package %u.", package);
return;
}
pkg->entry_capacity = new_cap;
pkg->entries = n;
}
char tmp_path[256];
snprintf(tmp_path, 256, "actemp/%u.bin", uid);
if (needs_compression) {
FILE *tmp_f = fopen(tmp_path, "wb");
if (!tmp_f) {
rtUnlockMutex(_guard);
return;
}
int required_size = LZ4_compressBound((int)size);
void *compressed_buffer = malloc(required_size);
if (!compressed_buffer) {
fclose(tmp_f);
rtUnlockMutex(_guard);
return;
}
int compressed_bytes =
LZ4_compress_default(buffer, compressed_buffer, (int)size, required_size);
if (compressed_bytes == 0) {
free(compressed_buffer);
fclose(tmp_f);
rtReportError("ASSETC", "Failed to compress asset %x of package %s", uid, pkg->name);
return;
}
rt_package_asset_header header;
XXH64_hash_t checksum = XXH3_64bits_withSeed(buffer, (size_t)compressed_bytes, 0);
XXH64_canonicalFromHash(&header.checksum, checksum);
header.decompressed_size = (uint32_t)size;
if (fwrite(&header, sizeof(header), 1, tmp_f) != 1) {
rtReportError("ASSETC", "Failed to write to actemp/%u.bin", uid);
free(compressed_buffer);
fclose(tmp_f);
return;
}
if (fwrite(buffer, compressed_bytes, 1, tmp_f) != 1) {
rtReportError("ASSETC", "Failed to write to actemp/%u.bin", uid);
free(compressed_buffer);
fclose(tmp_f);
return;
}
fclose(tmp_f);
pkg->entries[pkg->num_entries].disk_size =
(size_t)compressed_bytes + sizeof(rt_package_asset_header);
} else {
pkg->entries[pkg->num_entries].disk_size = rtGetFileSize(tmp_path);
if (pkg->entries[pkg->num_entries].disk_size == 0) {
rtReportError("ASSETC", "Failed to determine size of actemp/%u.bin", uid);
}
}
pkg->entries[pkg->num_entries].uid = uid;
++pkg->num_entries;
rtUnlockMutex(_guard);
}
void rtAddAssetToPackage(unsigned int package, rt_uid uid, void *buffer, size_t size) {
AddAssetToPackageImpl(package, uid, buffer, size, true);
}
void rtAddUnchangedAssetToPackage(unsigned int package, rt_uid uid) {
AddAssetToPackageImpl(package, uid, NULL, 0, false);
}
static rt_result SavePackage(rt_package *pkg) {
if (pkg->num_entries == 0) {
rtLog("ASSETC", "Package %s has no entries.", pkg->name);
return RT_SUCCESS;
}
size_t current_buffer_size = 0;
void *buffer = NULL;
size_t offset_in_file = 0;
rt_file_id package_fid = rtAddFile(pkg->name);
FILE *f = fopen(pkg->name, "wb");
if (!f) {
rtReportError("ASSETC", "Failed to open %s for writing.", pkg->name);
return RT_UNKNOWN_ERROR;
}
for (unsigned int i = 0; i < pkg->num_entries; ++i) {
char tmp_path[256];
snprintf(tmp_path, 256, "actemp/%u.bin", pkg->entries[i].uid);
FILE *tmp_f = fopen(tmp_path, "rb");
if (!tmp_f) {
rtReportError("ASSETC", "Failed to open %s for reading.", tmp_path);
fclose(f);
free(buffer);
return RT_UNKNOWN_ERROR;
}
if (current_buffer_size < pkg->entries[i].disk_size) {
void *tmp = realloc(buffer, pkg->entries[i].disk_size);
if (!tmp) {
rtReportError("ASSETC", "Failed to allocate buffer (%zu bytes) for reading %s.", pkg->entries[i].disk_size, tmp_path);
fclose(f);
fclose(tmp_f);
free(buffer);
return RT_UNKNOWN_ERROR;
}
buffer = tmp;
current_buffer_size = pkg->entries[i].disk_size;
}
if (fread(buffer, pkg->entries[i].disk_size, 1, tmp_f) != 1) {
rtReportError("ASSETC", "Failed to read %s.", tmp_path);
fclose(f);
fclose(tmp_f);
free(buffer);
return RT_UNKNOWN_ERROR;
}
if (fwrite(buffer, pkg->entries[i].disk_size, 1, f) != 1) {
rtReportError("ASSETC", "Failed to write (%zu bytes) to %s.", pkg->entries[i].disk_size, pkg->name);
fclose(f);
fclose(tmp_f);
free(buffer);
return RT_UNKNOWN_ERROR;
}
rtAddUIDTabEntry(package_fid,
pkg->entries[i].uid,
offset_in_file,
pkg->entries[i].disk_size);
offset_in_file += pkg->entries[i].disk_size;
fclose(tmp_f);
}
free(buffer);
fclose(f);
return RT_SUCCESS;
}
rt_result rtSavePackages(void) {
assert(rtIsMainThread());
/* Save a .txt file with one line per package.
* Enables us to re-init the file-tab in future runs. */
FILE *f = fopen("data/packages.txt", "w");
if (!f) {
rtReportError("ASSETC", "Failed to write to 'packages.txt'");
return RT_UNKNOWN_ERROR;
}
for (unsigned int i = 0; i < _package_count; ++i) {
if (_packages[i].num_entries == 0)
continue;
fprintf(f, "%s\n", _packages[i].name);
}
fclose(f);
for (unsigned int i = 0; i < _package_count; ++i) {
rt_result res = SavePackage(&_packages[i]);
if (res != RT_SUCCESS)
return res;
}
return RT_SUCCESS;
}

16
src/tools/assetc/packages.h
View File

@ -1,16 +0,0 @@
#ifndef RT_ASSETC_PACKAGES_H
#define RT_ASSETC_PACKAGES_H
#include "runtime/runtime.h"
#include "runtime/assets.h"
void rtInitPackages(void);
unsigned int rtAddPackageFile(rt_text_span name);
void rtAddAssetToPackage(unsigned int package, rt_uid uid, void *buffer, size_t size);
void rtAddUnchangedAssetToPackage(unsigned int package, rt_uid uid);
rt_result rtSavePackages(void);
#endif

423
src/tools/assetc/pipeline_processor.c
View File

@ -1,423 +0,0 @@
#include <assert.h>
#include <limits.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "runtime/aio.h"
#include "runtime/gfx.h"
#include "runtime/handles.h"
#include "runtime/runtime.h"
#include "runtime/buffer_manager.h"
#include "runtime/renderer_api.h"
#include "assetmeta.h"
#include "description_parser.h"
#include "options.h"
#include "processing.h"
#include "utils.h"
#include "processing_flags.h"
#include "dependency_tracking.h"
typedef struct {
rt_attribute_binding *uniform_bindings;
rt_attribute_binding *storage_bindings;
rt_attribute_binding *texture_bindings;
rt_uid vertex_shader;
rt_uid fragment_shader;
rt_uid compute_shader;
/* TODO Fixed function settings */
/* Sampler settings */
uint16_t uniform_binding_count;
uint16_t storage_binding_count;
uint16_t texture_binding_count;
} rt_parsed_pipeline_data;
static void
DbgPrintShaderFile(const rt_parse_state *state, unsigned int list_index, unsigned int indent) {
assert(list_index < state->statement_list_count);
const rt_parsed_stmt_list *list = &state->statement_lists[list_index];
unsigned int stmt_index = list->first;
for (unsigned int i = 0; i < list->count; ++i) {
const rt_parsed_stmt *stmt = &state->statements[stmt_index];
for (unsigned int j = 0; j < indent; ++j)
printf(" ");
printf("%.*s: ", stmt->attribute.length, stmt->attribute.start);
if (stmt->form == RT_STMT_FORM_VALUE) {
printf("%.*s\n", stmt->value.length, stmt->value.start);
} else {
printf("{\n");
DbgPrintShaderFile(state, stmt->list_index, indent + 2);
printf("}\n");
}
stmt_index = stmt->next;
}
assert(stmt_index == UINT_MAX || stmt_index == 0);
}
static bool ParseBindingIndex(rt_text_span span, unsigned int *index) {
if (span.length == 0)
return false;
int at = (int)span.length - 1;
unsigned int exp = 1;
unsigned int n = 0;
while (at >= 0) {
if (span.start[at] >= '0' && span.start[at] <= '9') {
unsigned int digit = (unsigned int)(span.start[at] - '0');
n += digit * exp;
} else {
rtReportError("GFX", "Unexpected non-digit character in binding index");
return false;
}
--at;
exp *= 10;
}
*index = n;
return true;
}
static rt_attribute_value ParseBindingValue(rt_text_span span) {
if (rtCompareSpanToString(span, "MATERIAL_ALBEDO") == 0) {
return RT_ATTRIBUTE_VALUE_MATERIAL_ALBEDO;
} else if (rtCompareSpanToString(span, "MATERIAL_NORMAL") == 0) {
return RT_ATTRIBUTE_VALUE_MATERIAL_NORMAL;
}
rtReportError("GFX", "Unsupported binding value %*.s", span.length, span.start);
return RT_ATTRIBUTE_VALUE_UNDEFINED;
}
static bool ParseBindings(rt_parse_state *state,
unsigned int root_list,
const char *name,
const char *file_path,
rt_attribute_binding **p_bindings,
uint16_t *p_binding_count) {
const rt_parsed_stmt *bindings = rtFindStatement(state, root_list, name);
if (bindings) {
if (bindings->form != RT_STMT_FORM_LIST) {
rtReportError("GFX",
"Expected list of bindings as the value of "
"\"%s\" in %s",
name,
file_path);
return false;
}
const rt_parsed_stmt_list *binding_list = &state->statement_lists[bindings->list_index];
rt_attribute_binding *shader_bindings =
rtAllocBuffer(sizeof(rt_attribute_binding) * binding_list->count);
if (!bindings) {
rtReportError("GFX", "Out of memory");
return false;
}
unsigned int binding_count = binding_list->count;
unsigned int stmt_index = binding_list->first;
for (unsigned int i = 0; i < binding_list->count; ++i) {
const rt_parsed_stmt *stmt = &state->statements[stmt_index];
if (!ParseBindingIndex(stmt->attribute, &shader_bindings[i].index)) {
rtReleaseBuffer(shader_bindings,
sizeof(rt_attribute_binding) * binding_list->count);
return false;
}
shader_bindings[i].value = ParseBindingValue(stmt->value);
if (shader_bindings[i].value == RT_ATTRIBUTE_VALUE_UNDEFINED) {
rtReleaseBuffer(shader_bindings,
sizeof(rt_attribute_binding) * binding_list->count);
return false;
}
stmt_index = stmt->next;
}
*p_bindings = shader_bindings;
*p_binding_count = (uint16_t)binding_count;
return true;
} else {
*p_bindings = NULL;
*p_binding_count = 0;
return true;
}
}
static rt_result ParseShader(rt_parse_state *state,
unsigned int root_list,
const char *name,
const char *file_path,
uint32_t processing_flags,
rt_uid *p_shader_uid) {
const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, name);
if (stmt) {
if (stmt->form != RT_STMT_FORM_LIST) {
rtReportError("GFX",
"Expected a list as the value of "
"\"%s\" in %s",
name,
file_path);
return RT_PROCESSING_FAILED;
}
const rt_parsed_stmt_list *shader_list = &state->statement_lists[stmt->list_index];
unsigned int stmt_index = shader_list->first;
for (unsigned int i = 0; i < shader_list->count; ++i) {
const rt_parsed_stmt *shader = &state->statements[stmt_index];
if (shader->form != RT_STMT_FORM_VALUE) {
rtReportError("GFX",
"Expected a list as the value of "
"\"%s.%*s\" in %s",
name,
(int)shader->attribute.length,
shader->attribute.start,
file_path);
return RT_PROCESSING_FAILED;
}
rt_renderer_backend_code backend = RT_INVALID_RENDERER_BACKEND_CODE;
if (rtCompareSpanToString(shader->attribute, "vk") == 0) {
backend = RT_RENDERER_BACKEND_CODE_VK;
} else {
rtReportError("GFX",
"Invalid renderer backend"
"\"%*s\" in %s of file %s",
(int)shader->attribute.length,
shader->attribute.start,
name,
file_path);
return RT_PROCESSING_FAILED;
}
if (backend == g_assetc_options.renderer_backend) {
rt_file_id shader_file = rtAddFileFromSpan(shader->value);
rt_uid uid = rtLookupUID(shader_file);
if (uid == RT_INVALID_UID) {
/* Add the shader file to processing and wait until its done
*/
if (rtAddFileToProcessingQueue(shader_file, processing_flags) != RT_SUCCESS)
return RT_PROCESSING_FAILED;
return RT_PROCESSING_TRY_AGAIN;
}
*p_shader_uid = uid;
/* Don't break, because that validates the file. (attributes are checked for invalid
* values). */
}
stmt_index = shader->next;
}
return RT_SUCCESS;
}
return RT_PROCESSING_FAILED;
}
static uint32_t
ParseOptimizationLevel(rt_parse_state *state, unsigned int root_list, const char *file_path) {
uint32_t optimization_level;
switch (g_assetc_options.optimization) {
case RT_ASSET_OPTIMIZATION_PERFORMANCE:
optimization_level = RT_SHADER_FLAG_OPTIMIZE_SPEED;
break;
case RT_ASSET_OPTIMIZATION_SPACE:
optimization_level = RT_SHADER_FLAG_OPTIMIZE_SIZE;
break;
default:
optimization_level = 0;
}
const rt_parsed_stmt *stmt = rtFindStatement(state, root_list, "optimization");
if (stmt) {
if (stmt->form != RT_STMT_FORM_VALUE) {
rtReportError("GFX",
"Expected a simple statement for"
"\"optimization\" in %s",
file_path);
return optimization_level;
}
if (rtCompareSpanToString(stmt->value, "speed") == 0) {
optimization_level = RT_SHADER_FLAG_OPTIMIZE_SPEED;
} else if (rtCompareSpanToString(stmt->value, "size") == 0) {
optimization_level = RT_SHADER_FLAG_OPTIMIZE_SIZE;
} else if (rtCompareSpanToString(stmt->value, "none") == 0) {
optimization_level = 0;
} else {
rtReportError("GFX",
"Expected one of 'speed', 'size' and 'none' for \"optimization\" in %s",
file_path);
}
}
return optimization_level;
}
static rt_result
ParsePipelineFile(rt_file_id fid, const char *text, size_t length, rt_parsed_pipeline_data *pipeline) {
/* This is the grammar for pipeline files:
* <stmt-list> ::= <stmt>*
* <stmt> ::= <attribute> ( ( <value> ';' ) | ( '{' <stmt-list> '}' ) )
* <attribute> ::= [:alnum:]*
* <value>:: = [:alnum:]* */
const char *file_path = rtGetFilePath(fid);
rt_parse_state state;
unsigned int root_list;
rt_result result = rtParseDescription(text, length, file_path, &root_list, &state);
if (result != RT_SUCCESS) {
goto out;
}
/* We allow the pipeline file to overwrite the optimization level */
uint32_t optimization = ParseOptimizationLevel(&state, root_list, file_path);
/* Process shader stages */
if (ParseShader(&state,
root_list,
"vertex",
file_path,
RT_SHADER_FLAG_VERTEX | optimization,
&pipeline->vertex_shader) == RT_PROCESSING_TRY_AGAIN) {
result = RT_PROCESSING_TRY_AGAIN;
goto out;
}
if (ParseShader(&state,
root_list,
"fragment",
file_path,
RT_SHADER_FLAG_FRAGMENT | optimization,
&pipeline->fragment_shader) == RT_PROCESSING_TRY_AGAIN) {
result = RT_PROCESSING_TRY_AGAIN;
goto out;
}
if (ParseShader(&state,
root_list,
"compute",
file_path,
RT_SHADER_FLAG_COMPUTE | optimization,
&pipeline->compute_shader) == RT_PROCESSING_TRY_AGAIN) {
result = RT_PROCESSING_TRY_AGAIN;
goto out;
}
/* Process bindings */
pipeline->texture_bindings = NULL;
pipeline->texture_binding_count = 0;
pipeline->uniform_bindings = NULL;
pipeline->uniform_binding_count = 0;
pipeline->storage_bindings = NULL;
pipeline->storage_binding_count = 0;
if (!ParseBindings(&state,
root_list,
"texture_bindings",
file_path,
&pipeline->texture_bindings,
&pipeline->texture_binding_count)) {
result = RT_AIO_STATE_FAILED;
goto out;
}
if (!ParseBindings(&state,
root_list,
"uniform_bindings",
file_path,
&pipeline->uniform_bindings,
&pipeline->uniform_binding_count)) {
result = RT_AIO_STATE_FAILED;
goto out;
}
if (!ParseBindings(&state,
root_list,
"storage_bindings",
file_path,
&pipeline->storage_bindings,
&pipeline->storage_binding_count)) {
result = RT_AIO_STATE_FAILED;
goto out;
}
out:
free(state.statements);
free(state.statement_lists);
return result;
}
rt_result rtProcessPipelineFile(rt_file_id file,
void *buffer,
size_t size,
uint32_t flags,
rt_processor_output *output) {
RT_UNUSED(flags);
rt_parsed_pipeline_data tmp;
memset(&tmp, 0, sizeof(tmp));
rt_result result = ParsePipelineFile(file, buffer, size, &tmp);
if (result == RT_SUCCESS) {
/* parsed_pipeline_data contains arrays of bindings.
* We need to convert these to a flat buffer that can be written to a file */
size_t outbuffer_size =
sizeof(rt_pipeline_info) +
sizeof(rt_attribute_binding) *
(tmp.storage_binding_count + tmp.texture_binding_count + tmp.uniform_binding_count);
void *out_buffer = rtAllocBuffer(outbuffer_size);
if (!buffer) {
return RT_PROCESSING_FAILED;
}
rt_pipeline_info *info = out_buffer;
info->vertex_shader = tmp.vertex_shader;
info->fragment_shader = tmp.fragment_shader;
info->compute_shader = tmp.compute_shader;
info->texture_binding_count = tmp.texture_binding_count;
info->uniform_binding_count = tmp.uniform_binding_count;
info->storage_binding_count = tmp.storage_binding_count;
rt_attribute_binding *texture_bindings = NULL;
if (tmp.texture_binding_count > 0) {
texture_bindings = (rt_attribute_binding *)(info + 1);
memcpy(texture_bindings,
tmp.texture_bindings,
sizeof(rt_attribute_binding) * tmp.texture_binding_count);
}
rt_attribute_binding *uniform_bindings = NULL;
if (tmp.uniform_binding_count > 0) {
uniform_bindings = texture_bindings + tmp.texture_binding_count;
memcpy(uniform_bindings,
tmp.uniform_bindings,
sizeof(rt_attribute_binding) * tmp.uniform_binding_count);
}
rt_attribute_binding *storage_bindings = NULL;
if (tmp.storage_binding_count > 0) {
storage_bindings = uniform_bindings + tmp.uniform_binding_count;
memcpy(storage_bindings,
tmp.storage_bindings,
sizeof(rt_attribute_binding) * tmp.storage_binding_count);
}
rtSetRelptr(&info->texture_bindings, texture_bindings);
rtSetRelptr(&info->uniform_bindings, uniform_bindings);
rtSetRelptr(&info->storage_bindings, storage_bindings);
rtReleaseBuffer(tmp.texture_bindings,
sizeof(rt_attribute_binding) * tmp.texture_binding_count);
rtReleaseBuffer(tmp.storage_bindings,
sizeof(rt_attribute_binding) * tmp.storage_binding_count);
rtReleaseBuffer(tmp.uniform_bindings,
sizeof(rt_attribute_binding) * tmp.uniform_binding_count);
output->data = out_buffer;
output->size = outbuffer_size;
output->asset_uid = rtCalculateUID(rtGetFilePath(file));
/* Store dependencies to shaders */
if (info->vertex_shader != RT_INVALID_UID)
result = rtAddAssetDependency(output->asset_uid, info->vertex_shader);
if (info->fragment_shader != RT_INVALID_UID)
result = rtAddAssetDependency(output->asset_uid, info->fragment_shader);
if (info->compute_shader != RT_INVALID_UID)
result = rtAddAssetDependency(output->asset_uid, info->compute_shader);
}
return result;
}

46
src/tools/assetc/processing.h
View File

@ -1,46 +0,0 @@
#ifndef RT_ASSETC_PROCESSING_H
#define RT_ASSETC_PROCESSING_H
#include "runtime/assets.h"
#include "runtime/file_tab.h"
enum {
RT_PROCESSING_FAILED = RT_CUSTOM_ERROR_START,
/* Used if the processing depends on other files beeing processed first. */
RT_PROCESSING_TRY_AGAIN,
};
typedef struct {
rt_file_id output_file;
} rt_asset_options;
typedef struct {
rt_uid asset_uid;
/* Allocated via the rtAllocBuffer API */
void *data;
size_t size;
} rt_processor_output;
typedef rt_result rt_processor_fn(rt_file_id file,
void *buffer,
size_t size,
uint32_t flags,
rt_processor_output *output);
rt_result rtAddAssetProcessor(const char *file_extension, rt_processor_fn fn);
/* Flags are file type specific */
rt_result rtAddFileToProcessingQueue(rt_file_id file, uint32_t flags);
rt_result rtStartProcessing(void);
void rtStopProcessing(void);
rt_uid rtCalculateUID(const char *name);
rt_result rtWriteUIDTab(void);
rt_result rtAddUIDTabEntry(rt_file_id package_fid, rt_uid uid, uint64_t offset, uint64_t size);
void rtWaitUntilProcessingIsFinished(void);
#endif

20
src/tools/assetc/processing_flags.h
View File

@ -1,20 +0,0 @@
#ifndef RT_ASSETC_PROCESSING_FLAGS_H
#define RT_ASSETC_PROCESSING_FLAGS_H
/* Shader processing flags */
enum {
/* Type is encoded in the lower bits */
RT_SHADER_FLAG_VERTEX = 0x01,
RT_SHADER_FLAG_FRAGMENT = 0x02,
RT_SHADER_FLAG_COMPUTE = 0x03,
RT_SHADER_FLAG_TYPE_MASK =
RT_SHADER_FLAG_VERTEX | RT_SHADER_FLAG_FRAGMENT | RT_SHADER_FLAG_COMPUTE,
RT_SHADER_FLAG_OPTIMIZE_SPEED = 0x04,
RT_SHADER_FLAG_OPTIMIZE_SIZE = 0x08,
RT_SHADER_FLAG_OPTIMIZE_MASK = RT_SHADER_FLAG_OPTIMIZE_SPEED | RT_SHADER_FLAG_OPTIMIZE_SIZE,
};
#endif

381
src/tools/assetc/processor.c
View File

@ -1,381 +0,0 @@
#include "processing.h"
#include "utils.h"
#include "description_parser.h"
#include "packages.h"
#include "assetmeta.h"
#include "assetsettings.h"
#include "runtime/aio.h"
#include "runtime/buffer_manager.h"
#include "runtime/file_tab.h"
#include "runtime/threading.h"
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include <stdbool.h>
typedef struct {
rt_file_id fid;
uint32_t flags;
/* How many times has this file been added? */
unsigned int turn;
} rt_file_processing_queue_entry;
#define QUEUE_LENGTH 1024
typedef struct {
rt_file_processing_queue_entry entries[QUEUE_LENGTH];
unsigned int head;
unsigned int tail;
} rt_file_processing_queue;
static rt_file_processing_queue _queues[2];
static rt_file_processing_queue *_processing_queue;
static rt_file_processing_queue *_retry_queue;
static rt_mutex *_guard;
static bool _keep_running;
/* A single file could have a lot of dependencies. */
#define MAX_TURNS 100
#define MAX_PROCESSING_THREADS 16
#define FORCE_SINGLE_THREAD 1
static unsigned int _num_processing_threads = 0;
static rt_thread *_processing_threads[MAX_PROCESSING_THREADS];
static unsigned int _processing_thread_count = 0;
static rt_result rtAddFileToProcessingQueueImpl(rt_file_processing_queue *queue,
rt_file_id file,
uint32_t flags,
unsigned int turn) {
rt_result result = RT_SUCCESS;
rtLog("ASSETC", "Adding %s to processing queue.", rtGetFilePath(file));
rt_file_processing_queue_entry entry = {
.fid = file,
.flags = flags,
.turn = turn,
};
if ((queue->head + 1) % QUEUE_LENGTH != queue->tail) {
unsigned int slot = queue->head;
queue->entries[slot] = entry;
queue->head = (queue->head + 1) % QUEUE_LENGTH;
} else {
rtReportError("ASSETC", "The processing queue is full!");
result = 1;
}
return result;
}
rt_result rtAddFileToProcessingQueue(rt_file_id file, uint32_t flags) {
assert(_guard != NULL);
rtLockMutex(_guard);
rt_result res = rtAddFileToProcessingQueueImpl(_processing_queue, file, flags, 1);
rtUnlockMutex(_guard);
return res;
}
#define MAX_PROCESSORS 256
static rt_processor_fn *_processor_fns[MAX_PROCESSORS];
static const char *_processor_exts[MAX_PROCESSORS];
static unsigned int _processor_count;
rt_result rtAddAssetProcessor(const char *file_extension, rt_processor_fn fn) {
/* Should only be called from main thread */
if (_processor_count == MAX_PROCESSORS) {
rtReportError("ASSETC", "Too many asset processor functions!");
return 1;
}
_processor_fns[_processor_count] = fn;
_processor_exts[_processor_count] = file_extension;
++_processor_count;
return RT_SUCCESS;
}
static void PopAndSwapSubmittedData(unsigned int at,
unsigned int *count,
rt_file_processing_queue_entry *queue_entries,
rt_aio_handle *handles,
void **buffers,
size_t *sizes) {
if (at < *count - 1) {
queue_entries[at] = queue_entries[*count - 1];
buffers[at] = buffers[*count - 1];
handles[at] = handles[*count - 1];
sizes[at] = sizes[*count - 1];
}
*count = *count - 1;
}
static rt_result ProcessLoadedFile(rt_file_processing_queue_entry entry, void *buffer, size_t size) {
/* Search for a matching processor function */
const char *path = rtGetFilePath(entry.fid);
size_t path_len = strlen(path);
for (unsigned int i = 0; i < _processor_count; ++i) {
size_t ext_len = strlen(_processor_exts[i]);
if (ext_len > path_len)
continue;
const char *path_end = &path[path_len - ext_len];
if (memcmp(path_end, _processor_exts[i], ext_len) == 0) {
/* Load the corresponding .as file.
* TODO: Using malloc here is probably relatively slow.
*/
rt_result res;
rt_asset_settings settings;
if ((res = rtLoadAssetSettings(path, &settings)) != RT_SUCCESS) {
return res;
}
/* Process the asset */
rt_processor_output out;
res = _processor_fns[i](entry.fid, buffer, size, entry.flags, &out);
if (res == RT_SUCCESS) {
/* Add the output to the appropriate package file */
rt_assetmeta meta;
meta.compiled_ts = rtGetCurrentTimestamp();
meta.last_changed = rtGetFileModificationTimestamp(entry.fid);
meta.processing_flags = entry.flags;
rtAddUIDMapping(entry.fid, out.asset_uid, &meta);
rtAddAssetToPackage(settings.package, out.asset_uid, out.data, out.size);
} else if (res == RT_PROCESSING_TRY_AGAIN) {
if (entry.turn < MAX_TURNS) {
rtLockMutex(_guard);
rtAddFileToProcessingQueueImpl(_retry_queue, entry.fid, entry.flags, entry.turn + 1);
rtUnlockMutex(_guard);
} else {
rtLog("ASSETC",
"File '%s' took too many turns to process: %u",
path,
entry.turn);
}
} else {
rtLog("ASSETC", "Failed to process file: %s (Result %u)", path, res);
}
return res;
}
}
rtLog("ASSETC", "No asset processor for file: %s", path);
return RT_UNKNOWN_ERROR;
}
static void ProcessingThread(void *_param) {
RT_UNUSED(_param);
rt_file_processing_queue_entry submitted_entries[RT_LOAD_BATCH_MAX_SIZE];
rt_aio_handle submitted_handles[RT_LOAD_BATCH_MAX_SIZE];
void *submitted_buffers[RT_LOAD_BATCH_MAX_SIZE];
size_t submitted_sizes[RT_LOAD_BATCH_MAX_SIZE];
unsigned int submitted_outstanding = 0;
while (_keep_running) {
rt_load_batch load_batch;
rt_file_processing_queue_entry load_entries[RT_LOAD_BATCH_MAX_SIZE];
void *load_buffers[RT_LOAD_BATCH_MAX_SIZE];
size_t load_sizes[RT_LOAD_BATCH_MAX_SIZE];
load_batch.num_loads = 0;
bool got_entry = false;
do {
got_entry = false;
rt_file_processing_queue_entry entry = {0};
rtLockMutex(_guard);
if (_processing_queue->head != _processing_queue->tail) {
entry = _processing_queue->entries[_processing_queue->tail];
_processing_queue->tail = (_processing_queue->tail + 1) % QUEUE_LENGTH;
got_entry = true;
} else if (load_batch.num_loads == 0) {
/* Switch the queues -> Retry all the entries that returned RT_PROCESSING_TRY_AGAIN */
if (_retry_queue->head != _retry_queue->tail) {
rt_file_processing_queue *tmp = _retry_queue;
_retry_queue = _processing_queue;
_processing_queue = tmp;
}
}
rtUnlockMutex(_guard);
/* Retry, if we did not get an entry */
if (!got_entry)
continue;
const char *path = rtGetFilePath(entry.fid);
if (!path) {
rtLog("ASSETC", "Invalid file id: %#x", entry.fid);
continue;
}
rtLog("ASSETC", "Processing %s", path);
size_t fsz = rtGetFileSize(path);
void *dest = rtAllocBuffer(fsz);
if (!dest) {
rtLog("ASSETC", "Ran out of memory for loading the file: %s", path);
continue;
}
memset(dest, 0, fsz);
load_sizes[load_batch.num_loads] = fsz;
load_buffers[load_batch.num_loads] = dest;
load_batch.loads[load_batch.num_loads].file = entry.fid;
load_batch.loads[load_batch.num_loads].dest = dest;
load_batch.loads[load_batch.num_loads].num_bytes = fsz;
load_batch.loads[load_batch.num_loads].offset = 0;
load_entries[load_batch.num_loads] = entry;
++load_batch.num_loads;
} while (got_entry && load_batch.num_loads < RT_LOAD_BATCH_MAX_SIZE);
rt_aio_handle load_handles[RT_LOAD_BATCH_MAX_SIZE];
if (load_batch.num_loads > 0) {
rt_result submit_result = rtSubmitLoadBatch(&load_batch, load_handles);
if (submit_result != RT_SUCCESS) {
rtLog("ASSETC", "SubmitLoadBatch failed: %u", submit_result);
continue;
}
}
/* Process the previously submitted loads */
while (submitted_outstanding > 0) {
rtLockMutex(_guard);
_processing_thread_count += 1;
rtUnlockMutex(_guard);
for (unsigned int i = 0; i < submitted_outstanding; ++i) {
rt_aio_state state = rtGetAIOState(submitted_handles[i]);
switch (state) {
case RT_AIO_STATE_PENDING:
continue;
case RT_AIO_STATE_FAILED:
rtLog("ASSETC",
"Loading file %s failed.",
rtGetFilePath(submitted_entries[i].fid));
rtReleaseAIO(submitted_handles[i]);
rtReleaseBuffer(submitted_buffers[i], submitted_sizes[i]);
PopAndSwapSubmittedData(i,
&submitted_outstanding,
submitted_entries,
submitted_handles,
submitted_buffers,
submitted_sizes);
--i;
break;
case RT_AIO_STATE_INVALID:
rtLog("ASSETC",
"Got invalid AIO handle for file: %s",
rtGetFilePath(submitted_entries[i].fid));
rtReleaseBuffer(submitted_buffers[i], submitted_sizes[i]);
PopAndSwapSubmittedData(i,
&submitted_outstanding,
submitted_entries,
submitted_handles,
submitted_buffers,
submitted_sizes);
--i;
break;
case RT_AIO_STATE_FINISHED:
ProcessLoadedFile(submitted_entries[i],
submitted_buffers[i],
submitted_sizes[i]);
rtReleaseAIO(submitted_handles[i]);
rtReleaseBuffer(submitted_buffers[i], submitted_sizes[i]);
PopAndSwapSubmittedData(i,
&submitted_outstanding,
submitted_entries,
submitted_handles,
submitted_buffers,
submitted_sizes);
--i;
}
}
rtLockMutex(_guard);
_processing_thread_count -= 1;
rtUnlockMutex(_guard);
}
/* Start new round */
assert(sizeof(submitted_entries) == sizeof(load_entries));
assert(sizeof(submitted_handles) == sizeof(load_handles));
assert(sizeof(submitted_buffers) == sizeof(load_buffers));
assert(sizeof(submitted_sizes) == sizeof(load_sizes));
memcpy(submitted_entries, load_entries, sizeof(submitted_entries));
memcpy(submitted_handles, load_handles, sizeof(submitted_handles));
memcpy(submitted_buffers, load_buffers, sizeof(submitted_buffers));
memcpy(submitted_sizes, load_sizes, sizeof(submitted_sizes));
submitted_outstanding = load_batch.num_loads;
}
}
rt_result rtStartProcessing(void) {
if (!_guard)
_guard = rtCreateMutex();
#if !FORCE_SINGLE_THREAD
_num_processing_threads = rtGetCPUCoreCount();
if (_num_processing_threads > MAX_PROCESSING_THREADS)
_num_processing_threads = MAX_PROCESSING_THREADS;
#else
_num_processing_threads = 1;
#endif
_processing_queue = &_queues[0];
_retry_queue = &_queues[1];
_keep_running = true;
for (unsigned int i = 0; i < _num_processing_threads; ++i) {
_processing_threads[i] = rtSpawnThread(ProcessingThread, NULL, "Processor");
if (!_processing_threads[i]) {
rtReportError("ASSETC", "Failed to spawn processing thread %u!", i);
_keep_running = false;
return 1;
}
}
return RT_SUCCESS;
}
void rtStopProcessing(void) {
_keep_running = false;
for (unsigned int i = 0; i < _num_processing_threads; ++i)
rtJoinThread(_processing_threads[i]);
}
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#else
#include <unistd.h>
#endif
void rtWaitUntilProcessingIsFinished(void) {
unsigned int done_counter = 0;
while (done_counter < 3) {
#ifdef _WIN32
Sleep(200);
#else
sleep(1);
#endif
rtLockMutex(_guard);
volatile bool done = _processing_queue->head == _processing_queue->tail &&
_retry_queue->head == _retry_queue->tail &&
_processing_thread_count == 0;
rtUnlockMutex(_guard);
if (done)
++done_counter;
else
done_counter = 0;
}
}

123
src/tools/assetc/shader_processor.c
View File

@ -1,123 +0,0 @@
#include "assetmeta.h"
#include "processing.h"
#include "processing_flags.h"
#include "utils.h"
#include "runtime/buffer_manager.h"
#include <shaderc/shaderc.h>
#include <string.h>
static shaderc_include_result *ResolveInclude(void *user_data,
const char *requested_source,
int type,
const char *requesting_source,
size_t include_depth) {
shaderc_include_result *result = NULL;
return result;
}
static void ReleaseIncludeResult(void *user_data, shaderc_include_result *result) {
}
rt_result rtProcessShaderFile(rt_file_id file,
void *buffer,
size_t size,
uint32_t flags,
rt_processor_output *output) {
/* If we determine that shader compilation takes too long, we can instead
* keep the compiler around */
shaderc_compiler_t compiler = shaderc_compiler_initialize();
if (!compiler) {
rtLog("ASSETC", "Failed to initialize the shaderc compiler.");
return RT_PROCESSING_FAILED;
}
const char *path = rtGetFilePath(file);
shaderc_compile_options_t options = shaderc_compile_options_initialize();
if (!options) {
rtLog("ASSETC", "Failed to initialize shader compile options.");
shaderc_compiler_release(compiler);
return RT_PROCESSING_FAILED;
}
shaderc_compile_options_set_include_callbacks(options,
ResolveInclude,
ReleaseIncludeResult,
NULL);
uint32_t optimize_from_flags = flags & RT_SHADER_FLAG_OPTIMIZE_MASK;
if (optimize_from_flags == RT_SHADER_FLAG_OPTIMIZE_SPEED)
shaderc_compile_options_set_optimization_level(options,
shaderc_optimization_level_performance);
else if (optimize_from_flags == RT_SHADER_FLAG_OPTIMIZE_SIZE)
shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_size);
else
shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_zero);
uint32_t type_from_flags = flags & RT_SHADER_FLAG_TYPE_MASK;
shaderc_shader_kind shaderc_kind;
switch (type_from_flags) {
case RT_SHADER_FLAG_VERTEX:
shaderc_kind = shaderc_glsl_vertex_shader;
break;
case RT_SHADER_FLAG_FRAGMENT:
shaderc_kind = shaderc_glsl_fragment_shader;
break;
case RT_SHADER_FLAG_COMPUTE:
shaderc_kind = shaderc_glsl_compute_shader;
break;
default:
rtLog("ASSETC", "Invalid shader stage flag %u", type_from_flags);
shaderc_compile_options_release(options);
shaderc_compiler_release(compiler);
return RT_PROCESSING_FAILED;
}
shaderc_compilation_result_t result = shaderc_compile_into_spv(compiler,
(const char *)buffer,
size,
shaderc_kind,
path,
"main",
options);
shaderc_compilation_status status = shaderc_result_get_compilation_status(result);
if (status != shaderc_compilation_status_success || shaderc_result_get_num_errors(result) > 0) {
rtLog("ASSETC",
"Compilation of %s failed: %s",
path,
shaderc_result_get_error_message(result));
shaderc_result_release(result);
shaderc_compile_options_release(options);
shaderc_compiler_release(compiler);
return RT_PROCESSING_FAILED;
}
size_t output_size = shaderc_result_get_length(result);
output->data = rtAllocBuffer(output_size);
if (!output->data) {
shaderc_result_release(result);
shaderc_compile_options_release(options);
shaderc_compiler_release(compiler);
rtLog("ASSETC", "Failed to allocate %zu bytes for shader compilation output.", output_size);
return RT_PROCESSING_FAILED;
}
const uint32_t *spv_bytes = (uint32_t *)shaderc_result_get_bytes(result);
memcpy(output->data, spv_bytes, output_size);
output->size = output_size;
shaderc_result_release(result);
shaderc_compile_options_release(options);
shaderc_compiler_release(compiler);
output->asset_uid = rtCalculateUID(path);
return RT_SUCCESS;
}

0
src/tools/assetc/uidmap.c
View File

69
src/tools/assetc/uidtable.c
View File

@ -1,69 +0,0 @@
#include "processing.h"
#include "utils.h"
#define RT_DEFINE_UIDTAB_FILE_STRUCTURES
#include "runtime/threading.h"
#include "runtime/uidtab.h"
#include <xxhash/xxhash.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
static rt_uidtab_entry *_entries;
static size_t _entry_capacity;
static size_t _entry_count;
rt_uid rtCalculateUID(const char *name) {
assert(sizeof(XXH32_hash_t) == sizeof(rt_uid));
size_t len = strlen(name);
return (rt_uid)XXH32(name, len, 0);
}
rt_result rtAddUIDTabEntry(rt_file_id package_fid, rt_uid uid, uint64_t offset, uint64_t size) {
assert(rtIsMainThread());
if (_entry_count == _entry_capacity) {
size_t new_cap = (_entry_capacity > 0) ? _entry_capacity * 2 : 256;
void *t = realloc(_entries, sizeof(rt_uidtab_entry) * new_cap);
if (!t)
return RT_UNKNOWN_ERROR;
_entry_capacity = new_cap;
_entries = t;
}
_entries[_entry_count].file = package_fid;
_entries[_entry_count].offset = offset;
_entries[_entry_count].size = size;
_entries[_entry_count].uid = uid;
_entry_count++;
return RT_SUCCESS;
}
rt_result rtWriteUIDTab(void) {
rt_uidtab_header header;
XXH64_hash_t checksum = XXH3_64bits(_entries, sizeof(rt_uidtab_entry) * _entry_count);
XXH64_canonicalFromHash(&header.checksum, checksum);
header.num_entries = (uint32_t)_entry_count;
FILE *f = fopen("data/uidtab.bin", "wb");
if (!f) {
rtReportError("ASSETC", "Failed to open 'uidtab.bin' for writing.");
return RT_UNKNOWN_ERROR;
}
if (fwrite(&header, sizeof(header), 1, f) != 1) {
rtReportError("ASSETC", "Failed to write header to 'uidtab.bin'");
fclose(f);
return RT_UNKNOWN_ERROR;
}
if (fwrite(_entries, sizeof(rt_uidtab_entry), _entry_count, f) != _entry_count) {
rtReportError("ASSETC", "Failed to write entries to 'uidtab.bin'");
fclose(f);
return RT_UNKNOWN_ERROR;
}
fclose(f);
return RT_SUCCESS;
}

101
src/tools/assetc/utils.c
View File

@ -1,101 +0,0 @@
#include "utils.h"
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#endif
size_t rtGetFileSize(const char *path) {
#ifdef _WIN32
WCHAR wpath[MAX_PATH];
MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH);
WIN32_FILE_ATTRIBUTE_DATA attribs;
if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs))
return 0;
return (size_t)attribs.nFileSizeHigh << 32 | (size_t)attribs.nFileSizeLow;
#endif
}
void rtSetWorkingDirectory(const char *path) {
#ifdef _WIN32
WCHAR wpath[MAX_PATH];
MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH);
SetCurrentDirectoryW(wpath);
#endif
}
uint64_t rtGetCurrentTimestamp(void) {
#ifdef _WIN32
FILETIME ft;
GetSystemTimeAsFileTime(&ft);
uint64_t ts = ft.dwLowDateTime;
ts |= (uint64_t)ft.dwHighDateTime << 32;
return ts;
#endif
}
uint64_t rtGetFileModificationTimestamp(rt_file_id fid) {
#ifdef _WIN32
const char *path = rtGetFilePath(fid);
if (!path) {
rtLog("ASSETC", "Tried to get modification timestamp of unknown file.");
return 0;
}
WCHAR wpath[MAX_PATH];
MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, path, -1, wpath, MAX_PATH);
WIN32_FILE_ATTRIBUTE_DATA attribs;
if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs))
return 0;
uint64_t ts = attribs.ftLastWriteTime.dwLowDateTime;
ts |= (uint64_t)attribs.ftLastWriteTime.dwHighDateTime << 32;
return ts;
#endif
}
rt_result rtIterateDirectory(const char *path, void *user, rt_iterate_directory_cb_func iterate_cb) {
#ifdef _WIN32
rt_result res;
WCHAR wpath[MAX_PATH];
char wildcard_path[MAX_PATH];
strncpy(wildcard_path, path, MAX_PATH);
strncat(wildcard_path, "\\*", MAX_PATH - strlen(path));
res = rtUTF8ToWStr(wildcard_path, wpath, MAX_PATH);
if (res != RT_SUCCESS)
return res;
WIN32_FIND_DATAW find;
HANDLE h = FindFirstFileW(wpath, &find);
if (h == INVALID_HANDLE_VALUE)
return RT_UNKNOWN_ERROR;
do {
char utf8_file[MAX_PATH];
res = rtWStrToUTF8(find.cFileName, utf8_file, MAX_PATH);
if (res != RT_SUCCESS)
break;
rtIterateDirElementType type;
if (find.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
type = RT_DIR_ELEMENT_TYPE_DIRECTORY;
else
type = RT_DIR_ELEMENT_TYPE_FILE;
res = iterate_cb(utf8_file, type, user);
if (res != RT_SUCCESS)
break;
} while (FindNextFileW(h, &find) != 0);
FindClose(h);
return res;
#endif
}
rt_result rtCreateDirectory(const char *path) {
#ifdef _WIN32
WCHAR wpath[MAX_PATH];
rt_result res = rtUTF8ToWStr(path, wpath, MAX_PATH);
if (res != RT_SUCCESS)
return res;
if (!CreateDirectoryW(wpath, NULL))
res = RT_UNKNOWN_ERROR;
return res;
#endif
}

27
src/tools/assetc/utils.h
View File

@ -1,27 +0,0 @@
#ifndef RT_ASSETC_UTILS_H
#define RT_ASSETC_UTILS_H
#include <stdint.h>
#include "runtime/file_tab.h"
size_t rtGetFileSize(const char *path);
void rtSetWorkingDirectory(const char *path);
uint64_t rtGetCurrentTimestamp(void);
uint64_t rtGetFileModificationTimestamp(rt_file_id fid);
typedef enum {
RT_DIR_ELEMENT_TYPE_FILE,
RT_DIR_ELEMENT_TYPE_DIRECTORY,
} rtIterateDirElementType;
typedef rt_result rt_iterate_directory_cb_func(const char *name, rtIterateDirElementType type, void *user);
rt_result rtIterateDirectory(const char *path, void *user, rt_iterate_directory_cb_func iterate_cb);
rt_result rtCreateDirectory(const char *path);
#endif