diff --git a/meson.build b/meson.build index bbd3984..8649bd9 100644 --- a/meson.build +++ b/meson.build @@ -56,10 +56,10 @@ runtime_lib = library('vyrt', 'src/runtime/aio.h', 'src/runtime/file_tab.h', 'src/runtime/buffer_manager.h', + 'src/runtime/assets.h', 'src/runtime/error_report.c', 'src/runtime/gfx_main.c', - 'src/runtime/gfx_shader_loading.c', 'src/runtime/config.c', 'src/runtime/runtime_cvars.c', 'src/runtime/threading_mutex.c', @@ -112,7 +112,20 @@ if vk_dep.found() static_renderer_lib = vk_renderer_lib endif +# Asset Compiler Tool +executable('assetc', + 'src/tools/assetc/processing.h', + 'src/tools/assetc/utils.h', + + 'src/tools/assetc/assetc.c', + 'src/tools/assetc/processor.c', + 'src/tools/assetc/shader_processor.c', + 'src/tools/assetc/utils.c', + include_directories : incdir, + link_with : [runtime_lib], + win_subsystem : 'console') +# Game game_link_libs = [] if get_option('default_library') == 'static' game_link_libs = [runtime_lib, static_renderer_lib] diff --git a/src/runtime/assets.h b/src/runtime/assets.h new file mode 100644 index 0000000..200a57b --- /dev/null +++ b/src/runtime/assets.h @@ -0,0 +1,9 @@ +#ifndef VY_ASSETS_H +#define VY_ASSETS_H + +#include + +/* Unique identifier for an asset. */ +typedef uint32_t vy_uid; + +#endif \ No newline at end of file diff --git a/src/runtime/gfx.h b/src/runtime/gfx.h index 5660c17..a25f57c 100644 --- a/src/runtime/gfx.h +++ b/src/runtime/gfx.h @@ -39,6 +39,7 @@ typedef enum { VY_ATTRIBUTE_VALUE_UNDEFINED, VY_ATTRIBUTE_VALUE_MATERIAL_ALBEDO, + VY_ATTRIBUTE_VALUE_MATERIAL_NORMAL, } vy_attribute_value; typedef struct { diff --git a/src/runtime/gfx_main.c b/src/runtime/gfx_main.c index db4c4d1..1c9214a 100644 --- a/src/runtime/gfx_main.c +++ b/src/runtime/gfx_main.c @@ -21,9 +21,6 @@ VY_CVAR_S(rt_Renderer, "Select the render backend. Available options: [vk], Default: vk", "vk"); -extern bool -vyLoadShaders(const char **paths, vy_shader *shaders, unsigned int count); - #ifdef VY_STATIC_LIB extern void VY_RENDERER_API_FN(RegisterCVars)(void); extern vy_result VY_RENDERER_API_FN(Init)(const vy_renderer_init_info *); @@ -98,11 +95,6 @@ VY_DLLEXPORT bool vyInitGFX(vy_renderer_init_info *renderer_info) { if (g_renderer.Init(renderer_info) != VY_SUCCESS) return false; - /* Init shader programs */ - const char *shader_files[] = {"shader/cell.shader"}; - vy_shader shaders[1]; - if (!vyLoadShaders(shader_files, shaders, 1)) - return false; return true; } diff --git a/src/tools/assetc/assetc.c b/src/tools/assetc/assetc.c new file mode 100644 index 0000000..615d208 --- /dev/null +++ b/src/tools/assetc/assetc.c @@ -0,0 +1,40 @@ +#include "processing.h" + +#include "runtime/aio.h" +#include "runtime/buffer_manager.h" + +#include + +extern vy_result vyProcessShaderFile(vy_file_id file, + void *buffer, + size_t size, + vy_processor_output *output); + +int main(int argc, char **argv) { + /* Setup */ + if (vyInitFileTab(4096) != VY_SUCCESS) { + return 1; + } + if (vyInitAIO(256) != VY_SUCCESS) { + return 1; + } + if (vyInitBufferManager() != VY_SUCCESS) { + return 1; + } + + if (vyAddAssetProcessor(".shader", vyProcessShaderFile) != VY_SUCCESS) + return 1; + + if (vyStartProcessing() != VY_SUCCESS) { + return 1; + } + + vyAddFileToProcessingQueue(vyAddFile("shader\\cell.shader")); + while (1) + _sleep(10); + + vyStopProcessing(); + vyShutdownFileTab(); + + return 0; +} \ No newline at end of file diff --git a/src/tools/assetc/processing.h b/src/tools/assetc/processing.h new file mode 100644 index 0000000..0e9e12e --- /dev/null +++ b/src/tools/assetc/processing.h @@ -0,0 +1,27 @@ +#ifndef VY_ASSETC_PROCESSING_H +#define VY_ASSETC_PROCESSING_H + +#include "runtime/file_tab.h" +#include "runtime/assets.h" + +enum { + VY_PROCESSING_FAILED = VY_SUCCESS + 1, + /* Used if the processing depends on other files beeing processed first. */ + VY_PROCESSING_TRY_AGAIN, +}; + +typedef struct { + vy_uid asset_uid; +} vy_processor_output; + +typedef vy_result vy_processor_fn(vy_file_id file, void *buffer, size_t size, vy_processor_output *output); + +vy_result vyAddAssetProcessor(const char *file_extension, vy_processor_fn fn); + +vy_result vyAddFileToProcessingQueue(vy_file_id file); + +vy_result vyStartProcessing(void); + +void vyStopProcessing(void); + +#endif diff --git a/src/tools/assetc/processor.c b/src/tools/assetc/processor.c new file mode 100644 index 0000000..21175fb --- /dev/null +++ b/src/tools/assetc/processor.c @@ -0,0 +1,276 @@ +#include "processing.h" +#include "utils.h" + +#include "runtime/file_tab.h" +#include "runtime/threading.h" +#include "runtime/aio.h" +#include "runtime/buffer_manager.h" + +#include +#include + +typedef struct { + vy_file_id fid; + + /* How many times has this file been added? */ + unsigned int turn; +} vy_file_processing_queue_entry; + +#define QUEUE_LENGTH 1024 +typedef struct { + vy_file_processing_queue_entry entries[QUEUE_LENGTH]; + unsigned int head; + unsigned int tail; +} vy_file_processing_queue; + +static vy_file_processing_queue _processing_queue; +static vy_mutex *_guard; +static bool _keep_running; + + +static vy_result vyAddFileToProcessingQueueImpl(vy_file_id file, unsigned int turn) { + if (!_guard) + _guard = vyCreateMutex(); + + vy_result result = VY_SUCCESS; + + vy_file_processing_queue_entry entry = { + .fid = file, + .turn = turn, + }; + vyLockMutex(_guard); + if ((_processing_queue.tail + 1) % QUEUE_LENGTH != _processing_queue.head) { + unsigned int slot = _processing_queue.head; + _processing_queue.entries[slot] = entry; + _processing_queue.head = (_processing_queue.head + 1) % QUEUE_LENGTH; + } else { + vyReportError("ASSETC", "The processing queue is full!"); + result = 1; + } + vyUnlockMutex(_guard); + return result; +} + + +vy_result vyAddFileToProcessingQueue(vy_file_id file) { + return vyAddFileToProcessingQueueImpl(file, 1); +} + + +#define MAX_PROCESSORS 256 +static vy_processor_fn *_processor_fns[MAX_PROCESSORS]; +static const char *_processor_exts[MAX_PROCESSORS]; +static unsigned int _processor_count; + +vy_result vyAddAssetProcessor(const char *file_extension, vy_processor_fn fn) { + /* Should only be called from main thread */ + if (_processor_count == MAX_PROCESSORS) { + vyReportError("ASSETC", "Too many asset processor functions!"); + return 1; + } + _processor_fns[_processor_count] = fn; + _processor_exts[_processor_count] = file_extension; + ++_processor_count; + return VY_SUCCESS; +} + +static void PopAndSwapSubmittedData(unsigned int at, + unsigned int *count, + vy_file_processing_queue_entry *queue_entries, + vy_aio_handle *handles, + void **buffers, + size_t *sizes) { + if (at < *count - 1) { + queue_entries[at] = queue_entries[*count - 1]; + buffers[at] = buffers[*count - 1]; + handles[at] = handles[*count - 1]; + sizes[at] = sizes[*count - 1]; + } + *count = *count - 1; +} + +static void ProcessLoadedFile(vy_file_processing_queue_entry entry, void *buffer, size_t size) { + /* Search for a matching processor function */ + const char *path = vyGetFilePath(entry.fid); + size_t path_len = strlen(path); + for (unsigned int i = 0; i < _processor_count; ++i) { + size_t ext_len = strlen(_processor_exts[i]); + if (ext_len > path_len) + continue; + + const char *path_end = &path[path_len - ext_len]; + if (memcmp(path_end, _processor_exts[i], ext_len) == 0) { + vy_processor_output out; + vy_result res = _processor_fns[i](entry.fid, buffer, size, &out); + if (res == VY_PROCESSING_FAILED) { + vyLog("ASSETC", "Failed to process file: %s", path); + } else if (res == VY_PROCESSING_TRY_AGAIN) { + if (entry.turn < 2) { + vyAddFileToProcessingQueueImpl(entry.fid, entry.turn + 1); + } else { + vyLog("ASSETC", + "File '%s' took too many turns to process: %u", + path, + entry.turn); + } + } + continue; + } + } + vyLog("ASSETC", "No asset processor for file: %s", path); +} + +static void ProcessingThread(void *_param) { + VY_UNUSED(_param); + + + vy_file_processing_queue_entry submitted_entries[VY_LOAD_BATCH_MAX_SIZE]; + vy_aio_handle submitted_handles[VY_LOAD_BATCH_MAX_SIZE]; + void *submitted_buffers[VY_LOAD_BATCH_MAX_SIZE]; + size_t submitted_sizes[VY_LOAD_BATCH_MAX_SIZE]; + unsigned int submitted_outstanding = 0; + + while (_keep_running) { + vy_load_batch load_batch; + vy_file_processing_queue_entry load_entries[VY_LOAD_BATCH_MAX_SIZE]; + void *load_buffers[VY_LOAD_BATCH_MAX_SIZE]; + size_t load_sizes[VY_LOAD_BATCH_MAX_SIZE]; + load_batch.num_loads = 0; + + bool got_entry = false; + do { + got_entry = false; + vy_file_processing_queue_entry entry; + vyLockMutex(_guard); + if (_processing_queue.head != _processing_queue.tail) { + unsigned int next = _processing_queue.tail; + entry = _processing_queue.entries[next]; + _processing_queue.tail = + (_processing_queue.tail + 1) % QUEUE_LENGTH; + got_entry = true; + } + vyUnlockMutex(_guard); + if (!got_entry) + continue; + + const char *path = vyGetFilePath(entry.fid); + if (!path) { + vyLog("ASSETC", "Invalid file id: %#x", entry.fid); + continue; + } + + size_t fsz = vyGetFileSize(path); + void *dest = vyAllocBuffer(fsz); + if (!dest) { + vyLog("ASSETC", + "Ran out of memory for loading the file: %s", + path); + continue; + } + + load_sizes[load_batch.num_loads] = fsz; + load_buffers[load_batch.num_loads] = dest; + load_batch.loads[load_batch.num_loads].file = entry.fid; + load_batch.loads[load_batch.num_loads].dest = dest; + load_batch.loads[load_batch.num_loads].num_bytes = fsz; + load_batch.loads[load_batch.num_loads].offset = 0; + load_entries[load_batch.num_loads] = entry; + ++load_batch.num_loads; + } + while (got_entry && load_batch.num_loads < VY_LOAD_BATCH_MAX_SIZE); + + vy_aio_handle load_handles[VY_LOAD_BATCH_MAX_SIZE]; + if (load_batch.num_loads > 0) { + vy_result submit_result = + vySubmitLoadBatch(&load_batch, load_handles); + if (submit_result != VY_SUCCESS) { + vyLog("ASSETC", "SubmitLoadBatch failed: %u", submit_result); + continue; + } + } + + + /* Process the previously submitted loads */ + while (submitted_outstanding > 0) { + for (unsigned int i = 0; i < submitted_outstanding; ++i) { + vy_aio_state state = vyGetAIOState(submitted_handles[i]); + switch (state) { + case VY_AIO_STATE_PENDING: + continue; + case VY_AIO_STATE_FAILED: + vyLog("ASSETC", + "Loading file %s failed.", + vyGetFilePath(submitted_entries[i].fid)); + PopAndSwapSubmittedData(i, + &submitted_outstanding, + submitted_entries, + submitted_handles, + submitted_buffers, + submitted_sizes); + --i; + break; + case VY_AIO_STATE_INVALID: + vyLog("ASSETC", + "Got invalid AIO handle for file: %s", + vyGetFilePath(submitted_entries[i].fid)); + PopAndSwapSubmittedData(i, + &submitted_outstanding, + submitted_entries, + submitted_handles, + submitted_buffers, + submitted_sizes); + --i; + break; + case VY_AIO_STATE_FINISHED: + ProcessLoadedFile(submitted_entries[i], + submitted_buffers[i], + submitted_sizes[i]); + PopAndSwapSubmittedData(i, + &submitted_outstanding, + submitted_entries, + submitted_handles, + submitted_buffers, + submitted_sizes); + --i; + } + } + } + + /* Start new round */ + assert(sizeof(submitted_entries) == sizeof(load_entries)); + assert(sizeof(submitted_handles) == sizeof(load_handles)); + assert(sizeof(submitted_buffers) == sizeof(load_buffers)); + assert(sizeof(submitted_sizes) == sizeof(load_sizes)); + memcpy(submitted_entries, load_entries, sizeof(submitted_entries)); + memcpy(submitted_handles, load_handles, sizeof(submitted_handles)); + memcpy(submitted_buffers, load_buffers, sizeof(submitted_buffers)); + memcpy(submitted_sizes, load_sizes, sizeof(submitted_sizes)); + submitted_outstanding = load_batch.num_loads; + } +} + +#define NUM_PROCESSING_THREADS 8 + +vy_thread *_processing_threads[NUM_PROCESSING_THREADS]; + +vy_result vyStartProcessing(void) { + if (!_guard) + _guard = vyCreateMutex(); + + _keep_running = true; + for (unsigned int i = 0; i < NUM_PROCESSING_THREADS; ++i) { + _processing_threads[i] = vySpawnThread(ProcessingThread, NULL); + if (!_processing_threads[i]) { + vyReportError("ASSETC", "Failed to spawn processing thread %u!", i); + _keep_running = false; + return 1; + } + } + return VY_SUCCESS; +} + +void vyStopProcessing(void) { + _keep_running = false; + for (unsigned int i = 0; i < NUM_PROCESSING_THREADS; ++i) + vyJoinThread(_processing_threads[i]); +} \ No newline at end of file diff --git a/src/runtime/gfx_shader_loading.c b/src/tools/assetc/shader_processor.c similarity index 83% rename from src/runtime/gfx_shader_loading.c rename to src/tools/assetc/shader_processor.c index 70b48c6..e65676f 100644 --- a/src/runtime/gfx_shader_loading.c +++ b/src/tools/assetc/shader_processor.c @@ -1,8 +1,3 @@ -/* TODO(Kevin): - * This should move into a standalone tool. - */ - - #include #include #include @@ -11,11 +6,12 @@ #include #include -#include "aio.h" -#include "gfx.h" -#include "handles.h" -#include "renderer_api.h" -#include "runtime.h" +#include "runtime/aio.h" +#include "runtime/gfx.h" +#include "runtime/handles.h" +#include "runtime/runtime.h" + +#include "processing.h" typedef enum { VY_STMT_FORM_VALUE, @@ -269,44 +265,6 @@ static const vy_parsed_stmt *FindStatement(const vy_parse_state *state, return NULL; } -#if 0 -static vy_fio_handle DispatchFileRead(vy_text_span path) { - vy_file_id fid = vyAddFileFromSpan(path); - if (fid == 0) - return 0; - vy_fio_handle fio = vyEnqueueRead(fid); - return fio; -} - -static vy_fio_handle DispatchShaderRead(const char *shader, - const vy_parse_state *state, - unsigned int root_list, - const char *file_path) { - const vy_parsed_stmt *path = FindStatement(state, root_list, shader); - if (!path) { - return 0; - } - if (path->form != VY_STMT_FORM_VALUE) { - vyReportError("GFX", - "Expected simple value for attribute \"%s\" in %s\n", - shader, - file_path); - return 0; - } - return DispatchFileRead(path->value); -} -#endif - -static vy_gfx_pipeline_handle CreatePipeline(vy_parse_state *state, - const char *file_path, - unsigned int root_list) { - /* Process the data */ - vy_aio_handle reads[3]; - vy_load_batch read_batch = {.num_loads = 0}; - - return (vy_gfx_pipeline_handle){0}; -} - static bool ParseBindingIndex(vy_text_span span, unsigned int *index) { if (span.length == 0) return false; @@ -332,6 +290,8 @@ static bool ParseBindingIndex(vy_text_span span, unsigned int *index) { static vy_attribute_value ParseBindingValue(vy_text_span span) { if (CompareSpanToString(span, "MATERIAL_ALBEDO")) { return VY_ATTRIBUTE_VALUE_MATERIAL_ALBEDO; + } else if (CompareSpanToString(span, "MATERIAL_NORMAL")) { + return VY_ATTRIBUTE_VALUE_MATERIAL_NORMAL; } vyReportError("GFX", "Unsupported binding value %*.s", @@ -420,11 +380,13 @@ ParseShaderFile(vy_file_id fid, const char *text, size_t length, vy_shader *shad DbgPrintShaderFile(&state, root_list, 0); + #if 0 shader->pipeline = CreatePipeline(&state, file_path, root_list); if (!VY_IS_HANDLE_VALID(shader->pipeline)) { result = false; goto out; } + #endif /* Process bindings */ shader->texture_bindings = NULL; @@ -470,42 +432,14 @@ out: return result; } -bool vyLoadShaders(const char **paths, vy_shader *shaders, unsigned int count) { - vy_aio_handle aios[64]; - vy_file_id fids[64]; - for (unsigned int i = 0; i < count; i += 64) { - unsigned int chunk_size = count - i; - if (chunk_size > 64) - chunk_size = 64; - vy_load_batch chunk; - chunk.num_loads = chunk_size; - for (unsigned int j = 0; j < chunk_size; ++j) { - vy_file_id fid = vyAddFile(paths[i + j]); - if (!fid) - return false; - fids[j] = fid; - - /* Setup the chunk - chunk.loads[j].offset = 0; - chunk.loads[j].file = fid; - chunk.loads[j]. - - */ - } - - unsigned int remaining = chunk_size; - while (remaining > 0) { - for (unsigned int j = 0; j < chunk_size; ++j) { - switch (vyGetAIOState(aios[j])) { - case VY_AIO_STATE_FINISHED: - /* - ParseShaderFile(fids[j], fbuf, &shaders[i + j]); - vyFreeFileBuffer(fbuf); - */ - --remaining; - } - } - } +vy_result vyProcessShaderFile(vy_file_id file, + void *buffer, + size_t size, + vy_processor_output *output) { + vy_shader tmp; + if (ParseShaderFile(file, buffer, size, &tmp)) { + return VY_SUCCESS; + } else { + return VY_PROCESSING_FAILED; } - return true; -} +} \ No newline at end of file diff --git a/src/tools/assetc/utils.c b/src/tools/assetc/utils.c new file mode 100644 index 0000000..4b15796 --- /dev/null +++ b/src/tools/assetc/utils.c @@ -0,0 +1,23 @@ +#include "utils.h" + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#endif + + +size_t vyGetFileSize(const char *path) { +#ifdef _WIN32 + WCHAR wpath[MAX_PATH]; + MultiByteToWideChar(CP_UTF8, + MB_PRECOMPOSED, + path, + -1, + wpath, + MAX_PATH); + WIN32_FILE_ATTRIBUTE_DATA attribs; + if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs)) + return 0; + return (size_t)attribs.nFileSizeHigh << 32 | (size_t)attribs.nFileSizeLow; +#endif +} \ No newline at end of file diff --git a/src/tools/assetc/utils.h b/src/tools/assetc/utils.h new file mode 100644 index 0000000..c664baf --- /dev/null +++ b/src/tools/assetc/utils.h @@ -0,0 +1,6 @@ +#ifndef VY_ASSETC_UTILS_H +#define VY_ASSETC_UTILS_H + +size_t vyGetFileSize(const char *path); + +#endif