kevin/rtengine
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

Started building the asset compiler

Browse Source
This commit is contained in:
Kevin Trogant 2023-11-24 00:43:35 +01:00
parent 360bf6cf1c
commit 9f3f40249f
10 changed files with 416 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
meson.build
View File

@ -56,10 +56,10 @@ runtime_lib = library('vyrt',
'src/runtime/aio.h',
'src/runtime/file_tab.h',
'src/runtime/buffer_manager.h',
'src/runtime/assets.h',
'src/runtime/error_report.c',
'src/runtime/gfx_main.c',
'src/runtime/gfx_shader_loading.c',
'src/runtime/config.c',
'src/runtime/runtime_cvars.c',
'src/runtime/threading_mutex.c',
@ -112,7 +112,20 @@ if vk_dep.found()
static_renderer_lib = vk_renderer_lib
endif
# Asset Compiler Tool
executable('assetc',
'src/tools/assetc/processing.h',
'src/tools/assetc/utils.h',
'src/tools/assetc/assetc.c',
'src/tools/assetc/processor.c',
'src/tools/assetc/shader_processor.c',
'src/tools/assetc/utils.c',
include_directories : incdir,
link_with : [runtime_lib],
win_subsystem : 'console')
# Game
game_link_libs = []
if get_option('default_library') == 'static'
game_link_libs = [runtime_lib, static_renderer_lib]

9
src/runtime/assets.h Normal file
View File

@ -0,0 +1,9 @@
#ifndef VY_ASSETS_H
#define VY_ASSETS_H
#include <stdint.h>
/* Unique identifier for an asset. */
typedef uint32_t vy_uid;
#endif

1
src/runtime/gfx.h
View File

@ -39,6 +39,7 @@ typedef enum {
VY_ATTRIBUTE_VALUE_UNDEFINED,
VY_ATTRIBUTE_VALUE_MATERIAL_ALBEDO,
VY_ATTRIBUTE_VALUE_MATERIAL_NORMAL,
} vy_attribute_value;
typedef struct {

8
src/runtime/gfx_main.c
View File

@ -21,9 +21,6 @@ VY_CVAR_S(rt_Renderer,
"Select the render backend. Available options: [vk], Default: vk",
"vk");
extern bool
vyLoadShaders(const char **paths, vy_shader *shaders, unsigned int count);
#ifdef VY_STATIC_LIB
extern void VY_RENDERER_API_FN(RegisterCVars)(void);
extern vy_result VY_RENDERER_API_FN(Init)(const vy_renderer_init_info *);
@ -98,11 +95,6 @@ VY_DLLEXPORT bool vyInitGFX(vy_renderer_init_info *renderer_info) {
if (g_renderer.Init(renderer_info) != VY_SUCCESS)
return false;
/* Init shader programs */
const char *shader_files[] = {"shader/cell.shader"};
vy_shader shaders[1];
if (!vyLoadShaders(shader_files, shaders, 1))
return false;
return true;
}

40
src/tools/assetc/assetc.c Normal file
View File

@ -0,0 +1,40 @@
#include "processing.h"
#include "runtime/aio.h"
#include "runtime/buffer_manager.h"
#include <stdlib.h>
extern vy_result vyProcessShaderFile(vy_file_id file,
void *buffer,
size_t size,
vy_processor_output *output);
int main(int argc, char **argv) {
/* Setup */
if (vyInitFileTab(4096) != VY_SUCCESS) {
return 1;
}
if (vyInitAIO(256) != VY_SUCCESS) {
return 1;
}
if (vyInitBufferManager() != VY_SUCCESS) {
return 1;
}
if (vyAddAssetProcessor(".shader", vyProcessShaderFile) != VY_SUCCESS)
return 1;
if (vyStartProcessing() != VY_SUCCESS) {
return 1;
}
vyAddFileToProcessingQueue(vyAddFile("shader\\cell.shader"));
while (1)
_sleep(10);
vyStopProcessing();
vyShutdownFileTab();
return 0;
}

27
src/tools/assetc/processing.h Normal file
View File

@ -0,0 +1,27 @@
#ifndef VY_ASSETC_PROCESSING_H
#define VY_ASSETC_PROCESSING_H
#include "runtime/file_tab.h"
#include "runtime/assets.h"
enum {
VY_PROCESSING_FAILED = VY_SUCCESS + 1,
/* Used if the processing depends on other files beeing processed first. */
VY_PROCESSING_TRY_AGAIN,
};
typedef struct {
vy_uid asset_uid;
} vy_processor_output;
typedef vy_result vy_processor_fn(vy_file_id file, void *buffer, size_t size, vy_processor_output *output);
vy_result vyAddAssetProcessor(const char *file_extension, vy_processor_fn fn);
vy_result vyAddFileToProcessingQueue(vy_file_id file);
vy_result vyStartProcessing(void);
void vyStopProcessing(void);
#endif

276
src/tools/assetc/processor.c Normal file
View File

@ -0,0 +1,276 @@
#include "processing.h"
#include "utils.h"
#include "runtime/file_tab.h"
#include "runtime/threading.h"
#include "runtime/aio.h"
#include "runtime/buffer_manager.h"
#include <string.h>
#include <assert.h>
typedef struct {
vy_file_id fid;
/* How many times has this file been added? */
unsigned int turn;
} vy_file_processing_queue_entry;
#define QUEUE_LENGTH 1024
typedef struct {
vy_file_processing_queue_entry entries[QUEUE_LENGTH];
unsigned int head;
unsigned int tail;
} vy_file_processing_queue;
static vy_file_processing_queue _processing_queue;
static vy_mutex *_guard;
static bool _keep_running;
static vy_result vyAddFileToProcessingQueueImpl(vy_file_id file, unsigned int turn) {
if (!_guard)
_guard = vyCreateMutex();
vy_result result = VY_SUCCESS;
vy_file_processing_queue_entry entry = {
.fid = file,
.turn = turn,
};
vyLockMutex(_guard);
if ((_processing_queue.tail + 1) % QUEUE_LENGTH != _processing_queue.head) {
unsigned int slot = _processing_queue.head;
_processing_queue.entries[slot] = entry;
_processing_queue.head = (_processing_queue.head + 1) % QUEUE_LENGTH;
} else {
vyReportError("ASSETC", "The processing queue is full!");
result = 1;
}
vyUnlockMutex(_guard);
return result;
}
vy_result vyAddFileToProcessingQueue(vy_file_id file) {
return vyAddFileToProcessingQueueImpl(file, 1);
}
#define MAX_PROCESSORS 256
static vy_processor_fn *_processor_fns[MAX_PROCESSORS];
static const char *_processor_exts[MAX_PROCESSORS];
static unsigned int _processor_count;
vy_result vyAddAssetProcessor(const char *file_extension, vy_processor_fn fn) {
/* Should only be called from main thread */
if (_processor_count == MAX_PROCESSORS) {
vyReportError("ASSETC", "Too many asset processor functions!");
return 1;
}
_processor_fns[_processor_count] = fn;
_processor_exts[_processor_count] = file_extension;
++_processor_count;
return VY_SUCCESS;
}
static void PopAndSwapSubmittedData(unsigned int at,
unsigned int *count,
vy_file_processing_queue_entry *queue_entries,
vy_aio_handle *handles,
void **buffers,
size_t *sizes) {
if (at < *count - 1) {
queue_entries[at] = queue_entries[*count - 1];
buffers[at] = buffers[*count - 1];
handles[at] = handles[*count - 1];
sizes[at] = sizes[*count - 1];
}
*count = *count - 1;
}
static void ProcessLoadedFile(vy_file_processing_queue_entry entry, void *buffer, size_t size) {
/* Search for a matching processor function */
const char *path = vyGetFilePath(entry.fid);
size_t path_len = strlen(path);
for (unsigned int i = 0; i < _processor_count; ++i) {
size_t ext_len = strlen(_processor_exts[i]);
if (ext_len > path_len)
continue;
const char *path_end = &path[path_len - ext_len];
if (memcmp(path_end, _processor_exts[i], ext_len) == 0) {
vy_processor_output out;
vy_result res = _processor_fns[i](entry.fid, buffer, size, &out);
if (res == VY_PROCESSING_FAILED) {
vyLog("ASSETC", "Failed to process file: %s", path);
} else if (res == VY_PROCESSING_TRY_AGAIN) {
if (entry.turn < 2) {
vyAddFileToProcessingQueueImpl(entry.fid, entry.turn + 1);
} else {
vyLog("ASSETC",
"File '%s' took too many turns to process: %u",
path,
entry.turn);
}
}
continue;
}
}
vyLog("ASSETC", "No asset processor for file: %s", path);
}
static void ProcessingThread(void *_param) {
VY_UNUSED(_param);
vy_file_processing_queue_entry submitted_entries[VY_LOAD_BATCH_MAX_SIZE];
vy_aio_handle submitted_handles[VY_LOAD_BATCH_MAX_SIZE];
void *submitted_buffers[VY_LOAD_BATCH_MAX_SIZE];
size_t submitted_sizes[VY_LOAD_BATCH_MAX_SIZE];
unsigned int submitted_outstanding = 0;
while (_keep_running) {
vy_load_batch load_batch;
vy_file_processing_queue_entry load_entries[VY_LOAD_BATCH_MAX_SIZE];
void *load_buffers[VY_LOAD_BATCH_MAX_SIZE];
size_t load_sizes[VY_LOAD_BATCH_MAX_SIZE];
load_batch.num_loads = 0;
bool got_entry = false;
do {
got_entry = false;
vy_file_processing_queue_entry entry;
vyLockMutex(_guard);
if (_processing_queue.head != _processing_queue.tail) {
unsigned int next = _processing_queue.tail;
entry = _processing_queue.entries[next];
_processing_queue.tail =
(_processing_queue.tail + 1) % QUEUE_LENGTH;
got_entry = true;
}
vyUnlockMutex(_guard);
if (!got_entry)
continue;
const char *path = vyGetFilePath(entry.fid);
if (!path) {
vyLog("ASSETC", "Invalid file id: %#x", entry.fid);
continue;
}
size_t fsz = vyGetFileSize(path);
void *dest = vyAllocBuffer(fsz);
if (!dest) {
vyLog("ASSETC",
"Ran out of memory for loading the file: %s",
path);
continue;
}
load_sizes[load_batch.num_loads] = fsz;
load_buffers[load_batch.num_loads] = dest;
load_batch.loads[load_batch.num_loads].file = entry.fid;
load_batch.loads[load_batch.num_loads].dest = dest;
load_batch.loads[load_batch.num_loads].num_bytes = fsz;
load_batch.loads[load_batch.num_loads].offset = 0;
load_entries[load_batch.num_loads] = entry;
++load_batch.num_loads;
}
while (got_entry && load_batch.num_loads < VY_LOAD_BATCH_MAX_SIZE);
vy_aio_handle load_handles[VY_LOAD_BATCH_MAX_SIZE];
if (load_batch.num_loads > 0) {
vy_result submit_result =
vySubmitLoadBatch(&load_batch, load_handles);
if (submit_result != VY_SUCCESS) {
vyLog("ASSETC", "SubmitLoadBatch failed: %u", submit_result);
continue;
}
}
/* Process the previously submitted loads */
while (submitted_outstanding > 0) {
for (unsigned int i = 0; i < submitted_outstanding; ++i) {
vy_aio_state state = vyGetAIOState(submitted_handles[i]);
switch (state) {
case VY_AIO_STATE_PENDING:
continue;
case VY_AIO_STATE_FAILED:
vyLog("ASSETC",
"Loading file %s failed.",
vyGetFilePath(submitted_entries[i].fid));
PopAndSwapSubmittedData(i,
&submitted_outstanding,
submitted_entries,
submitted_handles,
submitted_buffers,
submitted_sizes);
--i;
break;
case VY_AIO_STATE_INVALID:
vyLog("ASSETC",
"Got invalid AIO handle for file: %s",
vyGetFilePath(submitted_entries[i].fid));
PopAndSwapSubmittedData(i,
&submitted_outstanding,
submitted_entries,
submitted_handles,
submitted_buffers,
submitted_sizes);
--i;
break;
case VY_AIO_STATE_FINISHED:
ProcessLoadedFile(submitted_entries[i],
submitted_buffers[i],
submitted_sizes[i]);
PopAndSwapSubmittedData(i,
&submitted_outstanding,
submitted_entries,
submitted_handles,
submitted_buffers,
submitted_sizes);
--i;
}
}
}
/* Start new round */
assert(sizeof(submitted_entries) == sizeof(load_entries));
assert(sizeof(submitted_handles) == sizeof(load_handles));
assert(sizeof(submitted_buffers) == sizeof(load_buffers));
assert(sizeof(submitted_sizes) == sizeof(load_sizes));
memcpy(submitted_entries, load_entries, sizeof(submitted_entries));
memcpy(submitted_handles, load_handles, sizeof(submitted_handles));
memcpy(submitted_buffers, load_buffers, sizeof(submitted_buffers));
memcpy(submitted_sizes, load_sizes, sizeof(submitted_sizes));
submitted_outstanding = load_batch.num_loads;
}
}
#define NUM_PROCESSING_THREADS 8
vy_thread *_processing_threads[NUM_PROCESSING_THREADS];
vy_result vyStartProcessing(void) {
if (!_guard)
_guard = vyCreateMutex();
_keep_running = true;
for (unsigned int i = 0; i < NUM_PROCESSING_THREADS; ++i) {
_processing_threads[i] = vySpawnThread(ProcessingThread, NULL);
if (!_processing_threads[i]) {
vyReportError("ASSETC", "Failed to spawn processing thread %u!", i);
_keep_running = false;
return 1;
}
}
return VY_SUCCESS;
}
void vyStopProcessing(void) {
_keep_running = false;
for (unsigned int i = 0; i < NUM_PROCESSING_THREADS; ++i)
vyJoinThread(_processing_threads[i]);
}

104
src/runtime/gfx_shader_loading.c → src/tools/assetc/shader_processor.c
View File

@ -1,8 +1,3 @@
/* TODO(Kevin):
* This should move into a standalone tool.
*/
#include <assert.h>
#include <limits.h>
#include <stdbool.h>
@ -11,11 +6,12 @@
#include <stdlib.h>
#include <string.h>
#include "aio.h"
#include "gfx.h"
#include "handles.h"
#include "renderer_api.h"
#include "runtime.h"
#include "runtime/aio.h"
#include "runtime/gfx.h"
#include "runtime/handles.h"
#include "runtime/runtime.h"
#include "processing.h"
typedef enum {
VY_STMT_FORM_VALUE,
@ -269,44 +265,6 @@ static const vy_parsed_stmt *FindStatement(const vy_parse_state *state,
return NULL;
}
#if 0
static vy_fio_handle DispatchFileRead(vy_text_span path) {
vy_file_id fid = vyAddFileFromSpan(path);
if (fid == 0)
return 0;
vy_fio_handle fio = vyEnqueueRead(fid);
return fio;
}
static vy_fio_handle DispatchShaderRead(const char *shader,
const vy_parse_state *state,
unsigned int root_list,
const char *file_path) {
const vy_parsed_stmt *path = FindStatement(state, root_list, shader);
if (!path) {
return 0;
}
if (path->form != VY_STMT_FORM_VALUE) {
vyReportError("GFX",
"Expected simple value for attribute \"%s\" in %s\n",
shader,
file_path);
return 0;
}
return DispatchFileRead(path->value);
}
#endif
static vy_gfx_pipeline_handle CreatePipeline(vy_parse_state *state,
const char *file_path,
unsigned int root_list) {
/* Process the data */
vy_aio_handle reads[3];
vy_load_batch read_batch = {.num_loads = 0};
return (vy_gfx_pipeline_handle){0};
}
static bool ParseBindingIndex(vy_text_span span, unsigned int *index) {
if (span.length == 0)
return false;
@ -332,6 +290,8 @@ static bool ParseBindingIndex(vy_text_span span, unsigned int *index) {
static vy_attribute_value ParseBindingValue(vy_text_span span) {
if (CompareSpanToString(span, "MATERIAL_ALBEDO")) {
return VY_ATTRIBUTE_VALUE_MATERIAL_ALBEDO;
} else if (CompareSpanToString(span, "MATERIAL_NORMAL")) {
return VY_ATTRIBUTE_VALUE_MATERIAL_NORMAL;
}
vyReportError("GFX",
"Unsupported binding value %*.s",
@ -420,11 +380,13 @@ ParseShaderFile(vy_file_id fid, const char *text, size_t length, vy_shader *shad
DbgPrintShaderFile(&state, root_list, 0);
#if 0
shader->pipeline = CreatePipeline(&state, file_path, root_list);
if (!VY_IS_HANDLE_VALID(shader->pipeline)) {
result = false;
goto out;
}
#endif
/* Process bindings */
shader->texture_bindings = NULL;
@ -470,42 +432,14 @@ out:
return result;
}
bool vyLoadShaders(const char **paths, vy_shader *shaders, unsigned int count) {
vy_aio_handle aios[64];
vy_file_id fids[64];
for (unsigned int i = 0; i < count; i += 64) {
unsigned int chunk_size = count - i;
if (chunk_size > 64)
chunk_size = 64;
vy_load_batch chunk;
chunk.num_loads = chunk_size;
for (unsigned int j = 0; j < chunk_size; ++j) {
vy_file_id fid = vyAddFile(paths[i + j]);
if (!fid)
return false;
fids[j] = fid;
/* Setup the chunk
chunk.loads[j].offset = 0;
chunk.loads[j].file = fid;
chunk.loads[j].
*/
vy_result vyProcessShaderFile(vy_file_id file,
void *buffer,
size_t size,
vy_processor_output *output) {
vy_shader tmp;
if (ParseShaderFile(file, buffer, size, &tmp)) {
return VY_SUCCESS;
} else {
return VY_PROCESSING_FAILED;
}
unsigned int remaining = chunk_size;
while (remaining > 0) {
for (unsigned int j = 0; j < chunk_size; ++j) {
switch (vyGetAIOState(aios[j])) {
case VY_AIO_STATE_FINISHED:
/*
ParseShaderFile(fids[j], fbuf, &shaders[i + j]);
vyFreeFileBuffer(fbuf);
*/
--remaining;
}
}
}
}
return true;
}

23
src/tools/assetc/utils.c Normal file
View File

@ -0,0 +1,23 @@
#include "utils.h"
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#endif
size_t vyGetFileSize(const char *path) {
#ifdef _WIN32
WCHAR wpath[MAX_PATH];
MultiByteToWideChar(CP_UTF8,
MB_PRECOMPOSED,
path,
-1,
wpath,
MAX_PATH);
WIN32_FILE_ATTRIBUTE_DATA attribs;
if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attribs))
return 0;
return (size_t)attribs.nFileSizeHigh << 32 | (size_t)attribs.nFileSizeLow;
#endif
}

6
src/tools/assetc/utils.h Normal file
View File

@ -0,0 +1,6 @@
#ifndef VY_ASSETC_UTILS_H
#define VY_ASSETC_UTILS_H
size_t vyGetFileSize(const char *path);
#endif