started work on pipeline parsing

This commit is contained in:
Kevin Trogant 2024-01-16 16:03:30 +01:00
parent d5ab965d64
commit 448d448430
20 changed files with 255 additions and 208 deletions

1
assets/shader/test.as Normal file
View File

@ -0,0 +1 @@
package pipelines.pkg;

View File

@ -0,0 +1,7 @@
optimization speed;
vertex BEGIN
layout (location = 0) uniform vec3 foo;
void main() {
}
END

View File

@ -58,6 +58,7 @@ runtime_lib = library('vyrt',
'src/runtime/dynamic_libs.h',
'src/runtime/file_tab.h',
'src/runtime/gfx.h',
'src/runtime/handles.h',
'src/runtime/jobs.h',
'src/runtime/packages.h',
'src/runtime/renderer_api.h',
@ -113,7 +114,7 @@ if vk_dep.found()
'src/renderer/vk/init.c',
'src/renderer/vk/swapchain.c',
'src/renderer/vk/gfx_pipelines.c',
'src/renderer/vk/pipelines.c',
# Contrib Sources
'contrib/volk/volk.h',

View File

@ -1,3 +1,10 @@
#ifdef _WIN32
#include <Windows.h>
#endif
/* C standard library*/
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <math.h>

View File

@ -1,159 +0,0 @@
#include "runtime/gfx.h"
#include "runtime/runtime.h"
#include "runtime/renderer_api.h"
typedef struct {
unsigned int prog;
} vy_gl_pipeline;
#define NUM_SLOTS 256
typedef struct {
uint32_t generation_in_use[NUM_SLOTS];
vy_gl_pipeline pipelines[NUM_SLOTS];
} vy_pipeline_storage;
static vy_pipeline_storage _storage;
static vy_gfx_pipeline_handle StorePipeline(vy_gl_pipeline pipeline) {
/* Search for free slot */
uint32_t slot = NUM_SLOTS;
for (uint32_t i = 0; i < NUM_SLOTS; ++i) {
if ((_storage.generation_in_use[i] & 0x1) == 0) {
slot = i;
break;
}
}
if (slot == NUM_SLOTS) {
vyReportError("GL_GFX", "Ran out of pipeline storage slots");
return (vy_gfx_pipeline_handle){0};
}
uint32_t generation = _storage.generation_in_use[slot] >> 1;
generation = (generation + 1) & 0x1;
_storage.pipelines[slot] = pipeline;
_storage.generation_in_use[slot] = (generation << 1) | 0x1;
vy_gfx_pipeline_handle id;
id.index = (generation << 27) | slot;
return id;
}
static void ReleasePipelineSlot(vy_gfx_pipeline_handle id) {
uint32_t slot = id.index & 0x08ffffff;
uint32_t gen = (id.index >> 27) & 0x1f;
if (slot >= NUM_SLOTS)
return;
gen = gen << 1 | 0x1;
if (_storage.generation_in_use[slot] == gen)
_storage.generation_in_use[slot] &= ~0x1;
}
vy_gfx_pipeline_handle
VY_RENDERER_API_FN(CompileComputePipeline)(const vy_compute_pipeline_info *info) {
#if 0
char info_log[512];
GLuint prog = glCreateProgram();
GLuint shader = glCreateShader(GL_COMPUTE_SHADER);
GLchar *code = (GLchar *)info->compute_source;
GLint code_len = (GLint)info->compute_source_length;
glShaderSource(shader, 1, (const GLchar **)&code, &code_len);
glCompileShader(shader);
GLint success;
glGetShaderiv(shader, GL_COMPILE_STATUS, &success);
if (!success) {
glGetShaderInfoLog(shader, 512, NULL, info_log);
vyReportError("GL_GFX",
"Failed to compile compute shader\n%s",
info_log);
glDeleteProgram(prog);
glDeleteShader(shader);
return (vy_gfx_pipeline_handle){0};
}
glAttachShader(prog, shader);
glLinkProgram(prog);
glGetProgramiv(prog, GL_LINK_STATUS, &success);
if (!success) {
glGetProgramInfoLog(prog, 512, NULL, info_log);
vyReportError("GL_GFX", "Failed to link compute shader\n%s", info_log);
glDeleteShader(shader);
glDeleteProgram(prog);
return (vy_gfx_pipeline_handle){0};
}
glDeleteShader(shader);
#endif
vy_gl_pipeline pipeline;
pipeline.prog = 0;
return StorePipeline(pipeline);
}
vy_gfx_pipeline_handle
VY_RENDERER_API_FN(CompileGraphicsPipeline)(const vy_graphics_pipeline_info *info) {
#if 0
char info_log[512];
GLuint prog = glCreateProgram();
GLuint vertex_shader = glCreateShader(GL_VERTEX_SHADER);
GLuint fragment_shader = glCreateShader(GL_FRAGMENT_SHADER);
GLchar *code = (GLchar *)info->vertex_source;
GLint code_len = (GLint)info->vertex_source_length;
glShaderSource(vertex_shader, 1, (const GLchar **)&code, &code_len);
glCompileShader(vertex_shader);
GLint success;
glGetShaderiv(vertex_shader, GL_COMPILE_STATUS, &success);
if (!success) {
glGetShaderInfoLog(vertex_shader, 512, NULL, info_log);
vyReportError("GL_GFX",
"Failed to compile vertex shader\n%s",
info_log);
glDeleteShader(vertex_shader);
glDeleteShader(fragment_shader);
glDeleteProgram(prog);
return (vy_gfx_pipeline_handle){0};
}
glAttachShader(prog, vertex_shader);
code = (GLchar *)info->fragment_source;
code_len = (GLint)info->fragment_source_length;
glShaderSource(fragment_shader, 1, (const GLchar **)&code, &code_len);
glCompileShader(fragment_shader);
glGetShaderiv(fragment_shader, GL_COMPILE_STATUS, &success);
if (!success) {
glGetShaderInfoLog(fragment_shader, 512, NULL, info_log);
vyReportError("GFX", "Failed to compile fragment shader\n%s", info_log);
glDeleteShader(fragment_shader);
glDeleteShader(fragment_shader);
glDeleteProgram(prog);
return (vy_gfx_pipeline_handle){0};
}
glAttachShader(prog, fragment_shader);
glLinkProgram(prog);
glGetProgramiv(prog, GL_LINK_STATUS, &success);
if (!success) {
glGetProgramInfoLog(prog, 512, NULL, info_log);
vyReportError("GFX", "Failed to link graphics shader\n%s", info_log);
glDeleteShader(vertex_shader);
glDeleteShader(fragment_shader);
glDeleteProgram(prog);
return (vy_gfx_pipeline_handle){0};
}
glDeleteShader(vertex_shader);
glDeleteShader(fragment_shader);
#endif
vy_gl_pipeline pipeline;
pipeline.prog = 0;
return StorePipeline(pipeline);
}

View File

@ -420,6 +420,9 @@ static vy_result CreateDevice(void) {
return VY_SUCCESS;
}
extern vy_result InitPipelineManagement(void);
extern void ShutdownPipelineManagement(void);
vy_result VY_RENDERER_API_FN(Init)(const vy_renderer_init_info *info) {
vyLog("vk", "Init");
@ -444,6 +447,9 @@ vy_result VY_RENDERER_API_FN(Init)(const vy_renderer_init_info *info) {
if (res != VY_SUCCESS)
return res;
res = CreateDevice();
if (res != VY_SUCCESS)
return res;
res = InitPipelineManagement();
if (res != VY_SUCCESS)
return res;
res = vyCreateSwapchain();
@ -457,6 +463,7 @@ void VY_RENDERER_API_FN(Shutdown)(void) {
vyLog("vk", "Shutdown");
vkDeviceWaitIdle(g_gpu.device);
vyDestroySwapchain();
ShutdownPipelineManagement();
vkDestroyDevice(g_gpu.device, g_gpu.alloc_cb);
vkDestroySurfaceKHR(g_gpu.instance, g_gpu.surface, g_gpu.alloc_cb);
vkDestroyInstance(g_gpu.instance, g_gpu.alloc_cb);

103
src/renderer/vk/pipelines.c Normal file
View File

@ -0,0 +1,103 @@
#include "runtime/renderer_api.h"
#include "runtime/config.h"
#include "runtime/handles.h"
#include "runtime/threading.h"
#include "gpu.h"
#include "pipelines.h"
#include <volk/volk.h>
#include <stdlib.h>
VY_CVAR_I(r_VkMaxPipelineCount, "Maximum number of pipeline objects. Default: 1024", 1024);
typedef struct vy_pipeline_s {
uint32_t version;
vy_pipeline pipeline;
struct vy_pipeline_s *next_free;
} vy_pipeline_slot;
static vy_pipeline_slot *_pipelines;
static vy_pipeline_slot *_first_free;
static vy_rwlock _lock;
vy_result InitPipelineManagement(void) {
vy_create_rwlock_result lock_res = vyCreateRWLock();
if (!lock_res.ok)
return VY_UNKNOWN_ERROR;
_lock = lock_res.lock;
_pipelines = calloc(r_VkMaxPipelineCount.i, sizeof(vy_pipeline_slot));
if (!_pipelines) {
vyDestroyRWLock(&_lock);
return VY_OUT_OF_MEMORY;
}
/* Keep [0] unused to preserve 0 as the invalid handle */
_first_free = &_pipelines[1];
for (int i = 1; i < r_VkMaxPipelineCount.i - 1; ++i) {
_pipelines[i].next_free = &_pipelines[i + 1];
}
return VY_SUCCESS;
}
static void DestroyPipeline(vy_pipeline_slot *slot) {
if (slot->pipeline.pipeline) {
vkDestroyPipeline(g_gpu.device, slot->pipeline.pipeline, g_gpu.alloc_cb);
}
slot->next_free = _first_free;
_first_free = slot;
}
void ShutdownPipelineManagement(void) {
for (int i = 1; i < r_VkMaxPipelineCount.i; ++i) {
DestroyPipeline(&_pipelines[i]);
}
free(_pipelines);
vyDestroyRWLock(&_lock);
_first_free = NULL;
}
vy_pipeline_handle VY_RENDERER_API_FN(CompilePipeline)(const vy_pipeline_info *info) {
vy_pipeline_handle handle = VY_INVALID_HANDLE;
vyLockWrite(&_lock);
if (!_first_free) {
vyLog("VK", "No free pipeline slots!");
vyUnlockWrite(&_lock);
return handle;
}
vy_pipeline_slot *slot = _first_free;
_first_free = slot->next_free;
slot->version = (slot->version + 1) & VY_GFX_HANDLE_MAX_VERSION;
vyUnlockWrite(&_lock);
/* No other thread that calls compile gets the same slot.
* Another thread accessing the slot via GetPipeline would get a version mismatch.
* The same holds for DestroyPipeline
*/
return handle;
}
void VY_RENDERER_API_FN(DestroyPipeline)(vy_pipeline_handle handle) {
if (handle.index >= (uint32_t)r_VkMaxPipelineCount.i)
return;
vyLockWrite(&_lock);
if (_pipelines[handle.index].version == handle.version)
DestroyPipeline(&_pipelines[handle.index]);
else
vyLog("VK", "Tried to destroy a pipeline using an outdated handle.");
vyUnlockWrite(&_lock);
}
const vy_pipeline *vyGetPipeline(vy_pipeline_handle handle) {
if (handle.index >= (uint32_t)r_VkMaxPipelineCount.i)
return NULL;
vyLockRead(&_lock);
vy_pipeline *res = NULL;
if (_pipelines[handle.index].version == handle.version)
res = &_pipelines[handle.index].pipeline;
else
vyLog("VK", "Tried to access a pipeline using an outdated handle.");
vyUnlockRead(&_lock);
return res;
}

View File

@ -0,0 +1,15 @@
#ifndef VY_VK_PIPELINES_H
#define VY_VK_PIPELINES_H
#include <volk/volk.h>
#include "runtime/gfx.h"
typedef struct {
VkPipeline pipeline;
} vy_pipeline;
/* A pipeline is immutable after creation. */
const vy_pipeline *vyGetPipeline(vy_pipeline_handle handle);
#endif

View File

@ -318,40 +318,62 @@ VY_DLLEXPORT vy_get_asset_result vyGetAsset(vy_uid uid) {
}
}
vyLockRead(&_lock);
volatile vy_asset_cache_entry *entry = GetEntry(uid);
vyLockWrite(&_lock);
vy_asset_cache_entry *entry = GetEntry(uid);
if (entry) {
if (entry->state == CACHE_ENTRY_STATE_LOADED) {
++entry->refcount;
result.data = entry->buffer;
result.size = entry->size;
} else if (entry->state == CACHE_ENTRY_STATE_LOADING) {
/* Promote to write lock */
vyUnlockRead(&_lock);
vyLockWrite(&_lock);
if (entry->state == CACHE_ENTRY_STATE_LOADING) {
assert(entry->load != VY_AIO_INVALID_HANDLE);
++entry->refcount;
if (vyWaitForAIOCompletion(entry->load) == VY_AIO_STATE_FINISHED) {
if (DecompressEntry(uid, (vy_asset_cache_entry *)entry)) {
if (DecompressEntry(uid, entry)) {
result.data = entry->buffer;
result.size = entry->size;
} else {
result.result = VY_LOAD_FAILED;
}
} else {
ReleaseEntry((vy_asset_cache_entry *)entry);
ReleaseEntry(entry);
vyLog("ASSET_CACHE", "Failed to load asset %u", uid);
result.result = VY_LOAD_FAILED;
}
}
vyUnlockWrite(&_lock);
}
/* To match the unlock below */
vyLockRead(&_lock);
/* Remove from the reclaim list */
if (_first_reclaim == entry)
_first_reclaim = entry->next_reclaim;
if (_last_reclaim == entry)
_last_reclaim = entry->prev_reclaim;
if (entry->next_reclaim)
entry->next_reclaim->prev_reclaim = entry->prev_reclaim;
if (entry->prev_reclaim)
entry->prev_reclaim->next_reclaim = entry->next_reclaim;
}
}
vyUnlockRead(&_lock);
vyUnlockWrite(&_lock);
return result;
}
VY_DLLEXPORT void vyReleaseAsset(vy_uid uid) {
vyLockWrite(&_lock);
vy_asset_cache_entry *entry = GetEntry(uid);
if (entry && entry->refcount > 0) {
--entry->refcount;
if (entry->refcount == 0) {
/* add to the reclaim list */
if (_last_reclaim)
_last_reclaim->next_reclaim = entry;
if (!_first_reclaim)
_first_reclaim = entry;
entry->prev_reclaim = _last_reclaim;
entry->next_reclaim = NULL;
_last_reclaim = entry;
}
}
vyUnlockWrite(&_lock);
}

View File

@ -20,7 +20,7 @@ enum {
typedef uint8_t vy_renderer_backend_code;
enum {
VY_UNKNOWN_ASSET = VY_SUCCESS + 1,
VY_UNKNOWN_ASSET = VY_CUSTOM_ERROR_START,
VY_BUFFER_ALLOC_FAILED,
VY_LOAD_FAILED,
VY_ASSET_CACHE_FULL,

View File

@ -4,7 +4,7 @@
#include "runtime.h"
enum {
VY_BUFFER_MGR_OUT_OF_MEMORY = VY_SUCCESS + 1,
VY_BUFFER_MGR_OUT_OF_MEMORY = VY_CUSTOM_ERROR_START,
VY_BUFFER_MGR_MUTEX_CREATION_FAILED,
};

View File

@ -25,9 +25,12 @@ VY_DLLEXPORT void vyShutdownGFX(void);
/* Handles backend objects */
#define VY_GFX_HANDLE_MAX_VERSION 255
typedef struct {
uint32_t index;
} vy_gfx_pipeline_handle;
uint32_t version : 8;
uint32_t index : 24;
} vy_pipeline_handle;
/* Attributes are used to bind buffers (or textures) to symbolic values.
* For example, an attribute might be bound to "CELL_GRID", which would be

View File

@ -23,10 +23,8 @@ VY_CVAR_S(rt_Renderer, "Select the render backend. Available options: [vk], Defa
extern void VY_RENDERER_API_FN(RegisterCVars)(void);
extern vy_result VY_RENDERER_API_FN(Init)(const vy_renderer_init_info *);
extern void VY_RENDERER_API_FN(Shutdown)(void);
extern vy_gfx_pipeline_handle
VY_RENDERER_API_FN(CompileComputePipeline)(const vy_compute_pipeline_info *);
extern vy_gfx_pipeline_handle
VY_RENDERER_API_FN(CompileGraphicsPipeline)(const vy_graphics_pipeline_info *);
extern vy_pipeline_handle VY_RENDERER_API_FN(CompilePipeline)(const vy_pipeline_info *);
extern void VY_RENDERER_API_FN(DestroyPipeline)(vy_pipeline_handle handle);
#endif
static bool LoadRenderer(void) {
@ -50,8 +48,8 @@ static bool LoadRenderer(void) {
RETRIEVE_SYMBOL(RegisterCVars, vy_register_renderer_cvars_fn);
RETRIEVE_SYMBOL(Init, vy_init_renderer_fn);
RETRIEVE_SYMBOL(Shutdown, vy_shutdown_renderer_fn);
RETRIEVE_SYMBOL(CompileComputePipeline, vy_compile_compute_pipeline_fn);
RETRIEVE_SYMBOL(CompileGraphicsPipeline, vy_compile_graphics_pipeline_fn);
RETRIEVE_SYMBOL(CompilePipeline, vy_compile_pipeline_fn);
RETRIEVE_SYMBOL(DestroyPipeline, vy_destroy_pipeline_fn);
} else {
vyReportError("GFX",
"Unsupported renderer backend: (%s) %s",
@ -64,8 +62,8 @@ static bool LoadRenderer(void) {
g_renderer.RegisterCVars = &vyRenRegisterCVars;
g_renderer.Init = &vyRenInit;
g_renderer.Shutdown = &vyRenShutdown;
g_renderer.CompileComputePipeline = &vyRenCompileComputePipeline;
g_renderer.CompileGraphicsPipeline = &vyRenCompileGraphicsPipeline;
g_renderer.CompilePipeline = &vyRenCompilePipeline;
g_renderer.DestroyPipeline = &vyRenDestroyPipeline;
#endif
return true;
}

View File

@ -3,6 +3,9 @@
/* All handle types should contain a uint32_t index */
#define VY_INVALID_HANDLE \
{ .index = 0 }
#define VY_IS_HANDLE_VALID(handle) ((handle).index != 0)
#endif

View File

@ -26,19 +26,6 @@ struct vy_renderer_init_info_s {
#endif
};
typedef struct {
const char *compute_source;
size_t compute_source_length;
} vy_compute_pipeline_info;
typedef struct {
const char *vertex_source;
size_t vertex_source_length;
const char *fragment_source;
size_t fragment_source_length;
} vy_graphics_pipeline_info;
typedef struct {
vy_uid vertex_shader;
vy_uid fragment_shader;
@ -56,16 +43,15 @@ typedef struct {
typedef void vy_register_renderer_cvars_fn(void);
typedef vy_result vy_init_renderer_fn(const vy_renderer_init_info *info);
typedef void vy_shutdown_renderer_fn(void);
typedef vy_gfx_pipeline_handle vy_compile_compute_pipeline_fn(const vy_compute_pipeline_info *info);
typedef vy_gfx_pipeline_handle
vy_compile_graphics_pipeline_fn(const vy_graphics_pipeline_info *info);
typedef vy_pipeline_handle vy_compile_pipeline_fn(const vy_pipeline_info *info);
typedef void vy_destroy_pipeline_fn(vy_pipeline_handle handle);
typedef struct {
vy_register_renderer_cvars_fn *RegisterCVars;
vy_init_renderer_fn *Init;
vy_shutdown_renderer_fn *Shutdown;
vy_compile_compute_pipeline_fn *CompileComputePipeline;
vy_compile_graphics_pipeline_fn *CompileGraphicsPipeline;
vy_compile_pipeline_fn *CompilePipeline;
vy_destroy_pipeline_fn *DestroyPipeline;
} vy_renderer_api;
#define VY_RENDERER_API_FN(name) VY_DLLEXPORT vyRen##name

View File

@ -27,8 +27,16 @@
#define VY_GB(n) ((n)*1024U * 1024U * 1024U)
typedef unsigned int vy_result;
#define VY_SUCCESS 0
#define VY_UNKNOWN_ERROR (vy_result)-1
/* Default result codes */
enum {
VY_SUCCESS = 0,
VY_OUT_OF_MEMORY = 1,
VY_CUSTOM_ERROR_START,
VY_UNKNOWN_ERROR = (vy_result)-1,
};
typedef struct {
const char *start;
@ -49,8 +57,8 @@ VY_DLLEXPORT void vyReportError(const char *subsystem, const char *fmt, ...);
VY_DLLEXPORT void vyLog(const char *subsystem, const char *fmt, ...);
enum {
VY_INVALID_UNICODE = 1,
VY_INSUFFICIENT_BUFFER = 2,
VY_INVALID_UNICODE = VY_CUSTOM_ERROR_START,
VY_INSUFFICIENT_BUFFER,
};
/* Returns VY_SUCCESS if the string was successfully converted,

View File

@ -45,7 +45,7 @@ vy_result LoadUIDTable(void) {
if (!mem) {
fclose(f);
_tab.slots = 0;
return VY_BUFFER_ALLOC_FAILED;
return VY_OUT_OF_MEMORY;
}
_tab.uids = mem;
_tab.data = (vy_uid_data *)(_tab.uids + _tab.slots);

View File

@ -6,6 +6,8 @@
#include <limits.h>
#include <stdlib.h>
extern int memcmp(const void *s1, const void *s2, size_t n);
static bool IsAllowedChar(char c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
(c == '.') || (c == '_') || (c == '/');
@ -66,6 +68,37 @@ static bool ParseValue(vy_parse_state *state, vy_text_span *_value) {
return true;
}
#define BLOCK_BEGIN "BEGIN"
#define BLOCK_BEGIN_LENGTH 5
#define BLOCK_END "END"
#define BLOCK_END_LENGTH 3
VY_INLINE static bool IsBlockBegin(vy_parse_state *state) {
return (state->length - state->at >= BLOCK_BEGIN_LENGTH) &&
(memcmp(&state->text[state->at], BLOCK_BEGIN, BLOCK_BEGIN_LENGTH) == 0);
}
VY_INLINE static bool IsBlockEnd(vy_parse_state *state) {
return (state->length - state->at >= BLOCK_END_LENGTH) &&
(memcmp(&state->text[state->at], BLOCK_END, BLOCK_END_LENGTH) == 0);
}
static bool ParseBlock(vy_parse_state *state, vy_text_span *p_value) {
vy_text_span value;
value.start = &state->text[state->at];
value.length = 0;
while (state->at < state->length) {
if (state->text[state->at] == BLOCK_END[0] && IsBlockEnd(state)) {
*p_value = value;
return true;
}
++value.length;
++state->at;
}
/* did not find END */
return false;
}
static bool ParseStmtList(vy_parse_state *state, unsigned int *list_index);
static bool ParseStmt(vy_parse_state *state, unsigned int *stmt_index) {
@ -93,6 +126,16 @@ static bool ParseStmt(vy_parse_state *state, unsigned int *stmt_index) {
/* Consume '}' */
if (state->at < state->length && state->text[state->at] == '}')
++state->at;
} else if (IsBlockBegin(state)) {
/* Consume BEGIN */
state->at += BLOCK_BEGIN_LENGTH;
stmt.form = VY_STMT_FORM_BLOCK;
if (!ParseBlock(state, &stmt.block))
return false;
/* Consume END */
state->at += BLOCK_END_LENGTH;
} else {
stmt.form = VY_STMT_FORM_VALUE;
if (!ParseValue(state, &stmt.value))

View File

@ -6,6 +6,7 @@
typedef enum {
VY_STMT_FORM_VALUE,
VY_STMT_FORM_LIST,
VY_STMT_FORM_BLOCK,
} vy_stmt_form;
typedef struct {
@ -18,6 +19,7 @@ typedef struct {
vy_text_span attribute;
union {
vy_text_span value;
vy_text_span block;
unsigned int list_index;
};
/* For lists */

View File

@ -5,7 +5,7 @@
#include "runtime/file_tab.h"
enum {
VY_PROCESSING_FAILED = VY_SUCCESS + 1,
VY_PROCESSING_FAILED = VY_CUSTOM_ERROR_START,
/* Used if the processing depends on other files beeing processed first. */
VY_PROCESSING_TRY_AGAIN,
};