kevin/rtengine
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove fio

Browse Source
This commit is contained in:
Kevin Trogant 2023-11-22 15:21:11 +01:00
parent 1a4a2109ca
commit 937ca92d9f
5 changed files with 23 additions and 486 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
meson.build
View File

@ -44,7 +44,6 @@ runtime_lib = library('vyrt',
'src/runtime/runtime.h',
'src/runtime/gfx.h',
'src/runtime/renderer_api.h',
'src/runtime/fio.h',
'src/runtime/config.h',
'src/runtime/threading.h',
'src/runtime/app.h',
@ -61,7 +60,6 @@ runtime_lib = library('vyrt',
'src/runtime/threading_mutex.c',
'src/runtime/threading_thread.c',
'src/runtime/threading_cond.c',
'src/runtime/fio.c',
'src/runtime/app.c',
'src/runtime/dynamic_libs.c',
'src/runtime/jobs.c',

1
src/runtime/app.c
View File

@ -135,6 +135,7 @@ VY_DLLEXPORT int vyWin32Entry(HINSTANCE hInstance,
DestroyWindow(wnd);
UnregisterClassW(L"vyWndClass", hInstance);
vyShutdownAIO();
vyShutdownFIO();
return 0;

314
src/runtime/fio.c
View File

@ -1,314 +0,0 @@
#include "fio.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef __linux__
#include <pthread.h>
#elif defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#endif
#define FLAGS_FINISHED 0x001
#define FLAGS_RETRIEVED 0x002
#define FLAGS_IN_USE 0x004
typedef struct {
vy_file_id fid;
vy_file_buffer buffer;
unsigned int flags;
} vy_file_op;
/* Ringbuffer of file io operations */
typedef struct {
vy_file_op *ops;
unsigned int size;
unsigned int write_pos;
unsigned int read_pos;
#ifdef __linux__
pthread_mutex_t mutex;
pthread_cond_t pending_cond;
#elif defined(_WIN32)
CRITICAL_SECTION critical_section;
CONDITION_VARIABLE pending_cond;
#endif
} vy_fio_queue;
static vy_fio_queue _queue;
#ifdef __linux__
static pthread_t _thread;
#elif defined(_WIN32)
static HANDLE _fio_thread = NULL;
static HANDLE _fio_term_event = NULL;
#endif
static bool InitFIOQueue(unsigned int size) {
_queue.ops = calloc(size, sizeof(vy_file_op));
if (!_queue.ops)
return false;
_queue.write_pos = 0;
_queue.read_pos = 0;
_queue.size = size;
#ifdef __linux__
if (pthread_cond_init(&_queue.pending_cond, NULL) != 0)
return false;
if (pthread_mutex_init(&_queue.mutex, NULL) != 0)
return false;
#elif defined(_WIN32)
if (!InitializeCriticalSectionAndSpinCount(&_queue.critical_section, 4000))
return false;
InitializeConditionVariable(&_queue.pending_cond);
#endif
return true;
}
static void ShutdownFIOQueue(void) {
for (unsigned int i = 0; i < _queue.size; ++i) {
free(_queue.ops[i].buffer.data);
}
free(_queue.ops);
#ifdef __linux__
pthread_cond_destroy(&_queue.pending_cond);
pthread_mutex_destroy(&_queue.mutex);
#elif defined(_WIN32)
DeleteCriticalSection(&_queue.critical_section);
#endif
}
#ifdef __linux__
static void *linuxFIOThreadProc(void *);
#elif defined(_WIN32)
static DWORD WINAPI win32FIOThreadProc(_In_ LPVOID);
#endif
VY_DLLEXPORT bool vyInitFIO(const vy_fio_config *config) {
unsigned int queue_size = (config->queue_size) ? config->queue_size : 512;
if (!InitFIOQueue(queue_size))
return false;
#ifdef __linux__
if (pthread_create(&_thread, NULL, linuxFIOThreadProc, NULL) != 0)
return false;
#elif defined(_WIN32)
_fio_term_event = CreateEventW(NULL, FALSE, FALSE, NULL);
if (!_fio_term_event)
return false;
_fio_thread = CreateThread(NULL, 0, win32FIOThreadProc, NULL, 0, NULL);
if (!_fio_thread)
return false;
SetThreadDescription(_fio_thread, L"FIO Thread");
#endif
return true;
}
VY_DLLEXPORT void vyShutdownFIO(void) {
#ifdef __linux__
pthread_cancel(_thread);
pthread_join(_thread, NULL);
#elif defined(_WIN32)
if (SetEvent(_fio_term_event)) {
WakeAllConditionVariable(&_queue.pending_cond);
WaitForSingleObject(_fio_thread, INFINITE);
CloseHandle(_fio_thread);
CloseHandle(_fio_term_event);
} else {
vyReportError("FIO", "Failed to signal the termination event.");
}
#endif
ShutdownFIOQueue();
}
VY_DLLEXPORT vy_fio_handle vyEnqueueRead(vy_file_id fid) {
vy_fio_handle handle = 0;
do {
#ifdef __linux__
pthread_mutex_lock(&_queue.mutex);
#elif defined(_WIN32)
EnterCriticalSection(&_queue.critical_section);
#endif
if (_queue.ops[_queue.write_pos].flags == 0 ||
((_queue.ops[_queue.write_pos].flags & FLAGS_FINISHED) != 0 &&
(_queue.ops[_queue.write_pos].flags & FLAGS_RETRIEVED) != 0)) {
_queue.ops[_queue.write_pos].fid = fid;
_queue.ops[_queue.write_pos].flags = FLAGS_IN_USE;
handle = _queue.write_pos + 1;
_queue.write_pos = (_queue.write_pos + 1) % _queue.size;
#ifdef __linux__
pthread_cond_signal(&_queue.pending_cond);
pthread_mutex_unlock(&_queue.mutex);
#elif defined(_WIN32)
LeaveCriticalSection(&_queue.critical_section);
WakeAllConditionVariable(&_queue.pending_cond);
#endif
break;
}
#ifdef __linux__
pthread_mutex_unlock(&_queue.mutex);
#elif defined(_WIN32)
LeaveCriticalSection(&_queue.critical_section);
#endif
} while (1);
return handle;
}
VY_DLLEXPORT void vyAbortFIO(vy_fio_handle fio) {
if (fio == 0)
return;
#ifdef __linux__
pthread_mutex_lock(&_queue.mutex);
#elif defined(_WIN32)
EnterCriticalSection(&_queue.critical_section);
#endif
_queue.ops[fio - 1].flags = 0;
if (_queue.ops[fio - 1].buffer.data) {
free(_queue.ops[fio - 1].buffer.data);
_queue.ops[fio - 1].buffer.size = 0;
}
#ifdef __linux__
pthread_mutex_unlock(&_queue.mutex);
#elif defined(_WIN32)
LeaveCriticalSection(&_queue.critical_section);
#endif
}
VY_DLLEXPORT bool vyIsFIOFinished(vy_fio_handle fio) {
if (fio == 0)
return false;
#ifdef __linux__
pthread_mutex_lock(&_queue.mutex);
#elif defined(_WIN32)
EnterCriticalSection(&_queue.critical_section);
#endif
bool result = (_queue.ops[fio - 1].flags & FLAGS_FINISHED) != 0;
#ifdef __linux__
pthread_mutex_unlock(&_queue.mutex);
#elif defined(_WIN32)
LeaveCriticalSection(&_queue.critical_section);
#endif
return result;
}
VY_DLLEXPORT bool vyRetrieveReadBuffer(vy_fio_handle fio,
vy_file_buffer *buffer) {
if (fio == 0)
return false;
#ifdef __linux__
pthread_mutex_lock(&_queue.mutex);
#elif defined(_WIN32)
EnterCriticalSection(&_queue.critical_section);
#endif
bool is_finished = (_queue.ops[fio - 1].flags & FLAGS_FINISHED) != 0;
if (is_finished) {
size_t sz = _queue.ops[fio - 1].buffer.size;
buffer->data = malloc(sz);
if (!buffer->data)
return false;
buffer->size = sz;
memcpy(buffer->data, _queue.ops[fio - 1].buffer.data, sz);
buffer->flags = _queue.ops[fio - 1].buffer.flags;
_queue.ops[fio - 1].flags |= FLAGS_RETRIEVED;
}
#ifdef __linux__
pthread_mutex_unlock(&_queue.mutex);
#elif defined(_WIN32)
LeaveCriticalSection(&_queue.critical_section);
#endif
return is_finished;
}
VY_DLLEXPORT void vyFreeFileBuffer(vy_file_buffer buffer) {
free(buffer.data);
}
static void ProcessRead(vy_file_op *op) {
const char *path = vyGetFilePath(op->fid);
if (!path) {
op->flags |= FLAGS_FINISHED;
return;
}
op->buffer.data = NULL;
op->buffer.size = 0;
op->buffer.flags = 0;
FILE *file = fopen(path, "rb");
if (!file) {
op->flags |= FLAGS_FINISHED;
op->buffer.flags = VY_FILE_BUFFER_FLAG_FILE_NOT_FOUND;
return;
}
fseek(file, 0, SEEK_END);
long fsz = ftell(file);
fseek(file, 0, SEEK_SET);
op->buffer.data = malloc(fsz);
op->buffer.size = (size_t)fsz;
if (fread(op->buffer.data, fsz, 1, file) != 1) {
free(op->buffer.data);
op->buffer.data = NULL;
op->buffer.size = 0;
op->buffer.flags = VY_FILE_BUFFER_FLAG_READ_FAILED;
}
fclose(file);
op->flags |= FLAGS_FINISHED;
}
#ifdef __linux__
static void *linuxFIOThreadProc(void *_param) {
while (true) {
pthread_mutex_lock(&_queue.mutex);
while (_queue.write_pos == _queue.read_pos) {
pthread_cond_wait(&_queue.pending_cond, &_queue.mutex);
}
ProcessRead(&_queue.ops[_queue.read_pos]);
_queue.read_pos = (_queue.read_pos + 1) % _queue.size;
pthread_mutex_unlock(&_queue.mutex);
}
return NULL;
}
#elif defined(_WIN32)
static DWORD WINAPI win32FIOThreadProc(_In_ LPVOID lpParam) {
VY_UNUSED(lpParam);
bool keep_running = true;
while (keep_running) {
EnterCriticalSection(&_queue.critical_section);
while (_queue.read_pos == _queue.write_pos && keep_running) {
SleepConditionVariableCS(&_queue.pending_cond,
&_queue.critical_section,
INFINITE);
DWORD wfs = WaitForSingleObject(_fio_term_event, 0);
if (wfs != WAIT_FAILED) {
keep_running = wfs != WAIT_OBJECT_0;
} else {
vyLog("FIO", "ThreadProc wait error: %d", GetLastError());
keep_running = false;
}
}
/* It's possible that we were awoken during application shutdown. */
if (_queue.write_pos != _queue.read_pos) {
ProcessRead(&_queue.ops[_queue.read_pos]);
_queue.read_pos = (_queue.read_pos + 1) % _queue.size;
}
LeaveCriticalSection(&_queue.critical_section);
}
vyLog("FIO", "Exit FIO thread");
return 0;
}
#endif

49
src/runtime/fio.h
View File

@ -1,49 +0,0 @@
#ifndef VY_FIO_H
#define VY_FIO_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include "runtime.h"
#include "file_tab.h"
enum {
VY_FILE_BUFFER_FLAG_FILE_NOT_FOUND = 0x1,
VY_FILE_BUFFER_FLAG_READ_FAILED = 0x2,
};
typedef struct {
void *data;
size_t size;
uint32_t flags;
} vy_file_buffer;
static inline bool vyWasFileBufferSuccessful(const vy_file_buffer *fb) {
return fb->flags == 0;
}
typedef unsigned int vy_fio_handle;
typedef struct {
unsigned int queue_size;
unsigned int max_file_count;
} vy_fio_config;
VY_DLLEXPORT bool vyInitFIO(const vy_fio_config *config);
VY_DLLEXPORT void vyShutdownFIO(void);
VY_DLLEXPORT vy_fio_handle vyEnqueueRead(vy_file_id fid);
VY_DLLEXPORT void vyAbortFIO(vy_fio_handle fio);
VY_DLLEXPORT bool vyIsFIOFinished(vy_fio_handle fio);
VY_DLLEXPORT bool vyRetrieveReadBuffer(vy_fio_handle fio,
vy_file_buffer *buffer);
VY_DLLEXPORT void vyFreeFileBuffer(vy_file_buffer buffer);
#endif

143
src/runtime/gfx_shader_loading.c
View File

@ -6,7 +6,7 @@
#include <stdlib.h>
#include <string.h>
#include "fio.h"
#include "aio.h"
#include "gfx.h"
#include "handles.h"
#include "renderer_api.h"
@ -264,6 +264,7 @@ static const vy_parsed_stmt *FindStatement(const vy_parse_state *state,
return NULL;
}
#if 0
static vy_fio_handle DispatchFileRead(vy_text_span path) {
vy_file_id fid = vyAddFileFromSpan(path);
if (fid == 0)
@ -289,122 +290,15 @@ static vy_fio_handle DispatchShaderRead(const char *shader,
}
return DispatchFileRead(path->value);
}
#endif
static vy_gfx_pipeline_handle CreatePipeline(vy_parse_state *state,
const char *file_path,
unsigned int root_list) {
/* Process the data */
vy_fio_handle vertex_read =
DispatchShaderRead("vertex", state, root_list, file_path);
vy_fio_handle fragment_read =
DispatchShaderRead("fragment", state, root_list, file_path);
vy_fio_handle compute_read =
DispatchShaderRead("compute", state, root_list, file_path);
vy_aio_handle reads[3];
vy_load_batch read_batch = {.num_loads = 0};
if (compute_read) {
if (vertex_read || fragment_read) {
vyReportError("GFX",
"A shader program can not contain a compute "
"shader and graphics shaders: %s",
file_path);
vyAbortFIO(vertex_read);
vyAbortFIO(fragment_read);
vyAbortFIO(compute_read);
return (vy_gfx_pipeline_handle){0};
}
while (!vyIsFIOFinished(compute_read)) {
/* wait */
}
vy_file_buffer compute_code;
if (!vyRetrieveReadBuffer(compute_read, &compute_code) ||
!vyWasFileBufferSuccessful(&compute_code)) {
vyReportError("GFX",
"Failed to load compute shader required by: %s",
file_path);
return (vy_gfx_pipeline_handle){0};
}
vy_compute_pipeline_info info;
info.compute_source = compute_code.data;
info.compute_source_length = compute_code.size;
vy_gfx_pipeline_handle pipeline =
g_renderer.CompileComputePipeline(&info);
vyFreeFileBuffer(compute_code);
return pipeline;
} else if (vertex_read || fragment_read) {
if (compute_read) {
vyReportError("GFX",
"A shader program can not contain a compute "
"shader and graphics shaders: %s",
file_path);
vyAbortFIO(vertex_read);
vyAbortFIO(fragment_read);
vyAbortFIO(compute_read);
return (vy_gfx_pipeline_handle){0};
}
if (!vertex_read || !fragment_read) {
vyReportError("GFX",
"A shader program must contain at least a vertex and "
"a fragment shader: %s",
file_path);
vyAbortFIO(vertex_read);
vyAbortFIO(fragment_read);
vyAbortFIO(compute_read);
return (vy_gfx_pipeline_handle){0};
}
vy_graphics_pipeline_info info;
vy_file_buffer vertex_code = {NULL, 0};
vy_file_buffer fragment_code = {NULL, 0};
int remaining = 2;
while (remaining > 0) {
if (vyIsFIOFinished(vertex_read)) {
if (!vyRetrieveReadBuffer(vertex_read, &vertex_code) ||
!vyWasFileBufferSuccessful(&vertex_code)) {
vyReportError(
"GFX",
"Failed to load vertex shader required by: %s",
file_path);
vyFreeFileBuffer(fragment_code);
return (vy_gfx_pipeline_handle){0};
}
info.vertex_source = vertex_code.data;
info.vertex_source_length = vertex_code.size;
--remaining;
vertex_read = 0;
}
if (vyIsFIOFinished(fragment_read)) {
if (!vyRetrieveReadBuffer(fragment_read, &fragment_code) ||
!vyWasFileBufferSuccessful(&fragment_code)) {
vyReportError(
"GFX",
"Failed to load fragment shader required by: %s",
file_path);
vyFreeFileBuffer(vertex_code);
return (vy_gfx_pipeline_handle){0};
}
info.fragment_source = fragment_code.data;
info.fragment_source_length = fragment_code.size;
--remaining;
fragment_read = 0;
}
}
vy_gfx_pipeline_handle pipeline =
g_renderer.CompileGraphicsPipeline(&info);
vyFreeFileBuffer(vertex_code);
vyFreeFileBuffer(fragment_code);
return pipeline;
} else if (!compute_read && !vertex_read && !fragment_read) {
vyReportError("GFX",
"Either \"compute\" or \"vertex\" and \"fragment\" "
"are required in %s",
file_path);
vyAbortFIO(vertex_read);
vyAbortFIO(fragment_read);
vyAbortFIO(compute_read);
}
return (vy_gfx_pipeline_handle){0};
}
@ -495,16 +389,16 @@ static bool ParseBindings(vy_parse_state *state,
}
static bool
ParseShaderFile(vy_file_id fid, vy_file_buffer fbuf, vy_shader *shader) {
ParseShaderFile(vy_file_id fid, const char *text, size_t length, vy_shader *shader) {
/* This is the grammar for shader files:
* <stmt-list> ::= <stmt>*
* <stmt> ::= <attribute> ( ( <value> ';' ) | ( '{' <stmt-list> '}' ) )
* <attribute> ::= [:alnum:]*
* <value>:: = [:alnum:]* */
const char *file_path = vyGetFilePath(fid);
vy_parse_state state = {.text = (const char *)fbuf.data,
vy_parse_state state = {.text = text,
.at = 0,
.length = fbuf.size,
.length = length,
.line = 1,
.file = file_path,
.statements = NULL,
@ -572,30 +466,37 @@ out:
}
bool vyLoadShaders(const char **paths, vy_shader *shaders, unsigned int count) {
vy_fio_handle fios[64];
vy_aio_handle aios[64];
vy_file_id fids[64];
for (unsigned int i = 0; i < count; i += 64) {
unsigned int chunk_size = count - i;
if (chunk_size > 64)
chunk_size = 64;
vy_load_batch chunk;
chunk.num_loads = chunk_size;
for (unsigned int j = 0; j < chunk_size; ++j) {
vy_file_id fid = vyAddFile(paths[i + j]);
if (!fid)
return false;
fids[j] = fid;
fios[j] = vyEnqueueRead(fid);
/* Setup the chunk
chunk.loads[j].offset = 0;
chunk.loads[j].file = fid;
chunk.loads[j].
*/
}
unsigned int remaining = chunk_size;
while (remaining > 0) {
for (unsigned int j = 0; j < chunk_size; ++j) {
if (vyIsFIOFinished(fios[j])) {
vy_file_buffer fbuf;
if (!vyRetrieveReadBuffer(fios[j], &fbuf)) {
return false;
}
switch (vyGetAIOState(aios[j])) {
case VY_AIO_STATE_FINISHED:
/*
ParseShaderFile(fids[j], fbuf, &shaders[i + j]);
vyFreeFileBuffer(fbuf);
*/
--remaining;
}
}