Remove fio

meson.build

@@ -44,7 +44,6 @@ runtime_lib = library('vyrt',
   'src/runtime/runtime.h',
   'src/runtime/gfx.h',
   'src/runtime/renderer_api.h',
-  'src/runtime/fio.h',
   'src/runtime/config.h',
   'src/runtime/threading.h',
   'src/runtime/app.h',
@@ -61,7 +60,6 @@ runtime_lib = library('vyrt',
   'src/runtime/threading_mutex.c',
   'src/runtime/threading_thread.c',
   'src/runtime/threading_cond.c',
-  'src/runtime/fio.c',
   'src/runtime/app.c',
   'src/runtime/dynamic_libs.c',
   'src/runtime/jobs.c',

src/runtime/app.c

@@ -135,6 +135,7 @@ VY_DLLEXPORT int vyWin32Entry(HINSTANCE hInstance,
     DestroyWindow(wnd);
     UnregisterClassW(L"vyWndClass", hInstance);
 
+    vyShutdownAIO();
     vyShutdownFIO();
 
     return 0;

src/runtime/fio.c

@@ -1,314 +0,0 @@
-#include "fio.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#ifdef __linux__
-    #include <pthread.h>
-#elif defined(_WIN32)
-    #define WIN32_LEAN_AND_MEAN
-    #include <Windows.h>
-#endif
-
-#define FLAGS_FINISHED  0x001
-#define FLAGS_RETRIEVED 0x002
-#define FLAGS_IN_USE    0x004
-typedef struct {
-    vy_file_id fid;
-    vy_file_buffer buffer;
-    unsigned int flags;
-} vy_file_op;
-
-/* Ringbuffer of file io operations */
-typedef struct {
-    vy_file_op *ops;
-    unsigned int size;
-    unsigned int write_pos;
-    unsigned int read_pos;
-
-#ifdef __linux__
-    pthread_mutex_t mutex;
-    pthread_cond_t pending_cond;
-#elif defined(_WIN32)
-    CRITICAL_SECTION critical_section;
-    CONDITION_VARIABLE pending_cond;
-#endif
-} vy_fio_queue;
-
-static vy_fio_queue _queue;
-
-#ifdef __linux__
-static pthread_t _thread;
-#elif defined(_WIN32)
-static HANDLE _fio_thread     = NULL;
-static HANDLE _fio_term_event = NULL;
-#endif
-
-static bool InitFIOQueue(unsigned int size) {
-    _queue.ops = calloc(size, sizeof(vy_file_op));
-    if (!_queue.ops)
-        return false;
-    _queue.write_pos = 0;
-    _queue.read_pos  = 0;
-    _queue.size      = size;
-#ifdef __linux__
-    if (pthread_cond_init(&_queue.pending_cond, NULL) != 0)
-        return false;
-    if (pthread_mutex_init(&_queue.mutex, NULL) != 0)
-        return false;
-#elif defined(_WIN32)
-    if (!InitializeCriticalSectionAndSpinCount(&_queue.critical_section, 4000))
-        return false;
-    InitializeConditionVariable(&_queue.pending_cond);
-#endif
-    return true;
-}
-
-static void ShutdownFIOQueue(void) {
-    for (unsigned int i = 0; i < _queue.size; ++i) {
-        free(_queue.ops[i].buffer.data);
-    }
-    free(_queue.ops);
-#ifdef __linux__
-    pthread_cond_destroy(&_queue.pending_cond);
-    pthread_mutex_destroy(&_queue.mutex);
-#elif defined(_WIN32)
-    DeleteCriticalSection(&_queue.critical_section);
-#endif
-}
-
-#ifdef __linux__
-static void *linuxFIOThreadProc(void *);
-#elif defined(_WIN32)
-static DWORD WINAPI win32FIOThreadProc(_In_ LPVOID);
-#endif
-
-VY_DLLEXPORT bool vyInitFIO(const vy_fio_config *config) {
-    unsigned int queue_size = (config->queue_size) ? config->queue_size : 512;
-    if (!InitFIOQueue(queue_size))
-        return false;
-
-#ifdef __linux__
-    if (pthread_create(&_thread, NULL, linuxFIOThreadProc, NULL) != 0)
-        return false;
-#elif defined(_WIN32)
-    _fio_term_event = CreateEventW(NULL, FALSE, FALSE, NULL);
-    if (!_fio_term_event)
-        return false;
-    _fio_thread = CreateThread(NULL, 0, win32FIOThreadProc, NULL, 0, NULL);
-    if (!_fio_thread)
-        return false;
-    SetThreadDescription(_fio_thread, L"FIO Thread");
-#endif
-
-    return true;
-}
-
-VY_DLLEXPORT void vyShutdownFIO(void) {
-#ifdef __linux__
-    pthread_cancel(_thread);
-    pthread_join(_thread, NULL);
-#elif defined(_WIN32)
-    if (SetEvent(_fio_term_event)) {
-        WakeAllConditionVariable(&_queue.pending_cond);
-        WaitForSingleObject(_fio_thread, INFINITE);
-        CloseHandle(_fio_thread);
-        CloseHandle(_fio_term_event);
-    } else {
-        vyReportError("FIO", "Failed to signal the termination event.");
-    }
-#endif
-    ShutdownFIOQueue();
-}
-
-VY_DLLEXPORT vy_fio_handle vyEnqueueRead(vy_file_id fid) {
-    vy_fio_handle handle = 0;
-
-    do {
-#ifdef __linux__
-        pthread_mutex_lock(&_queue.mutex);
-#elif defined(_WIN32)
-        EnterCriticalSection(&_queue.critical_section);
-#endif
-
-        if (_queue.ops[_queue.write_pos].flags == 0 ||
-            ((_queue.ops[_queue.write_pos].flags & FLAGS_FINISHED) != 0 &&
-             (_queue.ops[_queue.write_pos].flags & FLAGS_RETRIEVED) != 0)) {
-
-            _queue.ops[_queue.write_pos].fid   = fid;
-            _queue.ops[_queue.write_pos].flags = FLAGS_IN_USE;
-
-            handle           = _queue.write_pos + 1;
-            _queue.write_pos = (_queue.write_pos + 1) % _queue.size;
-
-#ifdef __linux__
-            pthread_cond_signal(&_queue.pending_cond);
-            pthread_mutex_unlock(&_queue.mutex);
-#elif defined(_WIN32)
-            LeaveCriticalSection(&_queue.critical_section);
-            WakeAllConditionVariable(&_queue.pending_cond);
-#endif
-            break;
-        }
-
-#ifdef __linux__
-        pthread_mutex_unlock(&_queue.mutex);
-#elif defined(_WIN32)
-        LeaveCriticalSection(&_queue.critical_section);
-#endif
-    } while (1);
-
-    return handle;
-}
-
-VY_DLLEXPORT void vyAbortFIO(vy_fio_handle fio) {
-    if (fio == 0)
-        return;
-#ifdef __linux__
-    pthread_mutex_lock(&_queue.mutex);
-#elif defined(_WIN32)
-    EnterCriticalSection(&_queue.critical_section);
-#endif
-    _queue.ops[fio - 1].flags = 0;
-    if (_queue.ops[fio - 1].buffer.data) {
-        free(_queue.ops[fio - 1].buffer.data);
-        _queue.ops[fio - 1].buffer.size = 0;
-    }
-#ifdef __linux__
-    pthread_mutex_unlock(&_queue.mutex);
-#elif defined(_WIN32)
-    LeaveCriticalSection(&_queue.critical_section);
-#endif
-}
-
-VY_DLLEXPORT bool vyIsFIOFinished(vy_fio_handle fio) {
-    if (fio == 0)
-        return false;
-#ifdef __linux__
-    pthread_mutex_lock(&_queue.mutex);
-#elif defined(_WIN32)
-    EnterCriticalSection(&_queue.critical_section);
-#endif
-    bool result = (_queue.ops[fio - 1].flags & FLAGS_FINISHED) != 0;
-#ifdef __linux__
-    pthread_mutex_unlock(&_queue.mutex);
-#elif defined(_WIN32)
-    LeaveCriticalSection(&_queue.critical_section);
-#endif
-    return result;
-}
-
-VY_DLLEXPORT bool vyRetrieveReadBuffer(vy_fio_handle fio,
-                                       vy_file_buffer *buffer) {
-    if (fio == 0)
-        return false;
-
-#ifdef __linux__
-    pthread_mutex_lock(&_queue.mutex);
-#elif defined(_WIN32)
-    EnterCriticalSection(&_queue.critical_section);
-#endif
-
-    bool is_finished = (_queue.ops[fio - 1].flags & FLAGS_FINISHED) != 0;
-    if (is_finished) {
-        size_t sz    = _queue.ops[fio - 1].buffer.size;
-        buffer->data = malloc(sz);
-        if (!buffer->data)
-            return false;
-        buffer->size = sz;
-        memcpy(buffer->data, _queue.ops[fio - 1].buffer.data, sz);
-        buffer->flags = _queue.ops[fio - 1].buffer.flags;
-        _queue.ops[fio - 1].flags |= FLAGS_RETRIEVED;
-    }
-
-#ifdef __linux__
-    pthread_mutex_unlock(&_queue.mutex);
-#elif defined(_WIN32)
-    LeaveCriticalSection(&_queue.critical_section);
-#endif
-
-    return is_finished;
-}
-
-VY_DLLEXPORT void vyFreeFileBuffer(vy_file_buffer buffer) {
-    free(buffer.data);
-}
-
-static void ProcessRead(vy_file_op *op) {
-    const char *path = vyGetFilePath(op->fid);
-    if (!path) {
-        op->flags |= FLAGS_FINISHED;
-        return;
-    }
-
-    op->buffer.data  = NULL;
-    op->buffer.size  = 0;
-    op->buffer.flags = 0;
-
-    FILE *file = fopen(path, "rb");
-    if (!file) {
-        op->flags |= FLAGS_FINISHED;
-        op->buffer.flags = VY_FILE_BUFFER_FLAG_FILE_NOT_FOUND;
-        return;
-    }
-    fseek(file, 0, SEEK_END);
-    long fsz = ftell(file);
-    fseek(file, 0, SEEK_SET);
-
-    op->buffer.data = malloc(fsz);
-    op->buffer.size = (size_t)fsz;
-    if (fread(op->buffer.data, fsz, 1, file) != 1) {
-        free(op->buffer.data);
-        op->buffer.data  = NULL;
-        op->buffer.size  = 0;
-        op->buffer.flags = VY_FILE_BUFFER_FLAG_READ_FAILED;
-    }
-
-    fclose(file);
-    op->flags |= FLAGS_FINISHED;
-}
-
-#ifdef __linux__
-static void *linuxFIOThreadProc(void *_param) {
-    while (true) {
-        pthread_mutex_lock(&_queue.mutex);
-        while (_queue.write_pos == _queue.read_pos) {
-            pthread_cond_wait(&_queue.pending_cond, &_queue.mutex);
-        }
-        ProcessRead(&_queue.ops[_queue.read_pos]);
-        _queue.read_pos = (_queue.read_pos + 1) % _queue.size;
-        pthread_mutex_unlock(&_queue.mutex);
-    }
-    return NULL;
-}
-#elif defined(_WIN32)
-static DWORD WINAPI win32FIOThreadProc(_In_ LPVOID lpParam) {
-    VY_UNUSED(lpParam);
-
-    bool keep_running = true;
-    while (keep_running) {
-        EnterCriticalSection(&_queue.critical_section);
-        while (_queue.read_pos == _queue.write_pos && keep_running) {
-            SleepConditionVariableCS(&_queue.pending_cond,
-                                     &_queue.critical_section,
-                                     INFINITE);
-            DWORD wfs = WaitForSingleObject(_fio_term_event, 0);
-            if (wfs != WAIT_FAILED) {
-                keep_running = wfs != WAIT_OBJECT_0;
-            } else {
-                vyLog("FIO", "ThreadProc wait error: %d", GetLastError());
-                keep_running = false;
-            }
-        }
-        /* It's possible that we were awoken during application shutdown. */
-        if (_queue.write_pos != _queue.read_pos) {
-            ProcessRead(&_queue.ops[_queue.read_pos]);
-            _queue.read_pos = (_queue.read_pos + 1) % _queue.size;
-        }
-        LeaveCriticalSection(&_queue.critical_section);
-    }
-    vyLog("FIO", "Exit FIO thread");
-    return 0;
-}
-#endif

src/runtime/fio.h

@@ -1,49 +0,0 @@
-#ifndef VY_FIO_H
-#define VY_FIO_H
-
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-
-#include "runtime.h"
-#include "file_tab.h"
-
-enum {
-    VY_FILE_BUFFER_FLAG_FILE_NOT_FOUND = 0x1,
-    VY_FILE_BUFFER_FLAG_READ_FAILED    = 0x2,
-};
-
-typedef struct {
-    void *data;
-    size_t size;
-    uint32_t flags;
-} vy_file_buffer;
-
-static inline bool vyWasFileBufferSuccessful(const vy_file_buffer *fb) {
-    return fb->flags == 0;
-}
-
-
-typedef unsigned int vy_fio_handle;
-
-typedef struct {
-    unsigned int queue_size;
-    unsigned int max_file_count;
-} vy_fio_config;
-
-VY_DLLEXPORT bool vyInitFIO(const vy_fio_config *config);
-
-VY_DLLEXPORT void vyShutdownFIO(void);
-
-VY_DLLEXPORT vy_fio_handle vyEnqueueRead(vy_file_id fid);
-
-VY_DLLEXPORT void vyAbortFIO(vy_fio_handle fio);
-
-VY_DLLEXPORT bool vyIsFIOFinished(vy_fio_handle fio);
-
-VY_DLLEXPORT bool vyRetrieveReadBuffer(vy_fio_handle fio,
-                                       vy_file_buffer *buffer);
-
-VY_DLLEXPORT void vyFreeFileBuffer(vy_file_buffer buffer);
-
-#endif

src/runtime/gfx_shader_loading.c

@@ -6,7 +6,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "fio.h"
+#include "aio.h"
 #include "gfx.h"
 #include "handles.h"
 #include "renderer_api.h"
@@ -264,6 +264,7 @@ static const vy_parsed_stmt *FindStatement(const vy_parse_state *state,
     return NULL;
 }
 
+#if 0
 static vy_fio_handle DispatchFileRead(vy_text_span path) {
     vy_file_id fid = vyAddFileFromSpan(path);
     if (fid == 0)
@@ -289,122 +290,15 @@ static vy_fio_handle DispatchShaderRead(const char *shader,
     }
     return DispatchFileRead(path->value);
 }
+#endif
 
 static vy_gfx_pipeline_handle CreatePipeline(vy_parse_state *state,
                                              const char *file_path,
                                              unsigned int root_list) {
     /* Process the data */
-    vy_fio_handle vertex_read =
+    vy_aio_handle reads[3];
-        DispatchShaderRead("vertex", state, root_list, file_path);
+    vy_load_batch read_batch = {.num_loads = 0};
-    vy_fio_handle fragment_read =
-        DispatchShaderRead("fragment", state, root_list, file_path);
-    vy_fio_handle compute_read =
-        DispatchShaderRead("compute", state, root_list, file_path);
 
-    if (compute_read) {
-        if (vertex_read || fragment_read) {
-            vyReportError("GFX",
-                          "A shader program can not contain a compute "
-                          "shader and graphics shaders: %s",
-                          file_path);
-            vyAbortFIO(vertex_read);
-            vyAbortFIO(fragment_read);
-            vyAbortFIO(compute_read);
-            return (vy_gfx_pipeline_handle){0};
-        }
-        while (!vyIsFIOFinished(compute_read)) {
-            /* wait */
-        }
-        vy_file_buffer compute_code;
-        if (!vyRetrieveReadBuffer(compute_read, &compute_code) ||
-            !vyWasFileBufferSuccessful(&compute_code)) {
-            vyReportError("GFX",
-                          "Failed to load compute shader required by: %s",
-                          file_path);
-            return (vy_gfx_pipeline_handle){0};
-        }
-        vy_compute_pipeline_info info;
-        info.compute_source        = compute_code.data;
-        info.compute_source_length = compute_code.size;
-        vy_gfx_pipeline_handle pipeline =
-            g_renderer.CompileComputePipeline(&info);
-        vyFreeFileBuffer(compute_code);
-        return pipeline;
-    } else if (vertex_read || fragment_read) {
-        if (compute_read) {
-            vyReportError("GFX",
-                          "A shader program can not contain a compute "
-                          "shader and graphics shaders: %s",
-                          file_path);
-            vyAbortFIO(vertex_read);
-            vyAbortFIO(fragment_read);
-            vyAbortFIO(compute_read);
-            return (vy_gfx_pipeline_handle){0};
-        }
-        if (!vertex_read || !fragment_read) {
-            vyReportError("GFX",
-                          "A shader program must contain at least a vertex and "
-                          "a fragment shader: %s",
-                          file_path);
-            vyAbortFIO(vertex_read);
-            vyAbortFIO(fragment_read);
-            vyAbortFIO(compute_read);
-            return (vy_gfx_pipeline_handle){0};
-        }
-
-        vy_graphics_pipeline_info info;
-        vy_file_buffer vertex_code   = {NULL, 0};
-        vy_file_buffer fragment_code = {NULL, 0};
-
-        int remaining = 2;
-        while (remaining > 0) {
-            if (vyIsFIOFinished(vertex_read)) {
-                if (!vyRetrieveReadBuffer(vertex_read, &vertex_code) ||
-                    !vyWasFileBufferSuccessful(&vertex_code)) {
-                    vyReportError(
-                        "GFX",
-                        "Failed to load vertex shader required by: %s",
-                        file_path);
-                    vyFreeFileBuffer(fragment_code);
-                    return (vy_gfx_pipeline_handle){0};
-                }
-                info.vertex_source        = vertex_code.data;
-                info.vertex_source_length = vertex_code.size;
-                --remaining;
-                vertex_read = 0;
-            }
-
-            if (vyIsFIOFinished(fragment_read)) {
-                if (!vyRetrieveReadBuffer(fragment_read, &fragment_code) ||
-                    !vyWasFileBufferSuccessful(&fragment_code)) {
-                    vyReportError(
-                        "GFX",
-                        "Failed to load fragment shader required by: %s",
-                        file_path);
-                    vyFreeFileBuffer(vertex_code);
-                    return (vy_gfx_pipeline_handle){0};
-                }
-                info.fragment_source        = fragment_code.data;
-                info.fragment_source_length = fragment_code.size;
-                --remaining;
-                fragment_read = 0;
-            }
-        }
-
-        vy_gfx_pipeline_handle pipeline =
-            g_renderer.CompileGraphicsPipeline(&info);
-        vyFreeFileBuffer(vertex_code);
-        vyFreeFileBuffer(fragment_code);
-        return pipeline;
-    } else if (!compute_read && !vertex_read && !fragment_read) {
-        vyReportError("GFX",
-                      "Either \"compute\" or \"vertex\" and \"fragment\" "
-                      "are required in %s",
-                      file_path);
-        vyAbortFIO(vertex_read);
-        vyAbortFIO(fragment_read);
-        vyAbortFIO(compute_read);
-    }
     return (vy_gfx_pipeline_handle){0};
 }
 
@@ -495,16 +389,16 @@ static bool ParseBindings(vy_parse_state *state,
 }
 
 static bool
-ParseShaderFile(vy_file_id fid, vy_file_buffer fbuf, vy_shader *shader) {
+ParseShaderFile(vy_file_id fid, const char *text, size_t length, vy_shader *shader) {
     /* This is the grammar for shader files:
      * <stmt-list> ::= <stmt>*
      * <stmt> ::= <attribute> ( ( <value> ';' ) | ( '{' <stmt-list> '}' ) )
      * <attribute> ::= [:alnum:]*
      * <value>:: = [:alnum:]* */
     const char *file_path = vyGetFilePath(fid);
-    vy_parse_state state  = {.text                    = (const char *)fbuf.data,
+    vy_parse_state state  = {.text                    = text,
                              .at                      = 0,
-                             .length                  = fbuf.size,
+                             .length                  = length,
                              .line                    = 1,
                              .file                    = file_path,
                              .statements              = NULL,
@@ -572,30 +466,37 @@ out:
 }
 
 bool vyLoadShaders(const char **paths, vy_shader *shaders, unsigned int count) {
-    vy_fio_handle fios[64];
+    vy_aio_handle aios[64];
     vy_file_id fids[64];
     for (unsigned int i = 0; i < count; i += 64) {
         unsigned int chunk_size = count - i;
         if (chunk_size > 64)
             chunk_size = 64;
+        vy_load_batch chunk;
+        chunk.num_loads = chunk_size;
         for (unsigned int j = 0; j < chunk_size; ++j) {
             vy_file_id fid = vyAddFile(paths[i + j]);
             if (!fid)
                 return false;
             fids[j] = fid;
-            fios[j] = vyEnqueueRead(fid);
+
+            /* Setup the chunk
+            chunk.loads[j].offset = 0;
+            chunk.loads[j].file   = fid;
+            chunk.loads[j].
+
+            */
         }
 
         unsigned int remaining = chunk_size;
         while (remaining > 0) {
             for (unsigned int j = 0; j < chunk_size; ++j) {
-                if (vyIsFIOFinished(fios[j])) {
+                switch (vyGetAIOState(aios[j])) {
-                    vy_file_buffer fbuf;
+                case VY_AIO_STATE_FINISHED:
-                    if (!vyRetrieveReadBuffer(fios[j], &fbuf)) {
+                    /*
-                        return false;
-                    }
                     ParseShaderFile(fids[j], fbuf, &shaders[i + j]);
                     vyFreeFileBuffer(fbuf);
+                    */
                     --remaining;
                 }
             }