Multiple asset processors

- Added a (stub) for shader files

meson.build

@@ -112,6 +112,7 @@ if vk_dep.found()
   static_renderer_lib = vk_renderer_lib
 endif
 
+
 # Asset Compiler Tool
 executable('assetc',
   'src/tools/assetc/processing.h',
@@ -119,9 +120,12 @@ executable('assetc',
   
   'src/tools/assetc/assetc.c',
   'src/tools/assetc/processor.c',
+  'src/tools/assetc/pipeline_processor.c',
   'src/tools/assetc/shader_processor.c',
   'src/tools/assetc/utils.c',
+  'src/tools/assetc/uidmap.c',
   include_directories : incdir,
+  dependencies : [shaderc_dep],
   link_with : [runtime_lib],
   win_subsystem : 'console')
 

shader/cell.pipeline

@@ -1,5 +1,5 @@
-vertex shader/cell_vert.glsl;
-fragment shader/cell_frag.glsl;
+vertex shader/cell_vert.shader;
+fragment shader/cell_frag.shader;
 
 texture_bindings {
     0 MATERIAL_ALBEDO;

shader/cell_vert.shader

@@ -0,0 +1,4 @@
+#define VY_SHADER_TYPE_VERT
+#ifdef VY_SHADER_VK
+
+#endif

src/runtime/assets.h

@@ -6,4 +6,6 @@
 /* Unique identifier for an asset. */
 typedef uint32_t vy_uid;
 
+#define VY_INVALID_UID 0
+
 #endif

src/runtime/file_tab.c

@@ -79,6 +79,7 @@ VY_DLLEXPORT vy_file_id vyAddFileFromSpan(vy_text_span path) {
                 break;
             }
             memcpy(_file_tab.names + _file_tab.name_head, path.start, slen);
+            _file_tab.names[_file_tab.name_head + slen - 1] = '\0';
             _file_tab.name_offsets[at] = _file_tab.name_head;
             _file_tab.ids[at]          = fid;
             _file_tab.name_head += slen;

src/runtime/file_tab.h

@@ -7,6 +7,7 @@
 
 /* used to identify a file (XXH3 hash of the path) */
 typedef uint64_t vy_file_id;
+#define VY_INVALID_FILE_ID 0
 
 VY_DLLEXPORT vy_result vyInitFileTab(unsigned int max_files);
 

src/runtime/gfx.h

@@ -47,16 +47,4 @@ typedef struct {
     vy_attribute_value value;
 } vy_attribute_binding;
 
-typedef struct {
-    vy_attribute_binding *uniform_bindings;
-    vy_attribute_binding *storage_bindings;
-    vy_attribute_binding *texture_bindings;
-
-    vy_gfx_pipeline_handle pipeline;
-
-    unsigned int uniform_binding_count;
-    unsigned int storage_binding_count;
-    unsigned int texture_binding_count;
-} vy_shader;
-
 #endif

src/tools/assetc/assetc.c

@@ -5,6 +5,11 @@
 
 #include <stdlib.h>
 
+extern vy_result vyProcessPipelineFile(vy_file_id file,
+                                       void *buffer,
+                                       size_t size,
+                                       vy_processor_output *output);
+
 extern vy_result vyProcessShaderFile(vy_file_id file,
                                      void *buffer,
                                      size_t size,
@@ -22,6 +27,10 @@ int main(int argc, char **argv) {
         return 1;
     }
 
+    vyInitUIDMap();
+
+    if (vyAddAssetProcessor(".pipeline", vyProcessPipelineFile) != VY_SUCCESS)
+        return 1;
     if (vyAddAssetProcessor(".shader", vyProcessShaderFile) != VY_SUCCESS)
         return 1;
 
@@ -29,7 +38,7 @@ int main(int argc, char **argv) {
         return 1;
     }
 
-    vyAddFileToProcessingQueue(vyAddFile("shader\\cell.shader"));
+    vyAddFileToProcessingQueue(vyAddFile("shader\\cell.pipeline"));
     while (1)
         _sleep(10);
 

src/tools/assetc/pipeline_processor.c

@@ -0,0 +1,538 @@
+#include <assert.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "runtime/aio.h"
+#include "runtime/gfx.h"
+#include "runtime/handles.h"
+#include "runtime/runtime.h"
+
+#include "processing.h"
+
+typedef enum {
+    VY_STMT_FORM_VALUE,
+    VY_STMT_FORM_LIST,
+} vy_stmt_form;
+
+typedef struct {
+    unsigned int first;
+    unsigned int count;
+} vy_parsed_stmt_list;
+
+typedef struct {
+    vy_stmt_form form;
+    vy_text_span attribute;
+    union {
+        vy_text_span value;
+        unsigned int list_index;
+    };
+    /* For lists */
+    unsigned int next;
+} vy_parsed_stmt;
+
+typedef struct {
+    const char *file;
+    const char *text;
+    size_t at;
+    size_t length;
+    int line;
+
+    vy_parsed_stmt *statements;
+    unsigned int statement_count;
+    unsigned int statement_capacity;
+
+    vy_parsed_stmt_list *statement_lists;
+    unsigned int statement_list_count;
+    unsigned int statement_list_capacity;
+} vy_parse_state;
+
+static bool IsAllowedChar(char c) {
+    return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
+           (c >= 'A' && c <= 'Z') || (c == '.') || (c == '_') || (c == '/');
+}
+
+static bool IsWhitespace(char c) {
+    return c == ' ' || c == '\t' || c == '\n' || c == '\r';
+}
+
+static void SkipWhitespace(vy_parse_state *state) {
+    while (state->at < state->length && IsWhitespace(state->text[state->at])) {
+        if (state->text[state->at] == '\n')
+            ++state->line;
+        ++state->at;
+    }
+}
+
+static bool ParseAttribute(vy_parse_state *state, vy_text_span *_name) {
+    vy_text_span name;
+    name.start  = &state->text[state->at];
+    name.length = 0;
+    while (state->at < state->length && !IsWhitespace(state->text[state->at])) {
+        if (IsAllowedChar(state->text[state->at])) {
+            ++state->at;
+            ++name.length;
+        } else {
+            vyReportError("GFX",
+                          "%s:%d Unexpected character %c",
+                          state->file,
+                          state->line,
+                          state->text[state->at]);
+            return false;
+        }
+    }
+    *_name = name;
+    return true;
+}
+
+static bool ParseValue(vy_parse_state *state, vy_text_span *_value) {
+    vy_text_span value;
+    value.start  = &state->text[state->at];
+    value.length = 0;
+    while (state->at < state->length && !IsWhitespace(state->text[state->at]) &&
+           state->text[state->at] != ';') {
+        if (IsAllowedChar(state->text[state->at])) {
+            ++state->at;
+            ++value.length;
+        } else {
+            vyReportError("GFX",
+                          "%s:%d Unexpected character %c",
+                          state->file,
+                          state->line,
+                          state->text[state->at]);
+            return false;
+        }
+    }
+    *_value = value;
+    return true;
+}
+
+static bool ParseStmtList(vy_parse_state *state, unsigned int *list_index);
+
+static bool ParseStmt(vy_parse_state *state, unsigned int *stmt_index) {
+    vy_parsed_stmt stmt;
+    stmt.next = UINT_MAX; /* end of list */
+
+    SkipWhitespace(state);
+    if (!ParseAttribute(state, &stmt.attribute))
+        return false;
+    SkipWhitespace(state);
+
+    if (state->at == state->length) {
+        vyReportError("GFX",
+                      "%s:%d Expected either a value or '{'",
+                      state->file,
+                      state->line);
+        return false;
+    }
+
+    if (state->text[state->at] == '{') {
+        /* Consume '{' */
+        ++state->at;
+
+        stmt.form = VY_STMT_FORM_LIST;
+        if (!ParseStmtList(state, &stmt.list_index))
+            return false;
+
+        /* Consume '}' */
+        if (state->at < state->length && state->text[state->at] == '}')
+            ++state->at;
+    } else {
+        stmt.form = VY_STMT_FORM_VALUE;
+        if (!ParseValue(state, &stmt.value))
+            return false;
+
+        /* Consume ';' */
+        if (state->at < state->length && state->text[state->at] == ';')
+            ++state->at;
+    }
+
+    SkipWhitespace(state);
+
+    /* Add statement to array */
+    if (state->statement_count == state->statement_capacity) {
+        unsigned int cap = (state->statement_capacity > 0)
+                               ? state->statement_capacity * 2
+                               : 64;
+        vy_parsed_stmt *temp =
+            realloc(state->statements, sizeof(vy_parsed_stmt) * cap);
+        if (!temp) {
+            vyReportError("GFX",
+                          "While parsing %s: Out of memory\n",
+                          state->file);
+            return false;
+        }
+        state->statements         = temp;
+        state->statement_capacity = cap;
+    }
+    state->statements[state->statement_count] = stmt;
+    *stmt_index                               = state->statement_count++;
+
+    return true;
+}
+
+static bool ParseStmtList(vy_parse_state *state, unsigned int *list_index) {
+    vy_parsed_stmt_list list;
+    list.first = state->statement_count;
+    list.count = 0;
+
+    unsigned int last = UINT_MAX;
+
+    while (state->at < state->length && state->text[state->at] != '}') {
+        unsigned int stmt;
+        if (!ParseStmt(state, &stmt))
+            return false;
+
+        if (last != UINT_MAX)
+            state->statements[last].next = stmt;
+        last = stmt;
+
+        ++list.count;
+    }
+
+    /* Add list to array */
+    if (state->statement_list_count == state->statement_list_capacity) {
+        unsigned int cap = (state->statement_list_capacity > 0)
+                               ? state->statement_list_capacity * 2
+                               : 64;
+        vy_parsed_stmt_list *temp =
+            realloc(state->statement_lists, sizeof(vy_parsed_stmt_list) * cap);
+        if (!temp) {
+            vyReportError("GFX",
+                          "While parsing %s: Out of memory\n",
+                          state->file);
+            return false;
+        }
+        state->statement_lists         = temp;
+        state->statement_list_capacity = cap;
+    }
+    state->statement_lists[state->statement_list_count] = list;
+    *list_index = state->statement_list_count++;
+    return true;
+}
+
+static void DbgPrintShaderFile(const vy_parse_state *state,
+                               unsigned int list_index,
+                               unsigned int indent) {
+    assert(list_index < state->statement_list_count);
+    const vy_parsed_stmt_list *list = &state->statement_lists[list_index];
+
+    unsigned int stmt_index = list->first;
+    for (unsigned int i = 0; i < list->count; ++i) {
+        const vy_parsed_stmt *stmt = &state->statements[stmt_index];
+        for (unsigned int j = 0; j < indent; ++j)
+            printf(" ");
+        printf("%.*s: ", stmt->attribute.length, stmt->attribute.start);
+        if (stmt->form == VY_STMT_FORM_VALUE) {
+            printf("%.*s\n", stmt->value.length, stmt->value.start);
+        } else {
+            printf("{\n");
+            DbgPrintShaderFile(state, stmt->list_index, indent + 2);
+            printf("}\n");
+        }
+        stmt_index = stmt->next;
+    }
+    assert(stmt_index == UINT_MAX || stmt_index == 0);
+}
+
+static bool CompareSpanToString(vy_text_span span, const char *cmp) {
+    size_t cmp_len = strlen(cmp);
+    if (cmp_len != (size_t)span.length)
+        return false;
+    for (size_t i = 0; i < cmp_len; ++i) {
+        if (span.start[i] != cmp[i])
+            return false;
+    }
+    return true;
+}
+
+static const vy_parsed_stmt *FindStatement(const vy_parse_state *state,
+                                           unsigned int list_index,
+                                           const char *attribute) {
+    if (list_index >= state->statement_list_count)
+        return NULL;
+    const vy_parsed_stmt_list *list = &state->statement_lists[list_index];
+    unsigned int stmt_index         = list->first;
+    for (unsigned int i = 0; i < list->count; ++i) {
+        const vy_parsed_stmt *stmt = &state->statements[stmt_index];
+        if (CompareSpanToString(stmt->attribute, attribute))
+            return stmt;
+        stmt_index = stmt->next;
+    }
+    return NULL;
+}
+
+static bool ParseBindingIndex(vy_text_span span, unsigned int *index) {
+    if (span.length == 0)
+        return false;
+    int at           = (int)span.length - 1;
+    unsigned int exp = 1;
+    unsigned int n   = 0;
+    while (at >= 0) {
+        if (span.start[at] >= '0' && span.start[at] <= '9') {
+            unsigned int digit = (unsigned int)(span.start[at] - '0');
+            n += digit * exp;
+        } else {
+            vyReportError("GFX",
+                          "Unexpected non-digit character in binding index");
+            return false;
+        }
+        --at;
+        exp *= 10;
+    }
+    *index = n;
+    return true;
+}
+
+static vy_attribute_value ParseBindingValue(vy_text_span span) {
+    if (CompareSpanToString(span, "MATERIAL_ALBEDO")) {
+        return VY_ATTRIBUTE_VALUE_MATERIAL_ALBEDO;
+    } else if (CompareSpanToString(span, "MATERIAL_NORMAL")) {
+        return VY_ATTRIBUTE_VALUE_MATERIAL_NORMAL;
+    }
+    vyReportError("GFX",
+                  "Unsupported binding value %*.s",
+                  span.length,
+                  span.start);
+    return VY_ATTRIBUTE_VALUE_UNDEFINED;
+}
+
+static bool ParseBindings(vy_parse_state *state,
+                          unsigned int root_list,
+                          const char *name,
+                          const char *file_path,
+                          vy_attribute_binding **p_bindings,
+                          unsigned int *p_binding_count) {
+    const vy_parsed_stmt *bindings = FindStatement(state, root_list, name);
+    if (bindings) {
+        if (bindings->form != VY_STMT_FORM_LIST) {
+            vyReportError("GFX",
+                          "Expected list of bindings as the value of "
+                          "\"%s\" in %s",
+                          name,
+                          file_path);
+            return false;
+        }
+        const vy_parsed_stmt_list *binding_list =
+            &state->statement_lists[bindings->list_index];
+        vy_attribute_binding *shader_bindings =
+            malloc(sizeof(vy_attribute_binding) * binding_list->count);
+        if (!bindings) {
+            vyReportError("GFX", "Out of memory");
+            return false;
+        }
+        unsigned int binding_count = binding_list->count;
+
+        unsigned int stmt_index = binding_list->first;
+        for (unsigned int i = 0; i < binding_list->count; ++i) {
+            const vy_parsed_stmt *stmt = &state->statements[stmt_index];
+            if (!ParseBindingIndex(stmt->attribute,
+                                   &shader_bindings[i].index)) {
+                free(shader_bindings);
+                return false;
+            }
+
+            shader_bindings[i].value = ParseBindingValue(stmt->value);
+            if (shader_bindings[i].value == VY_ATTRIBUTE_VALUE_UNDEFINED) {
+                free(shader_bindings);
+                return false;
+            }
+            stmt_index = stmt->next;
+        }
+
+        *p_bindings      = shader_bindings;
+        *p_binding_count = binding_count;
+        return true;
+    } else {
+        *p_bindings      = NULL;
+        *p_binding_count = 0;
+        return true;
+    }
+}
+
+enum
+{
+    VY_SHADER_STAGE_BIT_begin,
+    
+    VY_SHADER_STAGE_BIT_VERTEX   = 1,
+    VY_SHADER_STAGE_BIT_FRAGMENT = 2,
+    VY_SHADER_STAGE_BIT_COMPUTE  = 3,
+
+    VY_SHADER_STAGE_BIT_count,
+};
+#define VY_SSBIT(n) (1 << (n))
+
+typedef struct {
+    vy_attribute_binding *uniform_bindings;
+    vy_attribute_binding *storage_bindings;
+    vy_attribute_binding *texture_bindings;
+
+    /* Toggels shader stages on or off */
+    uint32_t shader_stage_bitmask;
+    vy_uid vertex_shader;
+    vy_uid fragment_shader;
+    vy_uid compute_shader;
+
+    /* TODO Fixed function settings */
+
+    /* Sampler settings */
+
+    unsigned int uniform_binding_count;
+    unsigned int storage_binding_count;
+    unsigned int texture_binding_count;
+} vy_pipeline_data;
+
+static vy_result ParseShader(vy_parse_state *state,
+                             unsigned int root_list,
+                             const char *name,
+                             const char *file_path,
+                             vy_uid *p_shader_uid) {
+    const vy_parsed_stmt *stmt = FindStatement(state, root_list, name);
+    if (stmt) {
+        if (stmt->form != VY_STMT_FORM_VALUE) {
+            vyReportError("GFX",
+                          "Expected a file name as the value of "
+                          "\"%s\" in %s",
+                          name,
+                          file_path);
+            return VY_PROCESSING_FAILED;
+        }
+        vy_file_id shader_file = vyAddFileFromSpan(stmt->value);
+        vy_uid uid             = vyLookupUID(shader_file);
+        if (uid == VY_INVALID_UID) {
+            /* Add the shader file to processing and wait until its done */
+            if (vyAddFileToProcessingQueue(shader_file) != VY_SUCCESS)
+                return VY_PROCESSING_FAILED;
+            return VY_PROCESSING_TRY_AGAIN;
+        }
+
+        *p_shader_uid = uid;
+        return VY_SUCCESS;
+    }
+    return VY_PROCESSING_FAILED;
+}
+
+static vy_result ParsePipelineFile(vy_file_id fid,
+                                   const char *text,
+                                   size_t length,
+                                   vy_pipeline_data *pipeline) {
+    /* This is the grammar for pipeline files:
+     * <stmt-list> ::= <stmt>*
+     * <stmt> ::= <attribute> ( ( <value> ';' ) | ( '{' <stmt-list> '}' ) )
+     * <attribute> ::= [:alnum:]*
+     * <value>:: = [:alnum:]* */
+    const char *file_path = vyGetFilePath(fid);
+    vy_parse_state state  = {.text                    = text,
+                             .at                      = 0,
+                             .length                  = length,
+                             .line                    = 1,
+                             .file                    = file_path,
+                             .statements              = NULL,
+                             .statement_lists         = NULL,
+                             .statement_capacity      = 0,
+                             .statement_list_capacity = 0};
+
+    vy_result result = VY_SUCCESS;
+    unsigned int root_list = 0;
+    if (!ParseStmtList(&state, &root_list)) {
+        result = false;
+        goto out;
+    }
+
+    DbgPrintShaderFile(&state, root_list, 0);
+
+    /* Process shader stages */
+    pipeline->shader_stage_bitmask = 0;
+    vy_result shader_result;
+    if ((shader_result = ParseShader(&state,
+                                     root_list,
+                                     "vertex",
+                                     file_path,
+                                     &pipeline->vertex_shader)) == VY_SUCCESS) {
+
+        pipeline->shader_stage_bitmask |= VY_SHADER_STAGE_BIT_VERTEX;
+    } else if (shader_result == VY_PROCESSING_TRY_AGAIN) {
+        result = VY_PROCESSING_TRY_AGAIN;
+        goto out;
+    }
+    if ((shader_result = ParseShader(&state,
+                                     root_list,
+                                     "fragment",
+                                     file_path,
+                                     &pipeline->vertex_shader)) == VY_SUCCESS) {
+
+        pipeline->shader_stage_bitmask |= VY_SHADER_STAGE_BIT_FRAGMENT;
+    } else if (shader_result == VY_PROCESSING_TRY_AGAIN) {
+        result = VY_PROCESSING_TRY_AGAIN;
+        goto out;
+    }
+    if ((shader_result = ParseShader(&state,
+                                     root_list,
+                                     "compute",
+                                     file_path,
+                                     &pipeline->vertex_shader)) == VY_SUCCESS) {
+
+        pipeline->shader_stage_bitmask |= VY_SHADER_STAGE_BIT_COMPUTE;
+    } else if (shader_result == VY_PROCESSING_TRY_AGAIN) {
+        result = VY_PROCESSING_TRY_AGAIN;
+        goto out;
+    }
+
+    /* Process bindings */
+    pipeline->texture_bindings    = NULL;
+    pipeline->texture_binding_count = 0;
+    pipeline->uniform_bindings      = NULL;
+    pipeline->uniform_binding_count = 0;
+    pipeline->storage_bindings      = NULL;
+    pipeline->storage_binding_count = 0;
+
+    if (!ParseBindings(&state,
+                       root_list,
+                       "texture_bindings",
+                       file_path,
+                       &pipeline->texture_bindings,
+                       &pipeline->texture_binding_count)) {
+        result = VY_AIO_STATE_FAILED;
+        goto out;
+    }
+
+    if (!ParseBindings(&state,
+                       root_list,
+                       "uniform_bindings",
+                       file_path,
+                       &pipeline->uniform_bindings,
+                       &pipeline->uniform_binding_count)) {
+        result = VY_AIO_STATE_FAILED;
+        goto out;
+    }
+
+    if (!ParseBindings(&state,
+                       root_list,
+                       "storage_bindings",
+                       file_path,
+                       &pipeline->storage_bindings,
+                       &pipeline->storage_binding_count)) {
+        result = VY_AIO_STATE_FAILED;
+        goto out;
+    }
+
+out:
+    free(state.statements);
+    free(state.statement_lists);
+    return result;
+}
+
+vy_result vyProcessPipelineFile(vy_file_id file,
+                                void *buffer,
+                                size_t size,
+                                vy_processor_output *output) {
+    vy_pipeline_data tmp;
+    vy_result result = ParsePipelineFile(file, buffer, size, &tmp);
+    if (result == VY_SUCCESS) {
+    
+    }
+    return result;
+}

src/tools/assetc/processing.h

@@ -21,7 +21,11 @@ vy_result vyAddAssetProcessor(const char *file_extension, vy_processor_fn fn);
 vy_result vyAddFileToProcessingQueue(vy_file_id file);
 
 vy_result vyStartProcessing(void);
-
 void vyStopProcessing(void);
 
+
+void vyInitUIDMap(void);
+void vyAddUIDMapping(vy_file_id fid, vy_uid uid);
+vy_uid vyLookupUID(vy_file_id fid);
+
 #endif

src/tools/assetc/processor.c

@@ -114,7 +114,7 @@ static void ProcessLoadedFile(vy_file_processing_queue_entry entry, void *buffer
                           entry.turn);
                 }
             }
-            continue;
+            return;
         }
     }
     vyLog("ASSETC", "No asset processor for file: %s", path);
@@ -201,6 +201,7 @@ static void ProcessingThread(void *_param) {
                     vyLog("ASSETC",
                           "Loading file %s failed.",
                           vyGetFilePath(submitted_entries[i].fid));
+                    vyReleaseBuffer(submitted_buffers[i], submitted_sizes[i]);
                     PopAndSwapSubmittedData(i,
                                             &submitted_outstanding,
                                             submitted_entries,
@@ -213,6 +214,7 @@ static void ProcessingThread(void *_param) {
                     vyLog("ASSETC",
                           "Got invalid AIO handle for file: %s",
                           vyGetFilePath(submitted_entries[i].fid));
+                    vyReleaseBuffer(submitted_buffers[i], submitted_sizes[i]);
                     PopAndSwapSubmittedData(i,
                                             &submitted_outstanding,
                                             submitted_entries,
@@ -225,6 +227,7 @@ static void ProcessingThread(void *_param) {
                     ProcessLoadedFile(submitted_entries[i],
                                       submitted_buffers[i],
                                       submitted_sizes[i]);
+                    vyReleaseBuffer(submitted_buffers[i], submitted_sizes[i]);
                     PopAndSwapSubmittedData(i,
                                             &submitted_outstanding,
                                             submitted_entries,

src/tools/assetc/shader_processor.c

@@ -1,445 +1,16 @@
-#include <assert.h>
-#include <limits.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "runtime/aio.h"
-#include "runtime/gfx.h"
-#include "runtime/handles.h"
-#include "runtime/runtime.h"
-
 #include "processing.h"
 
-typedef enum {
-    VY_STMT_FORM_VALUE,
-    VY_STMT_FORM_LIST,
-} vy_stmt_form;
-
-typedef struct {
-    unsigned int first;
-    unsigned int count;
-} vy_parsed_stmt_list;
-
-typedef struct {
-    vy_stmt_form form;
-    vy_text_span attribute;
-    union {
-        vy_text_span value;
-        unsigned int list_index;
-    };
-    /* For lists */
-    unsigned int next;
-} vy_parsed_stmt;
-
-typedef struct {
-    const char *file;
-    const char *text;
-    size_t at;
-    size_t length;
-    int line;
-
-    vy_parsed_stmt *statements;
-    unsigned int statement_count;
-    unsigned int statement_capacity;
-
-    vy_parsed_stmt_list *statement_lists;
-    unsigned int statement_list_count;
-    unsigned int statement_list_capacity;
-} vy_parse_state;
-
-static bool IsAllowedChar(char c) {
-    return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
-           (c >= 'A' && c <= 'Z') || (c == '.') || (c == '_') || (c == '/');
-}
-
-static bool IsWhitespace(char c) {
-    return c == ' ' || c == '\t' || c == '\n' || c == '\r';
-}
-
-static void SkipWhitespace(vy_parse_state *state) {
-    while (state->at < state->length && IsWhitespace(state->text[state->at])) {
-        if (state->text[state->at] == '\n')
-            ++state->line;
-        ++state->at;
-    }
-}
-
-static bool ParseAttribute(vy_parse_state *state, vy_text_span *_name) {
-    vy_text_span name;
-    name.start  = &state->text[state->at];
-    name.length = 0;
-    while (state->at < state->length && !IsWhitespace(state->text[state->at])) {
-        if (IsAllowedChar(state->text[state->at])) {
-            ++state->at;
-            ++name.length;
-        } else {
-            vyReportError("GFX",
-                          "%s:%d Unexpected character %c",
-                          state->file,
-                          state->line,
-                          state->text[state->at]);
-            return false;
-        }
-    }
-    *_name = name;
-    return true;
-}
-
-static bool ParseValue(vy_parse_state *state, vy_text_span *_value) {
-    vy_text_span value;
-    value.start  = &state->text[state->at];
-    value.length = 0;
-    while (state->at < state->length && !IsWhitespace(state->text[state->at]) &&
-           state->text[state->at] != ';') {
-        if (IsAllowedChar(state->text[state->at])) {
-            ++state->at;
-            ++value.length;
-        } else {
-            vyReportError("GFX",
-                          "%s:%d Unexpected character %c",
-                          state->file,
-                          state->line,
-                          state->text[state->at]);
-            return false;
-        }
-    }
-    *_value = value;
-    return true;
-}
-
-static bool ParseStmtList(vy_parse_state *state, unsigned int *list_index);
-
-static bool ParseStmt(vy_parse_state *state, unsigned int *stmt_index) {
-    vy_parsed_stmt stmt;
-    stmt.next = UINT_MAX; /* end of list */
-
-    SkipWhitespace(state);
-    if (!ParseAttribute(state, &stmt.attribute))
-        return false;
-    SkipWhitespace(state);
-
-    if (state->at == state->length) {
-        vyReportError("GFX",
-                      "%s:%d Expected either a value or '{'",
-                      state->file,
-                      state->line);
-        return false;
-    }
-
-    if (state->text[state->at] == '{') {
-        /* Consume '{' */
-        ++state->at;
-
-        stmt.form = VY_STMT_FORM_LIST;
-        if (!ParseStmtList(state, &stmt.list_index))
-            return false;
-
-        /* Consume '}' */
-        if (state->at < state->length && state->text[state->at] == '}')
-            ++state->at;
-    } else {
-        stmt.form = VY_STMT_FORM_VALUE;
-        if (!ParseValue(state, &stmt.value))
-            return false;
-
-        /* Consume ';' */
-        if (state->at < state->length && state->text[state->at] == ';')
-            ++state->at;
-    }
-
-    SkipWhitespace(state);
-
-    /* Add statement to array */
-    if (state->statement_count == state->statement_capacity) {
-        unsigned int cap = (state->statement_capacity > 0)
-                               ? state->statement_capacity * 2
-                               : 64;
-        vy_parsed_stmt *temp =
-            realloc(state->statements, sizeof(vy_parsed_stmt) * cap);
-        if (!temp) {
-            vyReportError("GFX",
-                          "While parsing %s: Out of memory\n",
-                          state->file);
-            return false;
-        }
-        state->statements         = temp;
-        state->statement_capacity = cap;
-    }
-    state->statements[state->statement_count] = stmt;
-    *stmt_index                               = state->statement_count++;
-
-    return true;
-}
-
-static bool ParseStmtList(vy_parse_state *state, unsigned int *list_index) {
-    vy_parsed_stmt_list list;
-    list.first = state->statement_count;
-    list.count = 0;
-
-    unsigned int last = UINT_MAX;
-
-    while (state->at < state->length && state->text[state->at] != '}') {
-        unsigned int stmt;
-        if (!ParseStmt(state, &stmt))
-            return false;
-
-        if (last != UINT_MAX)
-            state->statements[last].next = stmt;
-        last = stmt;
-
-        ++list.count;
-    }
-
-    /* Add list to array */
-    if (state->statement_list_count == state->statement_list_capacity) {
-        unsigned int cap = (state->statement_list_capacity > 0)
-                               ? state->statement_list_capacity * 2
-                               : 64;
-        vy_parsed_stmt_list *temp =
-            realloc(state->statement_lists, sizeof(vy_parsed_stmt_list) * cap);
-        if (!temp) {
-            vyReportError("GFX",
-                          "While parsing %s: Out of memory\n",
-                          state->file);
-            return false;
-        }
-        state->statement_lists         = temp;
-        state->statement_list_capacity = cap;
-    }
-    state->statement_lists[state->statement_list_count] = list;
-    *list_index = state->statement_list_count++;
-    return true;
-}
-
-static void DbgPrintShaderFile(const vy_parse_state *state,
-                               unsigned int list_index,
-                               unsigned int indent) {
-    assert(list_index < state->statement_list_count);
-    const vy_parsed_stmt_list *list = &state->statement_lists[list_index];
-
-    unsigned int stmt_index = list->first;
-    for (unsigned int i = 0; i < list->count; ++i) {
-        const vy_parsed_stmt *stmt = &state->statements[stmt_index];
-        for (unsigned int j = 0; j < indent; ++j)
-            printf(" ");
-        printf("%.*s: ", stmt->attribute.length, stmt->attribute.start);
-        if (stmt->form == VY_STMT_FORM_VALUE) {
-            printf("%.*s\n", stmt->value.length, stmt->value.start);
-        } else {
-            printf("{\n");
-            DbgPrintShaderFile(state, stmt->list_index, indent + 2);
-            printf("}\n");
-        }
-        stmt_index = stmt->next;
-    }
-    assert(stmt_index == UINT_MAX || stmt_index == 0);
-}
-
-static bool CompareSpanToString(vy_text_span span, const char *cmp) {
-    size_t cmp_len = strlen(cmp);
-    if (cmp_len != (size_t)span.length)
-        return false;
-    for (size_t i = 0; i < cmp_len; ++i) {
-        if (span.start[i] != cmp[i])
-            return false;
-    }
-    return true;
-}
-
-static const vy_parsed_stmt *FindStatement(const vy_parse_state *state,
-                                           unsigned int list_index,
-                                           const char *attribute) {
-    if (list_index >= state->statement_list_count)
-        return NULL;
-    const vy_parsed_stmt_list *list = &state->statement_lists[list_index];
-    unsigned int stmt_index         = list->first;
-    for (unsigned int i = 0; i < list->count; ++i) {
-        const vy_parsed_stmt *stmt = &state->statements[stmt_index];
-        if (CompareSpanToString(stmt->attribute, attribute))
-            return stmt;
-        stmt_index = stmt->next;
-    }
-    return NULL;
-}
-
-static bool ParseBindingIndex(vy_text_span span, unsigned int *index) {
-    if (span.length == 0)
-        return false;
-    int at           = (int)span.length - 1;
-    unsigned int exp = 1;
-    unsigned int n   = 0;
-    while (at >= 0) {
-        if (span.start[at] >= '0' && span.start[at] <= '9') {
-            unsigned int digit = (unsigned int)(span.start[at] - '0');
-            n += digit * exp;
-        } else {
-            vyReportError("GFX",
-                          "Unexpected non-digit character in binding index");
-            return false;
-        }
-        --at;
-        exp *= 10;
-    }
-    *index = n;
-    return true;
-}
-
-static vy_attribute_value ParseBindingValue(vy_text_span span) {
-    if (CompareSpanToString(span, "MATERIAL_ALBEDO")) {
-        return VY_ATTRIBUTE_VALUE_MATERIAL_ALBEDO;
-    } else if (CompareSpanToString(span, "MATERIAL_NORMAL")) {
-        return VY_ATTRIBUTE_VALUE_MATERIAL_NORMAL;
-    }
-    vyReportError("GFX",
-                  "Unsupported binding value %*.s",
-                  span.length,
-                  span.start);
-    return VY_ATTRIBUTE_VALUE_UNDEFINED;
-}
-
-static bool ParseBindings(vy_parse_state *state,
-                          unsigned int root_list,
-                          const char *name,
-                          const char *file_path,
-                          vy_attribute_binding **p_bindings,
-                          unsigned int *p_binding_count) {
-    const vy_parsed_stmt *bindings = FindStatement(state, root_list, name);
-    if (bindings) {
-        if (bindings->form != VY_STMT_FORM_LIST) {
-            vyReportError("GFX",
-                          "Expected list of bindings as the value of "
-                          "\"%s\" in %s",
-                          name,
-                          file_path);
-            return false;
-        }
-        const vy_parsed_stmt_list *binding_list =
-            &state->statement_lists[bindings->list_index];
-        vy_attribute_binding *shader_bindings =
-            malloc(sizeof(vy_attribute_binding) * binding_list->count);
-        if (!bindings) {
-            vyReportError("GFX", "Out of memory");
-            return false;
-        }
-        unsigned int binding_count = binding_list->count;
-
-        unsigned int stmt_index = binding_list->first;
-        for (unsigned int i = 0; i < binding_list->count; ++i) {
-            const vy_parsed_stmt *stmt = &state->statements[stmt_index];
-            if (!ParseBindingIndex(stmt->attribute,
-                                   &shader_bindings[i].index)) {
-                free(shader_bindings);
-                return false;
-            }
-
-            shader_bindings[i].value = ParseBindingValue(stmt->value);
-            if (shader_bindings[i].value == VY_ATTRIBUTE_VALUE_UNDEFINED) {
-                free(shader_bindings);
-                return false;
-            }
-            stmt_index = stmt->next;
-        }
-
-        *p_bindings      = shader_bindings;
-        *p_binding_count = binding_count;
-        return true;
-    } else {
-        *p_bindings      = NULL;
-        *p_binding_count = 0;
-        return true;
-    }
-}
-
-static bool
-ParseShaderFile(vy_file_id fid, const char *text, size_t length, vy_shader *shader) {
-    /* This is the grammar for shader files:
-     * <stmt-list> ::= <stmt>*
-     * <stmt> ::= <attribute> ( ( <value> ';' ) | ( '{' <stmt-list> '}' ) )
-     * <attribute> ::= [:alnum:]*
-     * <value>:: = [:alnum:]* */
-    const char *file_path = vyGetFilePath(fid);
-    vy_parse_state state  = {.text                    = text,
-                             .at                      = 0,
-                             .length                  = length,
-                             .line                    = 1,
-                             .file                    = file_path,
-                             .statements              = NULL,
-                             .statement_lists         = NULL,
-                             .statement_capacity      = 0,
-                             .statement_list_capacity = 0};
-
-    bool result            = true;
-    unsigned int root_list = 0;
-    if (!ParseStmtList(&state, &root_list)) {
-        result = false;
-        goto out;
-    }
-
-    DbgPrintShaderFile(&state, root_list, 0);
-
-    #if 0
-    shader->pipeline = CreatePipeline(&state, file_path, root_list);
-    if (!VY_IS_HANDLE_VALID(shader->pipeline)) {
-        result = false;
-        goto out;
-    }
-    #endif
-
-    /* Process bindings */
-    shader->texture_bindings      = NULL;
-    shader->texture_binding_count = 0;
-    shader->uniform_bindings      = NULL;
-    shader->uniform_binding_count = 0;
-    shader->storage_bindings      = NULL;
-    shader->storage_binding_count = 0;
-
-    if (!ParseBindings(&state,
-                       root_list,
-                       "texture_bindings",
-                       file_path,
-                       &shader->texture_bindings,
-                       &shader->texture_binding_count)) {
-        result = false;
-        goto out;
-    }
-
-    if (!ParseBindings(&state,
-                       root_list,
-                       "uniform_bindings",
-                       file_path,
-                       &shader->uniform_bindings,
-                       &shader->uniform_binding_count)) {
-        result = false;
-        goto out;
-    }
-
-    if (!ParseBindings(&state,
-                       root_list,
-                       "storage_bindings",
-                       file_path,
-                       &shader->storage_bindings,
-                       &shader->storage_binding_count)) {
-        result = false;
-        goto out;
-    }
-
-out:
-    free(state.statements);
-    free(state.statement_lists);
-    return result;
-}
-
-vy_result vyProcessShaderFile(vy_file_id file,
+extern vy_result vyProcessShaderFile(vy_file_id file,
                                      void *buffer,
                                      size_t size,
                                      vy_processor_output *output) {
-    vy_shader tmp;
-    if (ParseShaderFile(file, buffer, size, &tmp)) {
+    /* Scan the first line for the shader type */
+    char *text = buffer;
+    for (size_t i = 0; i < size; ++i) {
+        if (text[i] == '\n' || text[i] == '\r') {
+            vyLog("ASSETC", "First Line");
+        }
+    }
+
     return VY_SUCCESS;
-    } else {
-        return VY_PROCESSING_FAILED;
-    }
 }

src/tools/assetc/uidmap.c

@@ -0,0 +1,57 @@
+#include "processing.h"
+
+#include "runtime/threading.h"
+
+#define MAP_SIZE 2048
+typedef struct {
+    vy_file_id fids[MAP_SIZE];
+    vy_uid uids[MAP_SIZE];
+
+    unsigned int used_slots;
+} vy_uid_map;
+
+static vy_uid_map _map;
+static vy_mutex *_guard;
+
+void vyInitUIDMap(void) {
+    _guard = vyCreateMutex();
+}
+
+vy_uid vyLookupUID(vy_file_id fid) {
+    vyLockMutex(_guard);
+    unsigned int i = 0;
+    vy_uid result  = VY_INVALID_UID;
+    while (i < MAP_SIZE) {
+        unsigned int slot = (fid + i) % MAP_SIZE;
+        if (_map.fids[slot] == fid) {
+            _map.fids[slot] = fid;
+            result          = _map.uids[slot];
+            break;
+        } else if (_map.fids[slot] == VY_INVALID_FILE_ID) {
+            break;
+        }
+    }
+    vyUnlockMutex(_guard);
+    return result;
+}
+
+void vyAddUIDMapping(vy_file_id fid, vy_uid uid) {
+    vyLockMutex(_guard);
+    float fill_rate = (float)_map.used_slots / MAP_SIZE;
+    if (fill_rate >= .5f) {
+        vyLog("ASSETC", "UID map is above 50% filled.");
+    }
+    unsigned int i = 0;
+    while (i < MAP_SIZE) {
+        unsigned int slot = (fid + i) % MAP_SIZE;
+        if (_map.fids[slot] == 0) {
+            _map.fids[slot] = fid;
+            _map.uids[slot] = uid;
+            break;
+        }
+    }
+    if (i == MAP_SIZE) {
+        vyReportError("ASSETC", "Failed to insert entry into UID map.");
+    }
+    vyUnlockMutex(_guard);
+}