assetc complete "lifetime"

- Add unchanged assets to packages
- Allows re-runs without messing up the data
- Inefficient: unchanged assets need to be loaded & kept in memory

.gitignore

@@ -2,3 +2,6 @@
 /subprojects/*
 !/subprojects/*.wrap
 
+# assetc directories
+/actemp/*
+/data/*

assets/shader/cell.pipeline

@@ -1,10 +1,10 @@
 optimization speed;
 
 vertex {
-    vk shader/cell_vert.glsl;
+    vk assets/shader/cell_vert.glsl;
 }
 fragment {
-    vk shader/cell_frag.glsl;
+    vk assets/shader/cell_frag.glsl;
 }
 
 texture_bindings {

meson.build

@@ -59,6 +59,7 @@ runtime_lib = library('vyrt',
   'src/runtime/file_tab.h',
   'src/runtime/gfx.h',
   'src/runtime/jobs.h',
+  'src/runtime/packages.h',
   'src/runtime/renderer_api.h',
   'src/runtime/runtime.h',
   'src/runtime/threading.h',
@@ -67,6 +68,7 @@ runtime_lib = library('vyrt',
   'src/runtime/aio.c',
   'src/runtime/app.c',
   'src/runtime/asset_dependencies.c',
+  'src/runtime/asset_loading.c',
   'src/runtime/buffer_manager.c',
   'src/runtime/config.c',
   'src/runtime/error_report.c',
@@ -74,6 +76,7 @@ runtime_lib = library('vyrt',
   'src/runtime/gfx_main.c',
   'src/runtime/init.c',
   'src/runtime/jobs.c',
+  'src/runtime/packages.c',
   'src/runtime/text.c',
   'src/runtime/threading_cond.c',
   'src/runtime/threading_mutex.c',

src/runtime/asset_dependencies.c

@@ -25,7 +25,7 @@ static vy_uid *_list_mem;
 
 vy_result LoadAssetDependencies(void) {
     vy_dependency_file_header header;
-    vy_file_id fid = vyAddFile("assets/deps.bin");
+    vy_file_id fid = vyAddFile("data/deps.bin");
 
     if (vySubmitSingleLoadSync((vy_file_load){.dest      = &header,
                                               .num_bytes = sizeof(header),

src/runtime/asset_loading.c

@@ -0,0 +1,62 @@
+#include "assets.h"
+#include "uidtab.h"
+#include "aio.h"
+#include "buffer_manager.h"
+
+#define VY_DEFINE_PACKAGE_FILE_STRUCTURES
+#include "packages.h"
+
+#include "lz4/lz4.h"
+
+VY_DLLEXPORT vy_result vyLoadAssetDirect(vy_uid uid, void **p_buffer, size_t *p_size) {
+    const vy_uid_data *data = vyGetUIDData(uid);
+    if (!data)
+        return VY_UNKNOWN_ASSET;
+
+    void *compressed_buffer = vyAllocBuffer(data->size);
+    if (!compressed_buffer) {
+        return VY_BUFFER_ALLOC_FAILED;
+    }
+
+    if (vySubmitSingleLoadSync((vy_file_load) {
+            .file      = data->pkg_file,
+            .offset    = data->offset,
+            .num_bytes = data->size,
+            .dest      = compressed_buffer,
+        }) != VY_AIO_STATE_FINISHED) {
+        vyReleaseBuffer(compressed_buffer, data->size);
+        return VY_LOAD_FAILED;
+    }
+    
+    const vy_package_asset_header *header = compressed_buffer;
+    
+    size_t compressed_size       = (data->size) - sizeof(*header);
+    XXH64_hash_t calculated_hash = XXH3_64bits((header + 1), compressed_size);
+    XXH64_hash_t file_hash       = XXH64_hashFromCanonical(&header->checksum);
+    if (calculated_hash != file_hash) {
+        vyLog("core", "Checksum mismatch for asset %u", uid);
+        vyReleaseBuffer(compressed_buffer, data->size);
+        return VY_LOAD_FAILED;
+    }
+
+    size_t size               = (size_t)header->decompressed_size;
+    void *decompressed_buffer = vyAllocBuffer(size);
+    if (!decompressed_buffer) {
+        vyReleaseBuffer(compressed_buffer, data->size);
+        return VY_BUFFER_ALLOC_FAILED;
+    }
+
+    if (LZ4_decompress_safe((const char *)(header + 1),
+                            (char *)decompressed_buffer,
+                            (int)compressed_size,
+                            (int)size) < 0) {
+        vyReleaseBuffer(compressed_buffer, data->size);
+        return VY_LOAD_FAILED;
+    }
+    
+    vyReleaseBuffer(compressed_buffer, data->size);
+    *p_buffer = decompressed_buffer;
+    *p_size   = size;
+
+    return VY_SUCCESS;
+}

src/runtime/init.c

@@ -36,6 +36,7 @@ extern vy_result LoadUIDTable(void);
 extern void ReleaseUIDTable(void);
 extern vy_result LoadAssetDependencies(void); 
 extern void ReleaseAssetDependencies(void);
+extern vy_result LoadPackageNames(void);
 
 VY_DLLEXPORT vy_result vyInitRuntime(void) {
     SetMainThreadId();
@@ -66,6 +67,10 @@ VY_DLLEXPORT vy_result vyInitRuntime(void) {
         return res;
     }
 
+    if ((res = LoadPackageNames()) != VY_SUCCESS) {
+        vyLog("CORE", "LoadPackageNames returned result: %u", res);
+    }
+
     return VY_SUCCESS;
 }
 

src/runtime/packages.c

@@ -0,0 +1,20 @@
+#include <stdio.h>
+#include "file_tab.h"
+
+vy_result LoadPackageNames(void) {
+    FILE *f = fopen("data/packages.txt", "r");
+    if (!f) {
+        return VY_UNKNOWN_ERROR;
+    }
+
+    while (!feof(f)) {
+        char line[256];
+        fscanf(f, "%255s\n", line);
+        line[255] = '\0';
+        vyAddFile(line);
+    }
+
+    fclose(f);
+
+    return VY_SUCCESS;
+}

src/runtime/packages.h

@@ -0,0 +1,18 @@
+#ifndef VY_PACKAGES_H
+#define VY_PACKAGES_H
+
+#ifdef VY_DEFINE_PACKAGE_FILE_STRUCTURES
+
+#include <stdint.h>
+#include "xxhash/xxhash.h"
+
+#pragma pack(push, 1)
+typedef struct {
+    XXH64_canonical_t checksum;
+    uint32_t decompressed_size;
+} vy_package_asset_header;
+#pragma pack(pop)
+
+#endif
+
+#endif

src/runtime/uidtab.c

@@ -21,7 +21,7 @@ vy_result LoadUIDTable(void) {
     /* We use stdio here, because we cannot load any asset in parallel to this.
      * This is because the uidtab is what tells us which assets exist.
      */
-    FILE *f = fopen("assets/uidtab.bin", "rb");
+    FILE *f = fopen("data/uidtab.bin", "rb");
     if (!f)
         return VY_LOAD_FAILED;
 

src/tools/assetc/assetc.c

@@ -100,7 +100,7 @@ int main(int argc, char **argv) {
         return 1;
     }
     vyInitPackages();
-    vySetWorkingDirectory(g_assetc_options.root_directory);
+    
 
     if (vyLoadAssetMeta() != VY_SUCCESS) {
         return 1;
@@ -119,6 +119,12 @@ int main(int argc, char **argv) {
         return 1;
     }
 
+    /* Create necessary directories */
+    vySetWorkingDirectory(g_assetc_options.root_directory);
+    vyCreateDirectory("assets");
+    vyCreateDirectory("actemp");
+    vyCreateDirectory("data");
+
     /* "Mainloop" */
     vyDiscoverAssets();
 

src/tools/assetc/assetmeta.c

@@ -1,5 +1,6 @@
 #include "assetmeta.h"
 #include "utils.h"
+#include "options.h"
 
 #include "runtime/threading.h"
 #include "runtime/aio.h"
@@ -7,6 +8,7 @@
 
 #include <xxhash/xxhash.h>
 #include <stdio.h>
+#include <string.h>
 
 #define MAP_SIZE 2048
 typedef struct {
@@ -40,7 +42,7 @@ vy_result vyLoadAssetMeta(void) {
     _guard = vyCreateMutex();
     
     /* Load the meta file */
-    size_t fsz = vyGetFileSize("meta.bin");
+    size_t fsz = vyGetFileSize("actemp/meta.bin");
     if (fsz == 0) {
         vyLog("ASSETC", "Metadata file 'meta.bin' not found. All assets will be processed.");
         return VY_SUCCESS;
@@ -54,7 +56,7 @@ vy_result vyLoadAssetMeta(void) {
     
     vy_load_batch load;
     load.loads[0].dest      = buffer;
-    load.loads[0].file = vyAddFile("meta.bin");
+    load.loads[0].file      = vyAddFile("actemp/meta.bin");
     load.loads[0].num_bytes = fsz;
     load.loads[0].offset    = 0;
     load.num_loads          = 1;
@@ -128,7 +130,7 @@ vy_result vySaveAssetMeta(void) {
     XXH64_canonicalFromHash(&header.checksum, hash);
     header._reserved = 0;
 
-    FILE *f = fopen("meta.bin", "wb");
+    FILE *f = fopen("actemp/meta.bin", "wb");
     if (!f) {
         vyReportError("ASSETC", "Failed to open 'meta.bin'");
         vyReleaseBuffer(entries, sizeof(vy_assetmeta_entry) * header.num_entries);

src/tools/assetc/dependency_tracking.c

@@ -151,7 +151,7 @@ vy_result vySaveAssetDependencies(void) {
         }
     }
     
-    FILE *f = fopen("deps.bin", "wb");
+    FILE *f = fopen("data/deps.bin", "wb");
     if (!f) {
         vyReportError("ASSETC", "Failed to open 'deps.bin' for writing.");
         return VY_UNKNOWN_ERROR;

src/tools/assetc/discovery.c

@@ -33,6 +33,8 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
             else if (memcmp(&name[name_len - 4], ".bin", 4) == 0)
                 return VY_SUCCESS;
         }
+        if (strcmp(name, "packages.txt") == 0)
+            return VY_SUCCESS;
         if (name[0] == '.') {
             return VY_SUCCESS;
         }
@@ -68,12 +70,17 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
 
                 /* We need to load the processed data and add it to the package */
                 vy_uid uid = vyLookupUID(fid);
-                const vy_uid_data *asset_data = vyGetUIDData(uid);
+                if (uid == VY_INVALID_UID) {
-                if (!data) {
+                    vyLog("ASSETC", "Failed to lookup UID of known asset %s", data->path_scratch);
-                    vyLog("ASSETC", "No data available for asset %s (%u)", data->path_scratch, uid);
                     return VY_UNKNOWN_ERROR;
                 }
-                
+                void *buffer;
+                size_t size;
+                if (vyLoadAssetDirect(uid, &buffer, &size) != VY_SUCCESS) {
+                    vyLog("ASSETC", "Failed to load asset %s", data->path_scratch);
+                    return VY_UNKNOWN_ASSET;
+                }
+                vyAddAssetToPackage(settings.package, uid, buffer, size);
             }
         } else {
             /* Process it */
@@ -109,10 +116,10 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
 }
 
 void vyDiscoverAssets(void) {
-    vy_discovery_data data;
-    data.path_scratch[0] = '.';
-    data.path_scratch[1] = '\0';
-    data.path_end        = 0;
 
-    vyIterateDirectory(".", &data, DirectoryHandler);
+    vy_discovery_data data;
+    memcpy(data.path_scratch, "assets", sizeof("assets"));
+    data.path_end = sizeof("assets") - 1;
+    
+    vyIterateDirectory("assets", &data, DirectoryHandler);
 }

src/tools/assetc/packages.c

@@ -1,9 +1,11 @@
 #include "packages.h"
 #include "processing.h"
 
+#define VY_DEFINE_PACKAGE_FILE_STRUCTURES
 #include "runtime/threading.h"
 #include "runtime/assets.h"
 #include "runtime/file_tab.h"
+#include "runtime/packages.h"
 
 #include <assert.h>
 #include <limits.h>
@@ -40,14 +42,14 @@ unsigned int vyAddPackageFile(vy_text_span name) {
     vyLockMutex(_guard);
 
     for (unsigned int i = 0; i < _package_count; ++i) {
-        if (vyCompareSpanToString(name, _packages[i].name) == 0) {
+        if (vyCompareSpanToString(name, _packages[i].name + 5) == 0) {
             vyUnlockMutex(_guard);
             return i;
         }
     }
 
     /* Create a new package */
-    _packages[_package_count].name = malloc(name.length + 1);
+    _packages[_package_count].name = malloc(name.length + 1 + 5);
     if (!_packages[_package_count].name) {
         vyReportError("ASSETC",
                       "Failed to allocate storage for new package %*.s",
@@ -56,8 +58,9 @@ unsigned int vyAddPackageFile(vy_text_span name) {
         vyUnlockMutex(_guard);
         return UINT_MAX;
     }
-    memcpy(_packages[_package_count].name, name.start, name.length);
+    memcpy(_packages[_package_count].name, "data/", 5);
-    _packages[_package_count].name[name.length] = '\0';
+    memcpy(_packages[_package_count].name + 5, name.start, name.length);
+    _packages[_package_count].name[name.length + 5] = '\0';
 
     unsigned int index = _package_count++;
 
@@ -119,13 +122,6 @@ void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffe
     AddAssetToPackageImpl(package, uid, buffer, size, false);
 }
 
-#pragma pack(push, 1)
-typedef struct {
-    XXH64_canonical_t checksum;
-    uint32_t decompressed_size;
-} vy_package_asset_header;
-#pragma pack(pop)
-
 static vy_result SavePackage(vy_package *pkg) {
     if (pkg->num_entries == 0) {
         vyLog("ASSETC", "Package %s has no entries.", pkg->name);
@@ -226,6 +222,20 @@ static vy_result SavePackage(vy_package *pkg) {
 vy_result vySavePackages(void) {
     assert(vyIsMainThread());
 
+    /* Save a .txt file with one line per package.
+     * Enables us to re-init the file-tab in future runs. */
+    FILE *f = fopen("data/packages.txt", "w");
+    if (!f) {
+        vyReportError("ASSETC", "Failed to write to 'packages.txt'");
+        return VY_UNKNOWN_ERROR;
+    }
+    for (unsigned int i = 0; i < _package_count; ++i) {
+        if (_packages[i].num_entries == 0)
+            continue;
+        fprintf(f, "%s\n", _packages[i].name);
+    }
+    fclose(f);
+
     for (unsigned int i = 0; i < _package_count; ++i) {
         vy_result res = SavePackage(&_packages[i]);
         if (res != VY_SUCCESS)

src/tools/assetc/uidtable.c

@@ -47,7 +47,7 @@ vy_result vyWriteUIDTab(void) {
     XXH64_canonicalFromHash(&header.checksum, checksum);
     header.num_entries = (uint32_t)_entry_count;
     
-    FILE *f = fopen("uidtab.bin", "wb");
+    FILE *f = fopen("data/uidtab.bin", "wb");
     if (!f) {
         vyReportError("ASSETC", "Failed to open 'uidtab.bin' for writing.");
         return VY_UNKNOWN_ERROR;

src/tools/assetc/utils.c

@@ -87,3 +87,15 @@ vy_result vyIterateDirectory(const char *path, void *user, vy_iterate_directory_
     return res;
 #endif
 }
+
+vy_result vyCreateDirectory(const char *path) {
+#ifdef _WIN32
+    WCHAR wpath[MAX_PATH];
+    vy_result res = vyUTF8ToWStr(path, wpath, MAX_PATH);
+    if (res != VY_SUCCESS)
+        return res;
+    if (!CreateDirectoryW(wpath, NULL))
+        res = VY_UNKNOWN_ERROR;
+    return res;
+#endif
+}

src/tools/assetc/utils.h

@@ -22,4 +22,6 @@ typedef vy_result vy_iterate_directory_cb_func(const char *name, vyIterateDirEle
 
 vy_result vyIterateDirectory(const char *path, void *user, vy_iterate_directory_cb_func iterate_cb);
 
+vy_result vyCreateDirectory(const char *path);
+
 #endif