don't keep processed assets in memory

- Slightly slower, but this will (in principle) scale to an unlimited
  number of assets

src/runtime/app.c

@@ -5,8 +5,6 @@
 #include "gfx.h"
 #include "renderer_api.h"
 
-extern void __RegisterRuntimeCVars(void);
-
 VY_CVAR_I(rt_Fullscreen, "Show window in fullscreen mode. [0/1] Default: 0", 0);
 VY_CVAR_I(rt_WindowWidth, "Window width. Default: 1024", 1024);
 VY_CVAR_I(rt_WindowHeight, "Window height. Default: 768", 768);

src/runtime/config.h

@@ -22,6 +22,7 @@ typedef struct {
     vy_cvar_type type;
 } vy_cvar;
 
+/* n variable name, d description string, v default value*/
 #define VY_CVAR_I(n, d, v)                                                                         \
     vy_cvar n = {.name = #n, .description = d, .i = (v), .type = VY_CVAR_TYPE_INT}
 #define VY_CVAR_F(n, d, v)                                                                         \

src/tools/assetc/discovery.c

@@ -74,13 +74,7 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
                     vyLog("ASSETC", "Failed to lookup UID of known asset %s", data->path_scratch);
                     return VY_UNKNOWN_ERROR;
                 }
-                void *buffer;
+                vyAddUnchangedAssetToPackage(settings.package, uid);
-                size_t size;
-                if (vyLoadAssetDirect(uid, &buffer, &size) != VY_SUCCESS) {
-                    vyLog("ASSETC", "Failed to load asset %s", data->path_scratch);
-                    return VY_UNKNOWN_ASSET;
-                }
-                vyAddAssetToPackage(settings.package, uid, buffer, size);
             }
         } else {
             /* Process it */

src/tools/assetc/packages.c

@@ -1,5 +1,6 @@
 #include "packages.h"
 #include "processing.h"
+#include "utils.h"
 
 #define VY_DEFINE_PACKAGE_FILE_STRUCTURES
 #include "runtime/threading.h"
@@ -19,9 +20,7 @@
 
 typedef struct {
     vy_uid uid;
-    void *buffer;
+    size_t disk_size;
-    size_t buffer_size;
-    bool needs_compression;
 } vy_package_entry;
 
 typedef struct {
@@ -105,12 +104,65 @@ static void AddAssetToPackageImpl(unsigned int package,
         pkg->entry_capacity = new_cap;
         pkg->entries        = n;
     }
-    pkg->entries[pkg->num_entries].buffer            = buffer;
+
-    pkg->entries[pkg->num_entries].buffer_size       = size;
+    char tmp_path[256];
+    snprintf(tmp_path, 256, "actemp/%u.bin", uid);
+    if (needs_compression) {
+        FILE *tmp_f = fopen(tmp_path, "wb");
+        if (!tmp_f) {
+            vyUnlockMutex(_guard);
+            return;
+        }
+
+        int required_size       = LZ4_compressBound((int)size);
+        void *compressed_buffer = malloc(required_size);
+        if (!compressed_buffer) {
+            fclose(tmp_f);
+            vyUnlockMutex(_guard);
+            return;
+        }
+
+        int compressed_bytes =
+            LZ4_compress_default(buffer, compressed_buffer, (int)size, required_size);
+        if (compressed_bytes == 0) {
+            free(compressed_buffer);
+            fclose(tmp_f);
+            vyReportError("ASSETC", "Failed to compress asset %x of package %s", uid, pkg->name);
+            return;
+        }
+
+        vy_package_asset_header header;
+        XXH64_hash_t checksum = XXH3_64bits_withSeed(buffer, (size_t)compressed_bytes, 0);
+        XXH64_canonicalFromHash(&header.checksum, checksum);
+        header.decompressed_size = (uint32_t)size;
+
+        if (fwrite(&header, sizeof(header), 1, tmp_f) != 1) {
+            vyReportError("ASSETC", "Failed to write to actemp/%u.bin", uid);
+            free(compressed_buffer);
+            fclose(tmp_f);
+            return;
+        }
+        if (fwrite(buffer, compressed_bytes, 1, tmp_f) != 1) {
+            vyReportError("ASSETC", "Failed to write to actemp/%u.bin", uid);
+            free(compressed_buffer);
+            fclose(tmp_f);
+            return;
+        }
+        fclose(tmp_f);
+
+        pkg->entries[pkg->num_entries].disk_size =
+            (size_t)compressed_bytes + sizeof(vy_package_asset_header);
+    } else {
+        pkg->entries[pkg->num_entries].disk_size = vyGetFileSize(tmp_path);
+        if (pkg->entries[pkg->num_entries].disk_size == 0) {
+            vyReportError("ASSETC", "Failed to determine size of actemp/%u.bin", uid);
+        }
+    }
+    
     pkg->entries[pkg->num_entries].uid               = uid;
-    pkg->entries[pkg->num_entries].needs_compression = needs_compression;
     ++pkg->num_entries;
 
+
     vyUnlockMutex(_guard);
 }
 
@@ -118,8 +170,8 @@ void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t
     AddAssetToPackageImpl(package, uid, buffer, size, true);
 }
 
-void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size) {
+void vyAddUnchangedAssetToPackage(unsigned int package, vy_uid uid) {
-    AddAssetToPackageImpl(package, uid, buffer, size, false);
+    AddAssetToPackageImpl(package, uid, NULL, 0, false);
 }
 
 static vy_result SavePackage(vy_package *pkg) {
@@ -128,8 +180,8 @@ static vy_result SavePackage(vy_package *pkg) {
         return VY_SUCCESS;
     }
 
-    int current_buffer_size = 0;
+    size_t current_buffer_size = 0;
-    void *buffer            = NULL;
+    void *buffer               = NULL;
 
     size_t offset_in_file = 0;
 
@@ -142,75 +194,52 @@ static vy_result SavePackage(vy_package *pkg) {
     }
 
     for (unsigned int i = 0; i < pkg->num_entries; ++i) {
-
+        char tmp_path[256];
-        if (pkg->entries[i].needs_compression) {
+        snprintf(tmp_path, 256, "actemp/%u.bin", pkg->entries[i].uid);
-            int required_size = LZ4_compressBound((int)pkg->entries[i].buffer_size);
+        FILE *tmp_f = fopen(tmp_path, "rb");
-            if (required_size > current_buffer_size) {
+        if (!tmp_f) {
-                void *tmp = realloc(buffer, (size_t)required_size);
+            vyReportError("ASSETC", "Failed to open %s for reading.", tmp_path);
-                if (!tmp)
+            fclose(f);
-                    return VY_UNKNOWN_ERROR;
+            free(buffer);
-                buffer              = tmp;
+            return VY_UNKNOWN_ERROR;
-                current_buffer_size = required_size;
-            }
-            int compressed_bytes = LZ4_compress_default(pkg->entries[i].buffer,
-                                                        buffer,
-                                                        (int)pkg->entries[i].buffer_size,
-                                                        current_buffer_size);
-            if (compressed_bytes == 0) {
-                free(buffer);
-                vyReportError("ASSETC", "Failed to compress entry %u of package %s", i, pkg->name);
-                return VY_UNKNOWN_ERROR;
-            }
-            vy_package_asset_header header;
-            XXH64_hash_t checksum = XXH3_64bits_withSeed(buffer, (size_t)compressed_bytes, 0);
-            XXH64_canonicalFromHash(&header.checksum, checksum);
-            header.decompressed_size = (uint32_t)pkg->entries[i].buffer_size;
-
-            if (fwrite(&header, sizeof(header), 1, f) != 1) {
-                vyReportError("ASSETC", "Failed to write to %s", pkg->name);
-                free(buffer);
-                fclose(f);
-                return VY_UNKNOWN_ERROR;
-            }
-            if (fwrite(buffer, compressed_bytes, 1, f) != 1) {
-                vyReportError("ASSETC", "Failed to write to %s", pkg->name);
-                free(buffer);
-                fclose(f);
-                return VY_UNKNOWN_ERROR;
-            }
-
-            vyAddUIDTabEntry(package_fid,
-                             pkg->entries[i].uid,
-                             offset_in_file,
-                             (size_t)compressed_bytes + sizeof(header));
-
-            offset_in_file += sizeof(header) + (size_t)compressed_bytes;
-        } else {
-            vy_package_asset_header header;
-            XXH64_hash_t checksum = XXH3_64bits_withSeed(pkg->entries[i].buffer, pkg->entries[i].buffer_size, 0);
-            XXH64_canonicalFromHash(&header.checksum, checksum);
-            header.decompressed_size = (uint32_t)pkg->entries[i].buffer_size;
-
-            if (fwrite(&header, sizeof(header), 1, f) != 1) {
-                vyReportError("ASSETC", "Failed to write to %s", pkg->name);
-                free(buffer);
-                fclose(f);
-                return VY_UNKNOWN_ERROR;
-            }
-            if (fwrite(pkg->entries[i].buffer, pkg->entries[i].buffer_size, 1, f) != 1) {
-                vyReportError("ASSETC", "Failed to write to %s", pkg->name);
-                free(buffer);
-                fclose(f);
-                return VY_UNKNOWN_ERROR;
-            }
-
-            vyAddUIDTabEntry(package_fid,
-                             pkg->entries[i].uid,
-                             offset_in_file,
-                             pkg->entries[i].buffer_size + sizeof(header));
-
-            offset_in_file += sizeof(header) + pkg->entries[i].buffer_size;
         }
+
+        if (current_buffer_size < pkg->entries[i].disk_size) {
+            void *tmp = realloc(buffer, pkg->entries[i].disk_size);
+            if (!tmp) {
+                vyReportError("ASSETC", "Failed to allocate buffer (%zu bytes) for reading %s.", pkg->entries[i].disk_size, tmp_path);
+                fclose(f);
+                fclose(tmp_f);
+                free(buffer);
+                return VY_UNKNOWN_ERROR;
+            }
+            buffer = tmp;
+            current_buffer_size = pkg->entries[i].disk_size;
+        }
+
+        if (fread(buffer, pkg->entries[i].disk_size, 1, tmp_f) != 1) {
+            vyReportError("ASSETC", "Failed to read %s.", tmp_path);
+            fclose(f);
+            fclose(tmp_f);
+            free(buffer);
+            return VY_UNKNOWN_ERROR;
+        }
+
+        if (fwrite(buffer, pkg->entries[i].disk_size, 1, f) != 1) {
+            vyReportError("ASSETC", "Failed to write (%zu bytes) to %s.", pkg->entries[i].disk_size, pkg->name);
+            fclose(f);
+            fclose(tmp_f);
+            free(buffer);
+            return VY_UNKNOWN_ERROR;
+        }
+
+        vyAddUIDTabEntry(package_fid,
+                         pkg->entries[i].uid,
+                         offset_in_file,
+                         pkg->entries[i].disk_size);
+        offset_in_file += pkg->entries[i].disk_size;
+
+        fclose(tmp_f);
     }
 
     free(buffer);

src/tools/assetc/packages.h

@@ -9,7 +9,7 @@ void vyInitPackages(void);
 unsigned int vyAddPackageFile(vy_text_span name);
 
 void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size);
-void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size);
+void vyAddUnchangedAssetToPackage(unsigned int package, vy_uid uid);
 
 vy_result vySavePackages(void);