kevin/rtengine
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

don't keep processed assets in memory

Browse Source 
- Slightly slower, but this will (in principle) scale to an unlimited
  number of assets
This commit is contained in:
Kevin Trogant 2023-12-25 23:32:55 +01:00
parent 5a4177c4dc
commit bf2129f92b
5 changed files with 110 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/runtime/app.c
View File

@ -5,8 +5,6 @@
#include "gfx.h"
#include "renderer_api.h"
extern void __RegisterRuntimeCVars(void);
VY_CVAR_I(rt_Fullscreen, "Show window in fullscreen mode. [0/1] Default: 0", 0);
VY_CVAR_I(rt_WindowWidth, "Window width. Default: 1024", 1024);
VY_CVAR_I(rt_WindowHeight, "Window height. Default: 768", 768);

1
src/runtime/config.h
View File

@ -22,6 +22,7 @@ typedef struct {
vy_cvar_type type;
} vy_cvar;
/* n variable name, d description string, v default value*/
#define VY_CVAR_I(n, d, v) \
vy_cvar n = {.name = #n, .description = d, .i = (v), .type = VY_CVAR_TYPE_INT}
#define VY_CVAR_F(n, d, v) \

8
src/tools/assetc/discovery.c
View File

@ -74,13 +74,7 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
vyLog("ASSETC", "Failed to lookup UID of known asset %s", data->path_scratch);
return VY_UNKNOWN_ERROR;
}
void *buffer;
size_t size;
if (vyLoadAssetDirect(uid, &buffer, &size) != VY_SUCCESS) {
vyLog("ASSETC", "Failed to load asset %s", data->path_scratch);
return VY_UNKNOWN_ASSET;
}
vyAddAssetToPackage(settings.package, uid, buffer, size);
vyAddUnchangedAssetToPackage(settings.package, uid);
}
} else {
/* Process it */

185
src/tools/assetc/packages.c
View File

@ -1,5 +1,6 @@
#include "packages.h"
#include "processing.h"
#include "utils.h"
#define VY_DEFINE_PACKAGE_FILE_STRUCTURES
#include "runtime/threading.h"
@ -19,9 +20,7 @@
typedef struct {
vy_uid uid;
void *buffer;
size_t buffer_size;
bool needs_compression;
size_t disk_size;
} vy_package_entry;
typedef struct {
@ -105,12 +104,65 @@ static void AddAssetToPackageImpl(unsigned int package,
pkg->entry_capacity = new_cap;
pkg->entries = n;
}
pkg->entries[pkg->num_entries].buffer = buffer;
pkg->entries[pkg->num_entries].buffer_size = size;
char tmp_path[256];
snprintf(tmp_path, 256, "actemp/%u.bin", uid);
if (needs_compression) {
FILE *tmp_f = fopen(tmp_path, "wb");
if (!tmp_f) {
vyUnlockMutex(_guard);
return;
}
int required_size = LZ4_compressBound((int)size);
void *compressed_buffer = malloc(required_size);
if (!compressed_buffer) {
fclose(tmp_f);
vyUnlockMutex(_guard);
return;
}
int compressed_bytes =
LZ4_compress_default(buffer, compressed_buffer, (int)size, required_size);
if (compressed_bytes == 0) {
free(compressed_buffer);
fclose(tmp_f);
vyReportError("ASSETC", "Failed to compress asset %x of package %s", uid, pkg->name);
return;
}
vy_package_asset_header header;
XXH64_hash_t checksum = XXH3_64bits_withSeed(buffer, (size_t)compressed_bytes, 0);
XXH64_canonicalFromHash(&header.checksum, checksum);
header.decompressed_size = (uint32_t)size;
if (fwrite(&header, sizeof(header), 1, tmp_f) != 1) {
vyReportError("ASSETC", "Failed to write to actemp/%u.bin", uid);
free(compressed_buffer);
fclose(tmp_f);
return;
}
if (fwrite(buffer, compressed_bytes, 1, tmp_f) != 1) {
vyReportError("ASSETC", "Failed to write to actemp/%u.bin", uid);
free(compressed_buffer);
fclose(tmp_f);
return;
}
fclose(tmp_f);
pkg->entries[pkg->num_entries].disk_size =
(size_t)compressed_bytes + sizeof(vy_package_asset_header);
} else {
pkg->entries[pkg->num_entries].disk_size = vyGetFileSize(tmp_path);
if (pkg->entries[pkg->num_entries].disk_size == 0) {
vyReportError("ASSETC", "Failed to determine size of actemp/%u.bin", uid);
}
}
pkg->entries[pkg->num_entries].uid = uid;
pkg->entries[pkg->num_entries].needs_compression = needs_compression;
++pkg->num_entries;
vyUnlockMutex(_guard);
}
@ -118,8 +170,8 @@ void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t
AddAssetToPackageImpl(package, uid, buffer, size, true);
}
void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size) {
AddAssetToPackageImpl(package, uid, buffer, size, false);
void vyAddUnchangedAssetToPackage(unsigned int package, vy_uid uid) {
AddAssetToPackageImpl(package, uid, NULL, 0, false);
}
static vy_result SavePackage(vy_package *pkg) {
@ -128,8 +180,8 @@ static vy_result SavePackage(vy_package *pkg) {
return VY_SUCCESS;
}
int current_buffer_size = 0;
void *buffer = NULL;
size_t current_buffer_size = 0;
void *buffer = NULL;
size_t offset_in_file = 0;
@ -142,75 +194,52 @@ static vy_result SavePackage(vy_package *pkg) {
}
for (unsigned int i = 0; i < pkg->num_entries; ++i) {
if (pkg->entries[i].needs_compression) {
int required_size = LZ4_compressBound((int)pkg->entries[i].buffer_size);
if (required_size > current_buffer_size) {
void *tmp = realloc(buffer, (size_t)required_size);
if (!tmp)
return VY_UNKNOWN_ERROR;
buffer = tmp;
current_buffer_size = required_size;
}
int compressed_bytes = LZ4_compress_default(pkg->entries[i].buffer,
buffer,
(int)pkg->entries[i].buffer_size,
current_buffer_size);
if (compressed_bytes == 0) {
free(buffer);
vyReportError("ASSETC", "Failed to compress entry %u of package %s", i, pkg->name);
return VY_UNKNOWN_ERROR;
}
vy_package_asset_header header;
XXH64_hash_t checksum = XXH3_64bits_withSeed(buffer, (size_t)compressed_bytes, 0);
XXH64_canonicalFromHash(&header.checksum, checksum);
header.decompressed_size = (uint32_t)pkg->entries[i].buffer_size;
if (fwrite(&header, sizeof(header), 1, f) != 1) {
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
free(buffer);
fclose(f);
return VY_UNKNOWN_ERROR;
}
if (fwrite(buffer, compressed_bytes, 1, f) != 1) {
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
free(buffer);
fclose(f);
return VY_UNKNOWN_ERROR;
}
vyAddUIDTabEntry(package_fid,
pkg->entries[i].uid,
offset_in_file,
(size_t)compressed_bytes + sizeof(header));
offset_in_file += sizeof(header) + (size_t)compressed_bytes;
} else {
vy_package_asset_header header;
XXH64_hash_t checksum = XXH3_64bits_withSeed(pkg->entries[i].buffer, pkg->entries[i].buffer_size, 0);
XXH64_canonicalFromHash(&header.checksum, checksum);
header.decompressed_size = (uint32_t)pkg->entries[i].buffer_size;
if (fwrite(&header, sizeof(header), 1, f) != 1) {
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
free(buffer);
fclose(f);
return VY_UNKNOWN_ERROR;
}
if (fwrite(pkg->entries[i].buffer, pkg->entries[i].buffer_size, 1, f) != 1) {
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
free(buffer);
fclose(f);
return VY_UNKNOWN_ERROR;
}
vyAddUIDTabEntry(package_fid,
pkg->entries[i].uid,
offset_in_file,
pkg->entries[i].buffer_size + sizeof(header));
offset_in_file += sizeof(header) + pkg->entries[i].buffer_size;
char tmp_path[256];
snprintf(tmp_path, 256, "actemp/%u.bin", pkg->entries[i].uid);
FILE *tmp_f = fopen(tmp_path, "rb");
if (!tmp_f) {
vyReportError("ASSETC", "Failed to open %s for reading.", tmp_path);
fclose(f);
free(buffer);
return VY_UNKNOWN_ERROR;
}
if (current_buffer_size < pkg->entries[i].disk_size) {
void *tmp = realloc(buffer, pkg->entries[i].disk_size);
if (!tmp) {
vyReportError("ASSETC", "Failed to allocate buffer (%zu bytes) for reading %s.", pkg->entries[i].disk_size, tmp_path);
fclose(f);
fclose(tmp_f);
free(buffer);
return VY_UNKNOWN_ERROR;
}
buffer = tmp;
current_buffer_size = pkg->entries[i].disk_size;
}
if (fread(buffer, pkg->entries[i].disk_size, 1, tmp_f) != 1) {
vyReportError("ASSETC", "Failed to read %s.", tmp_path);
fclose(f);
fclose(tmp_f);
free(buffer);
return VY_UNKNOWN_ERROR;
}
if (fwrite(buffer, pkg->entries[i].disk_size, 1, f) != 1) {
vyReportError("ASSETC", "Failed to write (%zu bytes) to %s.", pkg->entries[i].disk_size, pkg->name);
fclose(f);
fclose(tmp_f);
free(buffer);
return VY_UNKNOWN_ERROR;
}
vyAddUIDTabEntry(package_fid,
pkg->entries[i].uid,
offset_in_file,
pkg->entries[i].disk_size);
offset_in_file += pkg->entries[i].disk_size;
fclose(tmp_f);
}
free(buffer);

2
src/tools/assetc/packages.h
View File

@ -9,7 +9,7 @@ void vyInitPackages(void);
unsigned int vyAddPackageFile(vy_text_span name);
void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size);
void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size);
void vyAddUnchangedAssetToPackage(unsigned int package, vy_uid uid);
vy_result vySavePackages(void);