Move towards using DXC for compiling shaders

This is recommended by the vulkan documentation.
This commit is contained in:
Kevin Trogant 2024-01-25 23:33:29 +01:00
parent 3254af3786
commit 9670844bb2
34 changed files with 1230 additions and 32 deletions

View File

@ -4,7 +4,24 @@ vertex {
vk BEGIN
#include "test.hlsl"
void VsMain() {
struct VSInput
{
};
struct VSOutput
{
float4 Pos : SV_POSITION;
};
VSOutput VsMain(VSInput input, uint vertexIndex : SV_VertexID) {
VSOutput output = (VSOutput)0;
return output;
}
END
}
fragment {
vk BEGIN
END
}

1
contrib/.gitignore vendored
View File

@ -1,2 +1,3 @@
# Placed there by scripts/download_shaderc.bat
shaderc/*
dxc/*

View File

@ -1,4 +1,4 @@
project('voyage', 'c',
project('voyage', ['c', 'cpp'],
default_options: ['buildtype=debug',
'b_sanitize=address',
'c_std=c17',
@ -14,29 +14,31 @@ if compiler.get_argument_syntax() == 'gcc'
add_project_arguments(
['-Wconversion', '-Wno-sign-conversion',
'-Wdouble-promotion', '-Wno-unused-function', '-Wno-unused-parameter'],
language : 'c'
language : ['c', 'cpp']
)
add_project_arguments(['-fno-exceptions', '-fno-rtti'], language : 'cpp')
elif compiler.get_argument_syntax() == 'msvc'
add_project_arguments(
['/wd4146', '/wd4245', '/wd4100', '/D_CRT_SECURE_NO_WARNINGS'],
language: 'c'
language: ['c', 'cpp']
)
add_project_arguments(['/EHsc', '/GR-'], language : 'cpp')
if buildtype == 'debug'
add_project_arguments(['/RTCsu'], language : 'c')
add_project_arguments(['/RTCsu'], language : ['c', 'cpp'])
endif
endif
if get_option('default_library') == 'static'
add_project_arguments(['-DRT_STATIC_LIB'], language : 'c')
add_project_arguments(['-DRT_STATIC_LIB'], language : ['c', 'cpp'])
endif
if get_option('error_report_debugbreak')
add_project_arguments(['-DRT_ERROR_REPORT_DEBUGBREAK'], language : 'c')
add_project_arguments(['-DRT_ERROR_REPORT_DEBUGBREAK'], language : ['c', 'cpp'])
endif
# Debug specific flags
if buildtype == 'debug' or buildtype == 'debugoptimized'
add_project_arguments([ '-DRT_DEBUG'], language : 'c')
add_project_arguments([ '-DRT_DEBUG'], language : ['c', 'cpp'])
endif
# Gather dependencies
@ -49,9 +51,12 @@ vk_dep = dependency('vulkan', required : false)
windowing_dep = []
if get_option('use_xlib')
windowing_dep = dependency('x11', required : true)
add_project_arguments(['-DRT_USE_XLIB'], language : 'c')
add_project_arguments(['-DRT_USE_XLIB'], language : ['c', 'cpp'])
endif
# Copy file utility
copy_util = find_program('scripts/copy_util.py')
runtime_incdirs = common_incdirs
runtime_linkargs = []
runtime_additional_sources = []
@ -62,23 +67,45 @@ if get_option('build_asset_compiler')
runtime_cargs += ['-DRT_BUILD_ASSET_COMPILER']
# Shaderc for shaders
if get_option('enable_vulkan_shader_compiler')
shaderc_include = include_directories('contrib\\shaderc\\libshaderc\\include')
shaderc_libdir = 'NONE'
#if get_option('enable_vulkan_shader_compiler')
# shaderc_include = include_directories('contrib\\shaderc\\libshaderc\\include')
# shaderc_libdir = 'NONE'
# if host_machine.system() == 'windows'
# if buildtype == 'debug' or buildtype == 'debugoptimized'
# shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Debug'
# else
# shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Release'
# endif
# endif
# shaderc_dep = declare_dependency(link_args : ['-L'+shaderc_libdir, '-lshaderc_combined'],
# include_directories : shaderc_include)
# runtime_deps += shaderc_dep
# runtime_additional_sources += [
# 'src/runtime/vulkan_shader_compiler.c'
# ]
# runtime_cargs += ['-DRT_BUILD_VULKAN_SHADER_COMPILER']
#endif
# DXC for vulkan & directx shaders
if get_option('enable_dxc_shader_compiler')
# We package dxc binaries under contrib/dxc
dxc_include = include_directories('contrib' / 'dxc' / 'inc')
dxc_libdir = 'NONE'
if host_machine.system() == 'windows'
if buildtype == 'debug' or buildtype == 'debugoptimized'
shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Debug'
else
shaderc_libdir = meson.project_source_root() / 'contrib\\shaderc\\build-win\\libshaderc\\Release'
dxc_libdir = meson.project_source_root() / 'contrib' / 'dxc' / 'lib' / 'x64'
custom_target('copy dxcompiler.dll',
input : 'contrib' / 'dxc' / 'bin' / 'x64' / 'dxcompiler.dll',
output : 'dxcompiler.dll',
command : [copy_util, '@INPUT@', '@OUTPUT@'],
install : false,
build_by_default : true)
endif
endif
shaderc_dep = declare_dependency(link_args : ['-L'+shaderc_libdir, '-lshaderc_combined'],
include_directories : shaderc_include)
runtime_deps += shaderc_dep
dxc_dep = declare_dependency(link_args : ['-L'+dxc_libdir, '-ldxcompiler'], include_directories : dxc_include)
runtime_deps += dxc_dep
runtime_additional_sources += [
'src/runtime/vulkan_shader_compiler.c'
'src/runtime/dxc_shader_compiler.cpp'
]
runtime_cargs += ['-DRT_BUILD_VULKAN_SHADER_COMPILER']
runtime_cargs += ['-DRT_BUILD_DXC_SHADER_COMPILER']
endif
# Asset compiler sources
@ -143,7 +170,9 @@ runtime_lib = library('rt',
include_directories : runtime_incdirs,
link_args : runtime_linkargs,
c_args : runtime_cargs,
c_pch : 'pch/rt_pch.h')
c_pch : 'pch/rt_pch.h',
cpp_args : runtime_cargs,
cpp_pch : 'pch/rt_pch.h')
# Renderer libraries

View File

@ -1,4 +1,4 @@
option('use_xlib', type : 'boolean', value : false, description : 'Use Xlib for window creation under linux')
option('error_report_debugbreak', type : 'boolean', value : true, description : 'Debugbreak in ReportError')
option('build_asset_compiler', type : 'boolean', value : true, description : 'Enables or disables the asset compiler inside runtime.')
option('enable_vulkan_shader_compiler', type : 'boolean', value : true, description : 'Enables building the vulkan shader compiler (Requires shaderc).')
option('enable_dxc_shader_compiler', type : 'boolean', value : true, description : 'Enables building the dxc-based shader compiler.')

13
scripts/copy_util.py Normal file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env python3
import sys
import shutil
import os
if len(sys.argv) < 3:
os._exit(1)
shutil.copyfile(sys.argv[1], sys.argv[2])
shutil.copymode(sys.argv[1], sys.argv[2])

17
scripts/download_dxc.bat Normal file
View File

@ -0,0 +1,17 @@
@echo off
REM Wrapper around the powershell script
SETLOCAL
IF EXIST scripts\download_dxc.ps1 GOTO doit
IF EXIST download_dxc.ps1 GOTO goup
ECHO Failed to find download_dxc.ps1; you are probably running this script from an unexpected directory.
:goup
cd ..
:doit
powershell.exe -File scripts\download_dxc.ps1
ENDLOCAL

4
scripts/download_dxc.ps1 Normal file
View File

@ -0,0 +1,4 @@
$file = "dxc_2023_08_14.zip"
Invoke-WebRequest -Uri https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.7.2308/$file -OutFile contrib\$file
Expand-Archive -LiteralPath contrib\$file -DestinationPath contrib\dxc
Remove-Item -Path contrib\$file

View File

@ -6,6 +6,10 @@
#include "file_tab.h"
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
size_t offset; /**< Starting offset inside the file in bytes */
size_t num_bytes; /**< Number of bytes to load */
@ -66,4 +70,8 @@ RT_DLLEXPORT rt_result rtSubmitSingleLoad(rt_file_load load, rt_aio_handle *hand
* Returns the state that caused the wait for completion to return. */
RT_DLLEXPORT rt_aio_state rtSubmitSingleLoadSync(rt_file_load load);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -5,6 +5,10 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _WIN32
/* Forward declared here, to avoid including windows.h */
@ -21,4 +25,8 @@ RT_DLLEXPORT int rtXlibEntry(int argc, char **argv);
#endif
#ifdef __cplusplus
}
#endif
#endif

View File

@ -7,6 +7,10 @@
#include "file_tab.h"
#ifdef __cplusplus
extern "C" {
#endif
enum {
RT_ASSET_PROCESSING_FAILED = RT_CUSTOM_ERROR_START,
};
@ -19,4 +23,8 @@ typedef struct {
rt_loaded_asset LoadAsset(rt_file_id file);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -0,0 +1,379 @@
#include "assets.h"
#include "asset_dependencies.h"
#include "aio.h"
#include "buffer_manager.h"
#include "config.h"
#include "threading.h"
#include "uidtab.h"
#include <assert.h>
#include <stdlib.h>
RT_CVAR_I(rt_AssetCacheSize, "Number of asset cache entries. Default: 1024.", 1024);
/* asset_loading.c */
extern rt_result DecompressAsset(void *compressed_buffer,
size_t compressed_buffer_size,
void **p_decompressed,
size_t *p_decompressed_size);
typedef enum {
CACHE_ENTRY_STATE_FREE,
CACHE_ENTRY_STATE_LOADING,
CACHE_ENTRY_STATE_LOADED,
} rt_asset_cache_entry_state;
typedef struct rt_asset_cache_entry_s {
rt_asset_cache_entry_state state;
rt_aio_handle load;
void *buffer;
size_t size;
int refcount;
/* Reclaim list */
struct rt_asset_cache_entry_s *prev_reclaim;
struct rt_asset_cache_entry_s *next_reclaim;
} rt_asset_cache_entry;
static rt_uid *_uids;
static rt_asset_cache_entry *_entries;
static rt_asset_cache_entry *_first_reclaim;
static rt_asset_cache_entry *_last_reclaim;
/* Locked as writer when modifiying entries, as reader when searching */
static rt_rwlock _lock;
rt_result InitAssetCache(void) {
_entries = calloc((size_t)rt_AssetCacheSize.i, sizeof(rt_asset_cache_entry));
if (!_entries) {
return RT_BUFFER_ALLOC_FAILED;
}
_uids = calloc((size_t)rt_AssetCacheSize.i, sizeof(rt_uid));
if (!_uids) {
free(_entries);
return RT_BUFFER_ALLOC_FAILED;
}
rt_create_rwlock_result lock_res = rtCreateRWLock();
if (!lock_res.ok) {
free(_entries);
free(_uids);
return RT_UNKNOWN_ERROR;
}
_lock = lock_res.lock;
return RT_SUCCESS;
}
void ShutdownAssetCache(void) {
free(_entries);
free(_uids);
rtDestroyRWLock(&_lock);
_first_reclaim = NULL;
_last_reclaim = NULL;
}
static void ReleaseEntry(rt_asset_cache_entry *entry) {
if (entry->load != RT_AIO_INVALID_HANDLE) {
rtWaitForAIOCompletion(entry->load);
rtReleaseAIO(entry->load);
entry->load = RT_AIO_INVALID_HANDLE;
}
rtReleaseBuffer(entry->buffer, entry->size);
entry->buffer = NULL;
entry->size = 0;
entry->next_reclaim = NULL;
entry->prev_reclaim = NULL;
}
static void GarbageCollect(void) {
rtLockWrite(&_lock);
rt_asset_cache_entry *entry = _first_reclaim;
while (entry) {
assert(entry->refcount == 0);
rt_asset_cache_entry *next = entry->next_reclaim;
if (entry->state == CACHE_ENTRY_STATE_LOADED) {
ReleaseEntry(entry);
_first_reclaim = next;
}
entry = next;
}
rtUnlockWrite(&_lock);
}
static rt_asset_cache_entry *GetEntry(rt_uid uid) {
/* Hash lookup */
unsigned int mod = (unsigned int)rt_AssetCacheSize.i - 1;
for (unsigned int i = 0; i < (unsigned int)rt_AssetCacheSize.i; ++i) {
unsigned int slot = (uid + i) & mod;
if (_uids[slot] == uid) {
return &_entries[slot];
} else if (_uids[slot] == RT_INVALID_UID) {
break;
}
}
return NULL;
}
static bool IsAssetLoaded(rt_uid uid) {
const rt_asset_cache_entry *entry = GetEntry(uid);
if (entry)
return entry->state == CACHE_ENTRY_STATE_LOADED ||
entry->state == CACHE_ENTRY_STATE_LOADING;
else
return false;
}
static int InsertEntry(rt_uid uid) {
unsigned int mod = (unsigned int)rt_AssetCacheSize.i - 1;
for (unsigned int i = 0; i < (unsigned int)rt_AssetCacheSize.i; ++i) {
unsigned int slot = (uid + i) & mod;
if (_uids[slot] == 0) {
return (int)slot;
}
}
return -1;
}
static rt_result InsertAndLoadAssets(const rt_uid *uids, size_t count) {
rt_load_batch batch = {.num_loads = 0};
rt_result res = RT_SUCCESS;
count = (count < RT_LOAD_BATCH_MAX_SIZE) ? count : RT_LOAD_BATCH_MAX_SIZE;
rt_asset_cache_entry *load_entries[RT_LOAD_BATCH_MAX_SIZE];
for (size_t i = 0; i < count; ++i) {
rtLockRead(&_lock);
bool needs_load = !IsAssetLoaded(uids[i]);
rtUnlockRead(&_lock);
if (!needs_load)
continue;
rtLockWrite(&_lock);
/* It's possible that another thread loaded the asset in the meantime */
if (!IsAssetLoaded(uids[i])) {
const rt_uid_data *data = rtGetUIDData(uids[i]);
if (!data) {
rtUnlockWrite(&_lock);
rtLog("ASSET_CACHE", "Failed to get uid data for uid %u", uids[i]);
res = RT_UNKNOWN_ASSET;
continue;
}
void *compressed_data = rtAllocBuffer(data->size);
if (!compressed_data) {
/* Try again after garbage collection */
rtUnlockWrite(&_lock);
GarbageCollect();
compressed_data = rtAllocBuffer(data->size);
if (!compressed_data) {
rtLog("ASSET_CACHE",
"Failed to allocate intermediate buffer for uid %u",
uids[i]);
res = RT_BUFFER_ALLOC_FAILED;
continue;
}
rtLockWrite(&_lock);
}
int slot = InsertEntry(uids[i]);
if (slot == -1) {
rtUnlockWrite(&_lock);
rtLog("ASSET_CACHE", "Failed to insert new entry for uid %u", uids[i]);
res = RT_ASSET_CACHE_FULL;
break;
}
rt_asset_cache_entry *entry = &_entries[slot];
load_entries[batch.num_loads] = entry;
/* We set the refcount to 0, but don't insert the entry
* into the reclaim list, to ensure that its buffer does not get freed
* while the load still executes. Setting the refcount to 0 ensures
* that the count is correct, once the asset is accessed the first time. */
entry->state = CACHE_ENTRY_STATE_LOADING;
entry->refcount = 0;
entry->buffer = compressed_data;
entry->size = data->size;
entry->next_reclaim = NULL;
entry->prev_reclaim = NULL;
entry->load = RT_AIO_INVALID_HANDLE;
batch.loads[batch.num_loads].file = data->pkg_file;
batch.loads[batch.num_loads].num_bytes = data->size;
batch.loads[batch.num_loads].offset = data->offset;
batch.loads[batch.num_loads].dest = compressed_data;
++batch.num_loads;
}
}
rtUnlockWrite(&_lock);
/* Dispatch the load */
rt_aio_handle handles[RT_LOAD_BATCH_MAX_SIZE];
if ((res = rtSubmitLoadBatch(&batch, handles)) != RT_SUCCESS) {
rtLog("ASSET_CACHE", "Failed to submit %u asset loads.", batch.num_loads);
return res;
}
/* Set the aio handles of the inserted entries */
rtLockWrite(&_lock);
for (unsigned int i = 0; i < batch.num_loads; ++i) {
load_entries[batch.num_loads]->load = handles[i];
}
rtUnlockWrite(&_lock);
return res;
}
static bool DecompressEntry(rt_uid uid, rt_asset_cache_entry *entry) {
rtReleaseAIO(entry->load);
entry->load = RT_AIO_INVALID_HANDLE;
void *decompressed_buffer;
size_t decompressed_size;
rt_result dec_res =
DecompressAsset(entry->buffer, entry->size, &decompressed_buffer, &decompressed_size);
if (dec_res == RT_SUCCESS) {
rtReleaseBuffer(entry->buffer, entry->size);
entry->buffer = decompressed_buffer;
entry->size = decompressed_size;
entry->state = CACHE_ENTRY_STATE_LOADED;
return true;
} else if (dec_res == RT_BUFFER_ALLOC_FAILED) {
GarbageCollect();
/* Try again */
if (DecompressAsset(entry->buffer, entry->size, &decompressed_buffer, &decompressed_size) ==
RT_SUCCESS) {
rtReleaseBuffer(entry->buffer, entry->size);
entry->buffer = decompressed_buffer;
entry->size = decompressed_size;
entry->state = CACHE_ENTRY_STATE_LOADED;
return true;
}
/* Don't do anything yet. We might be able to to do this later, once some
* buffers become free. */
rtLog("ASSET_CACHE", "Failed to decompress asset %u", uid);
return false;
} else {
rtLog("ASSET_CACHE", "Failed to decompress asset %u", uid);
ReleaseEntry(entry);
ptrdiff_t idx = entry - _entries;
_uids[idx] = RT_INVALID_UID;
return false;
}
}
static void CheckCompletedLoads(const rt_uid *uids, size_t count) {
for (size_t i = 0; i < count; ++i) {
rtLockRead(&_lock);
volatile rt_asset_cache_entry *entry = (volatile rt_asset_cache_entry *)GetEntry(uids[i]);
if (!entry) {
rtUnlockRead(&_lock);
rtLog("ASSET_CACHE", "Passed unknown uid %u to CheckCompletedLoads()", uids[i]);
continue;
}
if (entry->state != CACHE_ENTRY_STATE_LOADING) {
rtUnlockRead(&_lock);
continue;
}
bool load_finished = rtGetAIOState(entry->load) == RT_AIO_STATE_FINISHED;
rtUnlockRead(&_lock);
if (load_finished) {
rtLockWrite(&_lock);
/* Ensure that no-one else handled this */
if (entry->state == CACHE_ENTRY_STATE_LOADING) {
DecompressEntry(uids[i], (rt_asset_cache_entry *)entry);
}
rtUnlockWrite(&_lock);
}
}
}
RT_DLLEXPORT rt_get_asset_result rtGetAsset(rt_uid uid) {
rt_get_asset_result result = {
.result = RT_SUCCESS,
};
rtLockRead(&_lock);
bool needs_load = !IsAssetLoaded(uid);
rtUnlockRead(&_lock);
if (needs_load) {
rt_uid load_uids[RT_LOAD_BATCH_MAX_SIZE];
size_t load_count = 1;
load_uids[0] = uid;
rt_asset_dependency_list deps = rtGetAssetDependencies(uid);
for (size_t i = 0; i < deps.count && i < RT_LOAD_BATCH_MAX_SIZE - 1; ++i) {
load_uids[i + 1] = deps.dependencies[i];
++load_count;
}
result.result = InsertAndLoadAssets(load_uids, load_count);
if (result.result == RT_SUCCESS) {
CheckCompletedLoads(load_uids, load_count);
}
}
rtLockWrite(&_lock);
rt_asset_cache_entry *entry = GetEntry(uid);
if (entry) {
if (entry->state == CACHE_ENTRY_STATE_LOADED) {
++entry->refcount;
result.data = entry->buffer;
result.size = entry->size;
} else if (entry->state == CACHE_ENTRY_STATE_LOADING) {
if (entry->state == CACHE_ENTRY_STATE_LOADING) {
assert(entry->load != RT_AIO_INVALID_HANDLE);
++entry->refcount;
if (rtWaitForAIOCompletion(entry->load) == RT_AIO_STATE_FINISHED) {
if (DecompressEntry(uid, entry)) {
result.data = entry->buffer;
result.size = entry->size;
} else {
result.result = RT_LOAD_FAILED;
}
} else {
ReleaseEntry(entry);
rtLog("ASSET_CACHE", "Failed to load asset %u", uid);
result.result = RT_LOAD_FAILED;
}
}
}
/* Remove from the reclaim list */
if (_first_reclaim == entry)
_first_reclaim = entry->next_reclaim;
if (_last_reclaim == entry)
_last_reclaim = entry->prev_reclaim;
if (entry->next_reclaim)
entry->next_reclaim->prev_reclaim = entry->prev_reclaim;
if (entry->prev_reclaim)
entry->prev_reclaim->next_reclaim = entry->next_reclaim;
}
rtUnlockWrite(&_lock);
return result;
}
RT_DLLEXPORT void rtReleaseAsset(rt_uid uid) {
rtLockWrite(&_lock);
rt_asset_cache_entry *entry = GetEntry(uid);
if (entry && entry->refcount > 0) {
--entry->refcount;
if (entry->refcount == 0) {
/* add to the reclaim list */
if (_last_reclaim)
_last_reclaim->next_reclaim = entry;
if (!_first_reclaim)
_first_reclaim = entry;
entry->prev_reclaim = _last_reclaim;
entry->next_reclaim = NULL;
_last_reclaim = entry;
}
}
rtUnlockWrite(&_lock);
}

View File

@ -0,0 +1,137 @@
#define RT_DEFINE_DEPENDENCY_FILE_STRUCTURES
#include "asset_dependencies.h"
#include "aio.h"
#include "buffer_manager.h"
#include <assert.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
typedef struct {
uint32_t begin;
uint32_t count;
} rt_dep_list;
typedef struct {
rt_uid *uids;
rt_dep_list *lists;
uint32_t capacity;
} rt_dep_map;
static rt_dep_map _map;
static rt_uid *_list_mem;
rt_result LoadAssetDependencies(void) {
rt_dependency_file_header header;
rt_file_id fid = rtAddFile("data/deps.bin");
if (rtSubmitSingleLoadSync((rt_file_load){.dest = &header,
.num_bytes = sizeof(header),
.offset = 0,
.file = fid}) != RT_AIO_STATE_FINISHED) {
rtReportError("core", "Failed to load deps.bin");
return RT_UNKNOWN_ERROR;
}
void *buffer = rtAllocBuffer(header.data_size);
if (rtSubmitSingleLoadSync((rt_file_load){.dest = buffer,
.num_bytes = header.data_size,
.offset = sizeof(header),
.file = fid}) != RT_AIO_STATE_FINISHED) {
rtReportError("core", "Failed to load deps.bin");
return RT_UNKNOWN_ERROR;
}
/* We know the exact number of list entries */
uint64_t total_list_entries =
(header.data_size - header.num_lists * sizeof(rt_dependency_file_list_header)) /
sizeof(rt_uid);
_list_mem = malloc(total_list_entries * sizeof(rt_uid));
if (!_list_mem) {
rtReleaseBuffer(buffer, header.data_size);
rtReportError("core", "Failed to allocate dependency list storage.");
return RT_UNKNOWN_ERROR;
}
_map.capacity = rtNextPowerOfTwo32(header.num_lists);
_map.uids = calloc(_map.capacity, sizeof(rt_uid));
if (!_map.uids) {
free(_list_mem);
rtReleaseBuffer(buffer, header.data_size);
rtReportError("core", "Failed to allocate dependency list storage.");
return RT_UNKNOWN_ERROR;
}
_map.lists = calloc(_map.capacity, sizeof(rt_dep_list));
if (!_map.uids) {
free(_list_mem);
free(_map.uids);
rtReleaseBuffer(buffer, header.data_size);
rtReportError("core", "Failed to allocate dependency list storage.");
return RT_UNKNOWN_ERROR;
}
uint32_t storage_at = 0;
rt_dependency_file_list_header *list = buffer;
for (uint32_t i = 0; i < header.num_lists; ++i) {
const rt_uid *entries = (rt_uid *)(list + 1);
/* Validate the checksum */
XXH64_hash_t file_hash = XXH64_hashFromCanonical(&list->checksum);
XXH64_hash_t calc_hash = XXH3_64bits(entries, sizeof(rt_uid) * list->num_entries);
if (file_hash != calc_hash) {
rtReportError("core", "Checksum mismatch in list %u", i);
rtReleaseBuffer(buffer, header.data_size);
return RT_UNKNOWN_ERROR;
}
/* Store the list */
memcpy(_list_mem + storage_at, entries, sizeof(rt_uid) * list->num_entries);
bool inserted = false;
for (uint32_t j = 0; j < _map.capacity; ++j) {
uint32_t slot = (list->uid + j) % _map.capacity;
if (_map.uids[slot] == RT_INVALID_UID) {
_map.uids[slot] = list->uid;
_map.lists[slot].begin = storage_at;
_map.lists[slot].count = list->num_entries;
inserted = true;
break;
}
}
storage_at += list->num_entries;
assert(inserted);
assert(storage_at <= total_list_entries);
list = (rt_dependency_file_list_header *)(entries + list->num_entries);
}
rtReleaseBuffer(buffer, header.data_size);
return RT_SUCCESS;
}
void ReleaseAssetDependencies(void) {
free(_list_mem);
free(_map.uids);
free(_map.lists);
}
RT_DLLEXPORT rt_asset_dependency_list rtGetAssetDependencies(rt_uid asset) {
rt_asset_dependency_list result = {
.dependencies = NULL,
.count = 0,
};
for (uint32_t i = 0; i < _map.capacity; ++i) {
uint32_t slot = (asset + i) % _map.capacity;
if (_map.uids[slot] == asset) {
result.dependencies = &_list_mem[_map.lists[slot].begin];
result.count = _map.lists[slot].count;
break;
} else if (_map.uids[slot] == RT_INVALID_UID) {
break;
}
}
return result;
}

View File

@ -0,0 +1,33 @@
#ifndef RT_ASSET_DEPENDENCIES_H
#define RT_ASSET_DEPENDENCIES_H
#include "assets.h"
#ifdef RT_DEFINE_DEPENDENCY_FILE_STRUCTURES
#include "xxhash/xxhash.h"
#pragma pack(push, 1)
typedef struct {
uint64_t data_size;
uint32_t num_lists;
} rt_dependency_file_header;
typedef struct {
rt_uid uid;
uint32_t num_entries;
XXH64_canonical_t checksum;
} rt_dependency_file_list_header;
#pragma pack(pop)
#endif
typedef struct {
const rt_uid *dependencies;
uint32_t count;
} rt_asset_dependency_list;
RT_DLLEXPORT rt_asset_dependency_list rtGetAssetDependencies(rt_uid asset);
#endif

View File

@ -0,0 +1,74 @@
#include "assets.h"
#include "uidtab.h"
#include "aio.h"
#include "buffer_manager.h"
#define RT_DEFINE_PACKAGE_FILE_STRUCTURES
#include "packages.h"
#include "lz4/lz4.h"
rt_result DecompressAsset(void *compressed_buffer,
size_t compressed_buffer_size,
void **p_decompressed,
size_t *p_decompressed_size) {
const rt_package_asset_header *header = compressed_buffer;
size_t compressed_size = (compressed_buffer_size) - sizeof(*header);
XXH64_hash_t calculated_hash = XXH3_64bits((header + 1), compressed_size);
XXH64_hash_t file_hash = XXH64_hashFromCanonical(&header->checksum);
if (calculated_hash != file_hash) {
rtLog("core", "Checksum mismatch for asset");
return RT_LOAD_FAILED;
}
size_t size = (size_t)header->decompressed_size;
void *decompressed_buffer = rtAllocBuffer(size);
if (!decompressed_buffer) {
return RT_BUFFER_ALLOC_FAILED;
}
if (LZ4_decompress_safe((const char *)(header + 1),
(char *)decompressed_buffer,
(int)compressed_size,
(int)size) < 0) {
return RT_UNKNOWN_ERROR;
}
*p_decompressed = decompressed_buffer;
*p_decompressed_size = size;
return RT_SUCCESS;
}
RT_DLLEXPORT rt_result rtLoadAssetDirect(rt_uid uid, void **p_buffer, size_t *p_size) {
const rt_uid_data *data = rtGetUIDData(uid);
if (!data)
return RT_UNKNOWN_ASSET;
void *compressed_buffer = rtAllocBuffer(data->size);
if (!compressed_buffer) {
return RT_BUFFER_ALLOC_FAILED;
}
if (rtSubmitSingleLoadSync((rt_file_load) {
.file = data->pkg_file,
.offset = data->offset,
.num_bytes = data->size,
.dest = compressed_buffer,
}) != RT_AIO_STATE_FINISHED) {
rtReleaseBuffer(compressed_buffer, data->size);
return RT_LOAD_FAILED;
}
void *decompressed_buffer;
size_t decompressed_size;
rt_result res = DecompressAsset(compressed_buffer, data->size, &decompressed_buffer, &decompressed_size);
rtReleaseBuffer(compressed_buffer, data->size);
*p_buffer = decompressed_buffer;
*p_size = decompressed_size;
return res;
}

View File

@ -0,0 +1,40 @@
#ifndef RT_ASSETS_H
#define RT_ASSETS_H
#include <stdint.h>
#include "runtime.h"
/* Unique identifier for an asset. */
typedef uint32_t rt_uid;
#define RT_INVALID_UID 0
/* Used to identify renderer backend dependent assets. */
enum {
RT_INVALID_RENDERER_BACKEND_CODE = 0,
RT_RENDERER_BACKEND_CODE_VK = 1,
RT_RENDERER_BACKEND_CODE_ONE_PAST_LAST,
};
typedef uint8_t rt_renderer_backend_code;
enum {
RT_UNKNOWN_ASSET = RT_CUSTOM_ERROR_START,
RT_BUFFER_ALLOC_FAILED,
RT_LOAD_FAILED,
RT_ASSET_CACHE_FULL,
};
/* Load an asset without using the cache */
RT_DLLEXPORT rt_result rtLoadAssetDirect(rt_uid uid, void **buffer, size_t *size);
typedef struct {
void *data;
size_t size;
rt_result result;
} rt_get_asset_result;
RT_DLLEXPORT rt_get_asset_result rtGetAsset(rt_uid uid);
#endif

View File

@ -0,0 +1,121 @@
#define RT_DEFINE_UIDTAB_FILE_STRUCTURES
#include "uidtab.h"
#include "aio.h"
#include "xxhash/xxhash.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
typedef struct {
rt_uid *uids;
rt_uid_data *data;
uint32_t slots;
} rt_uidtab;
static rt_uidtab _tab;
rt_result LoadUIDTable(void) {
/* We use stdio here, because we cannot load any asset in parallel to this.
* This is because the uidtab is what tells us which assets exist.
*/
FILE *f = fopen("data/uidtab.bin", "rb");
if (!f)
return RT_LOAD_FAILED;
rt_uidtab_header header;
if (fread(&header, sizeof(header), 1, f) != 1) {
fclose(f);
return RT_LOAD_FAILED;
}
/* TODO(Kevin): For some reason, the checksum calculation causes
* Memory access errors .
XXH3_state_t *checksum = XXH3_createState();
if (!checksum) {
fclose(f);
return RT_UNKNOWN_ERROR;
}
*/
_tab.slots = rtNextPowerOfTwo32(header.num_entries * 2);
void *mem = malloc((sizeof(rt_uid) + sizeof(rt_uid_data)) * _tab.slots);
if (!mem) {
fclose(f);
_tab.slots = 0;
return RT_OUT_OF_MEMORY;
}
_tab.uids = mem;
_tab.data = (rt_uid_data *)(_tab.uids + _tab.slots);
memset(mem, 0, (sizeof(rt_uid) + sizeof(rt_uid_data)) * _tab.slots);
uint32_t mod = _tab.slots - 1;
for (uint32_t i = 0; i < header.num_entries; ++i) {
rt_uidtab_entry entry;
if (fread(&entry, sizeof(entry), 1, f) != 1) {
free(mem);
_tab.slots = 0;
fclose(f);
return RT_LOAD_FAILED;
}
//XXH3_64bits_update(checksum, &entry, sizeof(entry));
/* Insert into hashtable */
bool inserted = false;
for (uint32_t j = 0; j < _tab.slots; ++j) {
uint32_t at = (entry.uid + j) & mod;
if (_tab.uids[at] == RT_INVALID_UID) {
_tab.uids[at] = entry.uid;
_tab.data[at].pkg_file = entry.file;
_tab.data[at].size = entry.size;
_tab.data[at].offset = entry.offset;
inserted = true;
break;
}
}
if (!inserted) {
rtReportError("core",
"Failed to insert an entry into the uid table. This should not happen.");
fclose(f);
free(mem);
_tab.slots = 0;
return RT_UNKNOWN_ERROR;
}
}
fclose(f);
/*
XXH64_hash_t checksum_hash = XXH3_64bits_digest(checksum);
XXH64_hash_t file_hash = XXH64_hashFromCanonical(&header.checksum);
XXH3_freeState(checksum);
if (checksum_hash != file_hash) {
rtLog("core",
"WARNING: uidtab.bin checksum does not match calculated checksum of loaded entries.");
}
*/
return RT_SUCCESS;
}
void ReleaseUIDTable(void) {
free(_tab.uids);
_tab.slots = 0;
}
RT_DLLEXPORT const rt_uid_data *rtGetUIDData(rt_uid uid) {
uint32_t mod = _tab.slots - 1;
for (uint32_t j = 0; j < _tab.slots; ++j) {
uint32_t at = (uid + j) & mod;
if (_tab.uids[at] == uid) {
return &_tab.data[at];
} else if (_tab.uids[at] == RT_INVALID_UID) {
break;
}
}
return NULL;
}

View File

@ -0,0 +1,36 @@
#ifndef RT_UIDTAB_H
#define RT_UIDTAB_H
#include "runtime.h"
#include "file_tab.h"
#include "assets.h"
#include "xxhash/xxhash.h"
#ifdef RT_DEFINE_UIDTAB_FILE_STRUCTURES
#pragma pack(push, 1)
typedef struct {
XXH64_canonical_t checksum;
uint32_t num_entries;
} rt_uidtab_header;
#pragma pack(pop)
#pragma pack(push, 1)
typedef struct {
rt_file_id file;
uint64_t offset;
uint64_t size;
rt_uid uid;
} rt_uidtab_entry;
#pragma pack(pop)
#endif
/* Data associated with an uid */
typedef struct {
rt_file_id pkg_file;
uint64_t offset;
uint64_t size;
} rt_uid_data;
RT_DLLEXPORT const rt_uid_data *rtGetUIDData(rt_uid uid);
#endif

View File

@ -3,6 +3,10 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
enum {
RT_BUFFER_MGR_OUT_OF_MEMORY = RT_CUSTOM_ERROR_START,
RT_BUFFER_MGR_MUTEX_CREATION_FAILED,
@ -14,4 +18,8 @@ RT_DLLEXPORT void rtReleaseBuffer(const void *begin, size_t size);
RT_DLLEXPORT void rtIncreaseBufferRefCount(const void *begin, size_t size);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -3,6 +3,10 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
RT_CVAR_TYPE_INT,
RT_CVAR_TYPE_FLOAT,
@ -36,4 +40,8 @@ RT_DLLEXPORT void rtRegisterCVAR(rt_cvar *cvar);
RT_DLLEXPORT rt_cvar *rtGetCVAR(const char *name);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -3,6 +3,10 @@
#include "runtime/runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
struct rt_arena_s;
typedef enum {
@ -51,6 +55,11 @@ rt_result rtParseDescription(const char *text,
rt_parse_state *state,
struct rt_arena_s *arena);
const rt_parsed_stmt *rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute);
const rt_parsed_stmt *
rtFindStatement(const rt_parse_state *state, unsigned int list_index, const char *attribute);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -0,0 +1,149 @@
#ifdef _WIN32
// Required by dxcapi.h, does not work with WIN32_LEAN_AND_MEAN
#include <Windows.h>
#endif
#include <dxcapi.h>
#include "asset_compiler.h"
#include "shader_compiler.h"
extern "C" rt_shader_bytecode CompileVulkanShader(rt_shader_stage stage,
rt_shader_optimization_level optimization,
rt_text_span code,
const char *file_path,
rt_arena *arena) {
rt_shader_bytecode bc = {0};
// NOTE(Kevin): This looks like some sort of autodetect?
// Maybe fix this to UTF8?
uint32_t codepage = DXC_CP_ACP;
// NOTE(Kevin): 6_1 is used at https://docs.vulkan.org/guide/latest/hlsl.html
// Check if this is what we want.
// For example: 6_2 is what allows the usage of 16 bit types
LPCWSTR target_profile = nullptr;
LPWSTR entry = nullptr;
switch (stage) {
case RT_SHADER_STAGE_VERTEX:
target_profile = L"vs_6_1";
entry = L"VsMain";
break;
case RT_SHADER_STAGE_FRAGMENT:
target_profile = L"ps_6_1";
entry = L"PsMain";
break;
case RT_SHADER_STAGE_COMPUTE:
target_profile = L"cs_6_1";
entry = L"CsMain";
break;
default:
rtReportError("AC", "Invalid shader stage %u for shader %s", stage, file_path);
return bc;
}
LPWSTR optimization_arg = nullptr;
switch (optimization) {
case RT_SHADER_OPTIMIZATION_NONE:
optimization_arg = L"-Od";
break;
case RT_SHADER_OPTIMIZATION_SIZE:
case RT_SHADER_OPTIMIZATION_SPEED:
optimization_arg = L"-O3";
break;
}
// Init DXC library and compiler
HRESULT hr;
IDxcLibrary *library;
hr = DxcCreateInstance(CLSID_DxcLibrary, IID_PPV_ARGS(&library));
if (FAILED(hr)) {
rtReportError("AC", "Failed to init the DXC library.");
return bc;
}
IDxcCompiler3 *compiler;
hr = DxcCreateInstance(CLSID_DxcCompiler, IID_PPV_ARGS(&compiler));
if (FAILED(hr)) {
library->Release();
rtReportError("AC", "Failed to init the DXC compiler.");
return bc;
}
IDxcUtils *utils;
hr = DxcCreateInstance(CLSID_DxcUtils, IID_PPV_ARGS(&utils));
if (FAILED(hr)) {
compiler->Release();
library->Release();
rtReportError("AC", "Failed to init the DXC utils.");
return bc;
}
IDxcIncludeHandler *include_handler;
hr = utils->CreateDefaultIncludeHandler(&include_handler);
if (FAILED(hr)) {
utils->Release();
compiler->Release();
library->Release();
rtReportError("AC", "Failed to init the DXC ínclude handler.");
return bc;
}
// Gather arguments.
// TODO(kevin): Maybe add -Qstrip_debug & -Qstrip_reflect?
WCHAR wname[MAX_PATH];
rtUTF8ToWStr(file_path, wname, MAX_PATH);
// clang-format off
LPCWSTR args[] = {
wname,
L"-E", entry,
L"-T", target_profile,
optimization_arg,
L"-D", L"RT_VULKAN",
L"-spirv"
};
// clang-format on
// Compile!
DxcBuffer buffer{};
buffer.Encoding = codepage;
buffer.Ptr = (LPVOID)code.start;
buffer.Size = (SIZE_T)code.length;
IDxcResult *result;
hr = compiler->Compile(&buffer,
args,
RT_ARRAY_COUNT(args),
include_handler,
IID_PPV_ARGS(&result));
if (SUCCEEDED(hr)) {
result->GetStatus(&hr);
}
if (FAILED(hr) && result) {
// Error occured
IDxcBlobEncoding *error_blob;
hr = result->GetErrorBuffer(&error_blob);
if (SUCCEEDED(hr) && error_blob) {
rtLog("AC", "Shader %s compilation failed: %s", (const char *)error_blob->GetBufferPointer());
error_blob->Release();
} else {
rtLog("AC", "Shader %s compilation failed. No error information available!");
}
include_handler->Release();
utils->Release();
compiler->Release();
library->Release();
return bc;
}
IDxcBlob *code_blob;
result->GetResult(&code_blob);
bc.bytes = rtArenaPush(arena, code_blob->GetBufferSize());
if (bc.bytes) {
bc.len = code_blob->GetBufferSize();
memcpy(bc.bytes, code_blob->GetBufferPointer(), bc.len);
} else {
rtLog("AC", "Out of memory while compiling %s", file_path);
}
include_handler->Release();
utils->Release();
compiler->Release();
library->Release();
return bc;
}

View File

@ -3,6 +3,10 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _WIN32
#define RT_DLLNAME(s) \
(".\\"s \
@ -23,4 +27,8 @@ RT_DLLEXPORT void *rtGetSymbol(rt_dynlib lib, const char *symbol);
RT_DLLEXPORT void rtCloseLib(rt_dynlib lib);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -5,6 +5,10 @@
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/* used to identify a file (XXH3 hash of the path) */
typedef uint64_t rt_file_id;
#define RT_INVALID_FILE_ID 0
@ -19,4 +23,8 @@ RT_DLLEXPORT rt_file_id rtAddFileFromSpan(rt_text_span path);
RT_DLLEXPORT const char *rtGetFilePath(rt_file_id fid);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -5,6 +5,10 @@
#include "runtime.h"
#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
RT_DIRENT_TYPE_FILE,
RT_DIRENT_TYPE_DIRECTORY,
@ -32,4 +36,8 @@ RT_DLLEXPORT uint64_t rtGetFileModificationTimestamp(const char *path);
/* Does not really fit here, but it is mostly useful for comparison with file timestamps. */
RT_DLLEXPORT uint64_t rtGetCurrentTimestamp(void);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -14,6 +14,10 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
/* In renderer_api.h -> Not necessary for almost all gfx usage */
typedef struct rt_renderer_init_info_s rt_renderer_init_info;
@ -50,4 +54,8 @@ typedef struct {
rt_attribute_value value;
} rt_attribute_binding;
#ifdef __cplusplus
}
#endif
#endif

View File

@ -5,8 +5,16 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef uint64_t rt_hash64;
RT_DLLEXPORT rt_hash64 rtHashBytes(const void *begin, size_t count);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -7,6 +7,10 @@
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef void rt_job_fn(void *param, uint32_t iteration);
typedef struct {
@ -17,4 +21,8 @@ typedef struct {
RT_DLLEXPORT void rtDispatchJob(const rt_job_decl *decl);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -3,6 +3,10 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct rt_arena_s {
void *base;
size_t at;
@ -34,7 +38,12 @@ RT_INLINE void rtArenaClear(rt_arena *arena) {
}
RT_INLINE rt_arena_rewindpoint rtGetArenaRewindPoint(rt_arena *arena) {
#ifndef __cplusplus
return (rt_arena_rewindpoint){.at = arena->at};
#else
rt_arena_rewindpoint rp = {arena->at};
return rp;
#endif
}
RT_INLINE void rtArenaRewind(rt_arena *arena, rt_arena_rewindpoint rewind) {
@ -54,4 +63,8 @@ RT_INLINE void rtArenaRewind(rt_arena *arena, rt_arena_rewindpoint rewind) {
#define RT_ARENA_PUSH_ARRAY_ZERO(_Arena, _Type, _N) rtArenaPushZero((_Arena), sizeof(_Type) * (_N))
#define RT_ARENA_POP_ARRAY(_Arena, _Type, _N) rtArenaPop((_Arena), sizeof(_Type) * (_N)
#ifdef __cplusplus
}
#endif
#endif

View File

@ -364,6 +364,8 @@ rt_result PipelineProcessor(rt_file_id file, rt_arena *arena) {
if (result != RT_SUCCESS)
goto out;
out:
rtReleaseBuffer(asset.buffer, asset.size);
return result;

View File

@ -9,6 +9,10 @@
#include "runtime.h"
#include "assets.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _WIN32
struct HINSTANCE__;
struct HWND__;
@ -61,4 +65,8 @@ typedef struct {
extern rt_renderer_api g_renderer;
#endif
#ifdef __cplusplus
}
#endif
#endif

View File

@ -6,6 +6,10 @@
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#if defined(_MSC_VER) && !defined(RT_STATIC_LIB)
#define RT_DLLEXPORT __declspec(dllexport)
#else
@ -115,4 +119,8 @@ RT_DLLEXPORT rt_result rtInitRuntime(void);
RT_DLLEXPORT void rtShutdownRuntime(void);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -1,6 +1,6 @@
#include "shader_compiler.h"
#ifdef RT_BUILD_VULKAN_SHADER_COMPILER
#ifdef RT_BUILD_DXC_SHADER_COMPILER
extern rt_shader_bytecode CompileVulkanShader(rt_shader_stage stage,
rt_shader_optimization_level optimization,
rt_text_span code,
@ -30,7 +30,7 @@ shader_compiler_fn(rt_shader_stage, rt_shader_optimization_level, rt_text_span,
static shader_compiler_fn *_compiler_funcs[RT_SHADER_TYPE_COUNT] = {
CompileNullShader,
#ifdef RT_BUILD_VULKAN_SHADER_COMPILER
#ifdef RT_BUILD_DXC_SHADER_COMPILER
CompileVulkanShader,
#else
CompileNullShader,

View File

@ -1,11 +1,14 @@
#ifndef RT_SHADER_COMPILER_H
#define RT_SHADER_COMPILER_H
#include "runtime.h"
#include "mem_arena.h"
#include "runtime.h"
typedef enum
{
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
RT_SHADER_TYPE_INVALID,
RT_SHADER_TYPE_VULKAN,
@ -29,6 +32,15 @@ typedef struct {
size_t len;
} rt_shader_bytecode;
rt_shader_bytecode CompileShader(rt_shader_type type, rt_shader_stage stage, rt_shader_optimization_level optimization, rt_text_span code, const char *input_file, rt_arena *arena);
rt_shader_bytecode CompileShader(rt_shader_type type,
rt_shader_stage stage,
rt_shader_optimization_level optimization,
rt_text_span code,
const char *input_file,
rt_arena *arena);
#ifdef __cplusplus
}
#endif
#endif

View File

@ -7,6 +7,10 @@
#include "runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
/* Mutexes */
typedef struct rt_mutex_s rt_mutex;
@ -77,4 +81,8 @@ RT_DLLEXPORT bool rtIsMainThread(void);
RT_DLLEXPORT void rtSleep(unsigned int milliseconds);
#ifdef __cplusplus
}
#endif
#endif