tracking asset dependencies
This commit is contained in:
Kevin Trogant
parent
361436ee46
commit
213638009b
6
assets/.gitignore
vendored
Normal file
6
assets/.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
# Ignore output generated by assetc
|
||||
meta.bin
|
||||
uidtab.bin
|
||||
deps.bin
|
||||
*.pkg
|
||||
|
||||
@ -1 +1,2 @@
|
||||
package shaders.pkg;
|
||||
processing_flags 0002;
|
||||
@ -1 +1,2 @@
|
||||
package shaders.pkg;
|
||||
processing_flags 0001;
|
||||
24
contrib/lz4/LICENSE
Normal file
24
contrib/lz4/LICENSE
Normal file
@ -0,0 +1,24 @@
|
||||
LZ4 Library
|
||||
Copyright (c) 2011-2020, Yann Collet
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice, this
|
||||
list of conditions and the following disclaimer in the documentation and/or
|
||||
other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
2766
contrib/lz4/lz4.c
Normal file
2766
contrib/lz4/lz4.c
Normal file
File diff suppressed because it is too large
Load Diff
862
contrib/lz4/lz4.h
Normal file
862
contrib/lz4/lz4.h
Normal file
@ -0,0 +1,862 @@
|
||||
/*
|
||||
* LZ4 - Fast LZ compression algorithm
|
||||
* Header File
|
||||
* Copyright (C) 2011-2020, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 homepage : http://www.lz4.org
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
*/
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef LZ4_H_2983827168210
|
||||
#define LZ4_H_2983827168210
|
||||
|
||||
/* --- Dependency --- */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/**
|
||||
Introduction
|
||||
|
||||
LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,
|
||||
scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
|
||||
multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
|
||||
|
||||
The LZ4 compression library provides in-memory compression and decompression functions.
|
||||
It gives full buffer control to user.
|
||||
Compression can be done in:
|
||||
- a single step (described as Simple Functions)
|
||||
- a single step, reusing a context (described in Advanced Functions)
|
||||
- unbounded multiple steps (described as Streaming compression)
|
||||
|
||||
lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
|
||||
Decompressing such a compressed block requires additional metadata.
|
||||
Exact metadata depends on exact decompression function.
|
||||
For the typical case of LZ4_decompress_safe(),
|
||||
metadata includes block's compressed size, and maximum bound of decompressed size.
|
||||
Each application is free to encode and pass such metadata in whichever way it wants.
|
||||
|
||||
lz4.h only handle blocks, it can not generate Frames.
|
||||
|
||||
Blocks are different from Frames (doc/lz4_Frame_format.md).
|
||||
Frames bundle both blocks and metadata in a specified manner.
|
||||
Embedding metadata is required for compressed data to be self-contained and portable.
|
||||
Frame format is delivered through a companion API, declared in lz4frame.h.
|
||||
The `lz4` CLI can only manage frames.
|
||||
*/
|
||||
|
||||
/*^***************************************************************
|
||||
* Export parameters
|
||||
*****************************************************************/
|
||||
/*
|
||||
* LZ4_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL
|
||||
* LZ4LIB_VISIBILITY :
|
||||
* Control library symbols visibility.
|
||||
*/
|
||||
#ifndef LZ4LIB_VISIBILITY
|
||||
# if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||
# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||
# else
|
||||
# define LZ4LIB_VISIBILITY
|
||||
# endif
|
||||
#endif
|
||||
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
|
||||
# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY
|
||||
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
|
||||
# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||
#else
|
||||
# define LZ4LIB_API LZ4LIB_VISIBILITY
|
||||
#endif
|
||||
|
||||
/*! LZ4_FREESTANDING :
|
||||
* When this macro is set to 1, it enables "freestanding mode" that is
|
||||
* suitable for typical freestanding environment which doesn't support
|
||||
* standard C library.
|
||||
*
|
||||
* - LZ4_FREESTANDING is a compile-time switch.
|
||||
* - It requires the following macros to be defined:
|
||||
* LZ4_memcpy, LZ4_memmove, LZ4_memset.
|
||||
* - It only enables LZ4/HC functions which don't use heap.
|
||||
* All LZ4F_* functions are not supported.
|
||||
* - See tests/freestanding.c to check its basic setup.
|
||||
*/
|
||||
#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1)
|
||||
# define LZ4_HEAPMODE 0
|
||||
# define LZ4HC_HEAPMODE 0
|
||||
# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1
|
||||
# if !defined(LZ4_memcpy)
|
||||
# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'."
|
||||
# endif
|
||||
# if !defined(LZ4_memset)
|
||||
# error "LZ4_FREESTANDING requires macro 'LZ4_memset'."
|
||||
# endif
|
||||
# if !defined(LZ4_memmove)
|
||||
# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'."
|
||||
# endif
|
||||
#elif ! defined(LZ4_FREESTANDING)
|
||||
# define LZ4_FREESTANDING 0
|
||||
#endif
|
||||
|
||||
|
||||
/*------ Version ------*/
|
||||
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
|
||||
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
|
||||
#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
|
||||
|
||||
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
|
||||
|
||||
#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
|
||||
#define LZ4_QUOTE(str) #str
|
||||
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
|
||||
#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */
|
||||
|
||||
LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */
|
||||
LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Tuning parameter
|
||||
**************************************/
|
||||
#define LZ4_MEMORY_USAGE_MIN 10
|
||||
#define LZ4_MEMORY_USAGE_DEFAULT 14
|
||||
#define LZ4_MEMORY_USAGE_MAX 20
|
||||
|
||||
/*!
|
||||
* LZ4_MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; )
|
||||
* Increasing memory usage improves compression ratio, at the cost of speed.
|
||||
* Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
|
||||
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
|
||||
*/
|
||||
#ifndef LZ4_MEMORY_USAGE
|
||||
# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
|
||||
#endif
|
||||
|
||||
#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN)
|
||||
# error "LZ4_MEMORY_USAGE is too small !"
|
||||
#endif
|
||||
|
||||
#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX)
|
||||
# error "LZ4_MEMORY_USAGE is too large !"
|
||||
#endif
|
||||
|
||||
/*-************************************
|
||||
* Simple Functions
|
||||
**************************************/
|
||||
/*! LZ4_compress_default() :
|
||||
* Compresses 'srcSize' bytes from buffer 'src'
|
||||
* into already allocated 'dst' buffer of size 'dstCapacity'.
|
||||
* Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
|
||||
* It also runs faster, so it's a recommended setting.
|
||||
* If the function cannot compress 'src' into a more limited 'dst' budget,
|
||||
* compression stops *immediately*, and the function result is zero.
|
||||
* In which case, 'dst' content is undefined (invalid).
|
||||
* srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
|
||||
* dstCapacity : size of buffer 'dst' (which must be already allocated)
|
||||
* @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
|
||||
* or 0 if compression fails
|
||||
* Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
|
||||
|
||||
/*! LZ4_decompress_safe() :
|
||||
* @compressedSize : is the exact complete size of the compressed block.
|
||||
* @dstCapacity : is the size of destination buffer (which must be already allocated),
|
||||
* is an upper bound of decompressed size.
|
||||
* @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
|
||||
* If destination buffer is not large enough, decoding will stop and output an error code (negative value).
|
||||
* If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
* Note 1 : This function is protected against malicious data packets :
|
||||
* it will never writes outside 'dst' buffer, nor read outside 'source' buffer,
|
||||
* even if the compressed block is maliciously modified to order the decoder to do these actions.
|
||||
* In such case, the decoder stops immediately, and considers the compressed block malformed.
|
||||
* Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.
|
||||
* The implementation is free to send / store / derive this information in whichever way is most beneficial.
|
||||
* If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Advanced Functions
|
||||
**************************************/
|
||||
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
|
||||
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
|
||||
|
||||
/*! LZ4_compressBound() :
|
||||
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
|
||||
This function is primarily useful for memory allocation purposes (destination buffer size).
|
||||
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
|
||||
Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
|
||||
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
|
||||
return : maximum output size in a "worst case" scenario
|
||||
or 0, if input size is incorrect (too large or negative)
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compressBound(int inputSize);
|
||||
|
||||
/*! LZ4_compress_fast() :
|
||||
Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
|
||||
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
|
||||
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
|
||||
An acceleration value of "1" is the same as regular LZ4_compress_default()
|
||||
Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c).
|
||||
Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c).
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
|
||||
/*! LZ4_compress_fast_extState() :
|
||||
* Same as LZ4_compress_fast(), using an externally allocated memory space for its state.
|
||||
* Use LZ4_sizeofState() to know how much memory must be allocated,
|
||||
* and allocate it on 8-bytes boundaries (using `malloc()` typically).
|
||||
* Then, provide this buffer as `void* state` to compression function.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_sizeofState(void);
|
||||
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
|
||||
/*! LZ4_compress_destSize() :
|
||||
* Reverse the logic : compresses as much data as possible from 'src' buffer
|
||||
* into already allocated buffer 'dst', of size >= 'targetDestSize'.
|
||||
* This function either compresses the entire 'src' content into 'dst' if it's large enough,
|
||||
* or fill 'dst' buffer completely with as much data as possible from 'src'.
|
||||
* note: acceleration parameter is fixed to "default".
|
||||
*
|
||||
* *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
|
||||
* New value is necessarily <= input value.
|
||||
* @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
|
||||
* or 0 if compression fails.
|
||||
*
|
||||
* Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+):
|
||||
* the produced compressed content could, in specific circumstances,
|
||||
* require to be decompressed into a destination buffer larger
|
||||
* by at least 1 byte than the content to decompress.
|
||||
* If an application uses `LZ4_compress_destSize()`,
|
||||
* it's highly recommended to update liblz4 to v1.9.2 or better.
|
||||
* If this can't be done or ensured,
|
||||
* the receiving decompression function should provide
|
||||
* a dstCapacity which is > decompressedSize, by at least 1 byte.
|
||||
* See https://github.com/lz4/lz4/issues/859 for details
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
|
||||
|
||||
|
||||
/*! LZ4_decompress_safe_partial() :
|
||||
* Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
|
||||
* into destination buffer 'dst' of size 'dstCapacity'.
|
||||
* Up to 'targetOutputSize' bytes will be decoded.
|
||||
* The function stops decoding on reaching this objective.
|
||||
* This can be useful to boost performance
|
||||
* whenever only the beginning of a block is required.
|
||||
*
|
||||
* @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize)
|
||||
* If source stream is detected malformed, function returns a negative result.
|
||||
*
|
||||
* Note 1 : @return can be < targetOutputSize, if compressed block contains less data.
|
||||
*
|
||||
* Note 2 : targetOutputSize must be <= dstCapacity
|
||||
*
|
||||
* Note 3 : this function effectively stops decoding on reaching targetOutputSize,
|
||||
* so dstCapacity is kind of redundant.
|
||||
* This is because in older versions of this function,
|
||||
* decoding operation would still write complete sequences.
|
||||
* Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize,
|
||||
* it could write more bytes, though only up to dstCapacity.
|
||||
* Some "margin" used to be required for this operation to work properly.
|
||||
* Thankfully, this is no longer necessary.
|
||||
* The function nonetheless keeps the same signature, in an effort to preserve API compatibility.
|
||||
*
|
||||
* Note 4 : If srcSize is the exact size of the block,
|
||||
* then targetOutputSize can be any value,
|
||||
* including larger than the block's decompressed size.
|
||||
* The function will, at most, generate block's decompressed size.
|
||||
*
|
||||
* Note 5 : If srcSize is _larger_ than block's compressed size,
|
||||
* then targetOutputSize **MUST** be <= block's decompressed size.
|
||||
* Otherwise, *silent corruption will occur*.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
|
||||
|
||||
|
||||
/*-*********************************************
|
||||
* Streaming Compression Functions
|
||||
***********************************************/
|
||||
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
|
||||
|
||||
/**
|
||||
Note about RC_INVOKED
|
||||
|
||||
- RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio).
|
||||
https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros
|
||||
|
||||
- Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars)
|
||||
and reports warning "RC4011: identifier truncated".
|
||||
|
||||
- To eliminate the warning, we surround long preprocessor symbol with
|
||||
"#if !defined(RC_INVOKED) ... #endif" block that means
|
||||
"skip this block when rc.exe is trying to read it".
|
||||
*/
|
||||
#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
|
||||
#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
|
||||
LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
|
||||
LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
|
||||
#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
|
||||
#endif
|
||||
|
||||
/*! LZ4_resetStream_fast() : v1.9.0+
|
||||
* Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
|
||||
* (e.g., LZ4_compress_fast_continue()).
|
||||
*
|
||||
* An LZ4_stream_t must be initialized once before usage.
|
||||
* This is automatically done when created by LZ4_createStream().
|
||||
* However, should the LZ4_stream_t be simply declared on stack (for example),
|
||||
* it's necessary to initialize it first, using LZ4_initStream().
|
||||
*
|
||||
* After init, start any new stream with LZ4_resetStream_fast().
|
||||
* A same LZ4_stream_t can be re-used multiple times consecutively
|
||||
* and compress multiple streams,
|
||||
* provided that it starts each new stream with LZ4_resetStream_fast().
|
||||
*
|
||||
* LZ4_resetStream_fast() is much faster than LZ4_initStream(),
|
||||
* but is not compatible with memory regions containing garbage data.
|
||||
*
|
||||
* Note: it's only useful to call LZ4_resetStream_fast()
|
||||
* in the context of streaming compression.
|
||||
* The *extState* functions perform their own resets.
|
||||
* Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive.
|
||||
*/
|
||||
LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
|
||||
|
||||
/*! LZ4_loadDict() :
|
||||
* Use this function to reference a static dictionary into LZ4_stream_t.
|
||||
* The dictionary must remain available during compression.
|
||||
* LZ4_loadDict() triggers a reset, so any previous data will be forgotten.
|
||||
* The same dictionary will have to be loaded on decompression side for successful decoding.
|
||||
* Dictionary are useful for better compression of small data (KB range).
|
||||
* While LZ4 accept any input as dictionary,
|
||||
* results are generally better when using Zstandard's Dictionary Builder.
|
||||
* Loading a size of 0 is allowed, and is the same as reset.
|
||||
* @return : loaded dictionary size, in bytes (necessarily <= 64 KB)
|
||||
*/
|
||||
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
|
||||
|
||||
/*! LZ4_compress_fast_continue() :
|
||||
* Compress 'src' content using data from previously compressed blocks, for better compression ratio.
|
||||
* 'dst' buffer must be already allocated.
|
||||
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
|
||||
*
|
||||
* @return : size of compressed block
|
||||
* or 0 if there is an error (typically, cannot fit into 'dst').
|
||||
*
|
||||
* Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.
|
||||
* Each block has precise boundaries.
|
||||
* Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata.
|
||||
* It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.
|
||||
*
|
||||
* Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory !
|
||||
*
|
||||
* Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.
|
||||
* Make sure that buffers are separated, by at least one byte.
|
||||
* This construction ensures that each block only depends on previous block.
|
||||
*
|
||||
* Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
|
||||
*
|
||||
* Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
/*! LZ4_saveDict() :
|
||||
* If last 64KB data cannot be guaranteed to remain available at its current memory location,
|
||||
* save it into a safer place (char* safeBuffer).
|
||||
* This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
|
||||
* but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
|
||||
* @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
|
||||
*/
|
||||
LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
|
||||
|
||||
|
||||
/*-**********************************************
|
||||
* Streaming Decompression Functions
|
||||
* Bufferless synchronous API
|
||||
************************************************/
|
||||
typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */
|
||||
|
||||
/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
|
||||
* creation / destruction of streaming decompression tracking context.
|
||||
* A tracking context can be re-used multiple times.
|
||||
*/
|
||||
#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
|
||||
#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
|
||||
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
|
||||
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
|
||||
#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
|
||||
#endif
|
||||
|
||||
/*! LZ4_setStreamDecode() :
|
||||
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
|
||||
* Use this function to start decompression of a new stream of blocks.
|
||||
* A dictionary can optionally be set. Use NULL or size 0 for a reset order.
|
||||
* Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
|
||||
* @return : 1 if OK, 0 if error
|
||||
*/
|
||||
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
|
||||
|
||||
/*! LZ4_decoderRingBufferSize() : v1.8.2+
|
||||
* Note : in a ring buffer scenario (optional),
|
||||
* blocks are presumed decompressed next to each other
|
||||
* up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
|
||||
* at which stage it resumes from beginning of ring buffer.
|
||||
* When setting such a ring buffer for streaming decompression,
|
||||
* provides the minimum size of this ring buffer
|
||||
* to be compatible with any source respecting maxBlockSize condition.
|
||||
* @return : minimum ring buffer size,
|
||||
* or 0 if there is an error (invalid maxBlockSize).
|
||||
*/
|
||||
LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
|
||||
#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */
|
||||
|
||||
/*! LZ4_decompress_safe_continue() :
|
||||
* This decoding function allows decompression of consecutive blocks in "streaming" mode.
|
||||
* The difference with the usual independent blocks is that
|
||||
* new blocks are allowed to find references into former blocks.
|
||||
* A block is an unsplittable entity, and must be presented entirely to the decompression function.
|
||||
* LZ4_decompress_safe_continue() only accepts one block at a time.
|
||||
* It's modeled after `LZ4_decompress_safe()` and behaves similarly.
|
||||
*
|
||||
* @LZ4_streamDecode : decompression state, tracking the position in memory of past data
|
||||
* @compressedSize : exact complete size of one compressed block.
|
||||
* @dstCapacity : size of destination buffer (which must be already allocated),
|
||||
* must be an upper bound of decompressed size.
|
||||
* @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
|
||||
* If destination buffer is not large enough, decoding will stop and output an error code (negative value).
|
||||
* If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||
*
|
||||
* The last 64KB of previously decoded data *must* remain available and unmodified
|
||||
* at the memory position where they were previously decoded.
|
||||
* If less than 64KB of data has been decoded, all the data must be present.
|
||||
*
|
||||
* Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
|
||||
* - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
|
||||
* maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
|
||||
* In which case, encoding and decoding buffers do not need to be synchronized.
|
||||
* Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
|
||||
* - Synchronized mode :
|
||||
* Decompression buffer size is _exactly_ the same as compression buffer size,
|
||||
* and follows exactly same update rule (block boundaries at same positions),
|
||||
* and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
|
||||
* _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
|
||||
* - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
|
||||
* In which case, encoding and decoding buffers do not need to be synchronized,
|
||||
* and encoding ring buffer can have any size, including small ones ( < 64 KB).
|
||||
*
|
||||
* Whenever these conditions are not possible,
|
||||
* save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
|
||||
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
|
||||
*/
|
||||
LZ4LIB_API int
|
||||
LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode,
|
||||
const char* src, char* dst,
|
||||
int srcSize, int dstCapacity);
|
||||
|
||||
|
||||
/*! LZ4_decompress_safe_usingDict() :
|
||||
* Works the same as
|
||||
* a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue()
|
||||
* However, it's stateless: it doesn't need any LZ4_streamDecode_t state.
|
||||
* Dictionary is presumed stable : it must remain accessible and unmodified during decompression.
|
||||
* Performance tip : Decompression speed can be substantially increased
|
||||
* when dst == dictStart + dictSize.
|
||||
*/
|
||||
LZ4LIB_API int
|
||||
LZ4_decompress_safe_usingDict(const char* src, char* dst,
|
||||
int srcSize, int dstCapacity,
|
||||
const char* dictStart, int dictSize);
|
||||
|
||||
/*! LZ4_decompress_safe_partial_usingDict() :
|
||||
* Behaves the same as LZ4_decompress_safe_partial()
|
||||
* with the added ability to specify a memory segment for past data.
|
||||
* Performance tip : Decompression speed can be substantially increased
|
||||
* when dst == dictStart + dictSize.
|
||||
*/
|
||||
LZ4LIB_API int
|
||||
LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
|
||||
int compressedSize,
|
||||
int targetOutputSize, int maxOutputSize,
|
||||
const char* dictStart, int dictSize);
|
||||
|
||||
#endif /* LZ4_H_2983827168210 */
|
||||
|
||||
|
||||
/*^*************************************
|
||||
* !!!!!! STATIC LINKING ONLY !!!!!!
|
||||
***************************************/
|
||||
|
||||
/*-****************************************************************************
|
||||
* Experimental section
|
||||
*
|
||||
* Symbols declared in this section must be considered unstable. Their
|
||||
* signatures or semantics may change, or they may be removed altogether in the
|
||||
* future. They are therefore only safe to depend on when the caller is
|
||||
* statically linked against the library.
|
||||
*
|
||||
* To protect against unsafe usage, not only are the declarations guarded,
|
||||
* the definitions are hidden by default
|
||||
* when building LZ4 as a shared/dynamic library.
|
||||
*
|
||||
* In order to access these declarations,
|
||||
* define LZ4_STATIC_LINKING_ONLY in your application
|
||||
* before including LZ4's headers.
|
||||
*
|
||||
* In order to make their implementations accessible dynamically, you must
|
||||
* define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
|
||||
******************************************************************************/
|
||||
|
||||
#ifdef LZ4_STATIC_LINKING_ONLY
|
||||
|
||||
#ifndef LZ4_STATIC_3504398509
|
||||
#define LZ4_STATIC_3504398509
|
||||
|
||||
#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
|
||||
#define LZ4LIB_STATIC_API LZ4LIB_API
|
||||
#else
|
||||
#define LZ4LIB_STATIC_API
|
||||
#endif
|
||||
|
||||
|
||||
/*! LZ4_compress_fast_extState_fastReset() :
|
||||
* A variant of LZ4_compress_fast_extState().
|
||||
*
|
||||
* Using this variant avoids an expensive initialization step.
|
||||
* It is only safe to call if the state buffer is known to be correctly initialized already
|
||||
* (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized").
|
||||
* From a high level, the difference is that
|
||||
* this function initializes the provided state with a call to something like LZ4_resetStream_fast()
|
||||
* while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
|
||||
*/
|
||||
LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||
|
||||
/*! LZ4_attach_dictionary() :
|
||||
* This is an experimental API that allows
|
||||
* efficient use of a static dictionary many times.
|
||||
*
|
||||
* Rather than re-loading the dictionary buffer into a working context before
|
||||
* each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
|
||||
* working LZ4_stream_t, this function introduces a no-copy setup mechanism,
|
||||
* in which the working stream references the dictionary stream in-place.
|
||||
*
|
||||
* Several assumptions are made about the state of the dictionary stream.
|
||||
* Currently, only streams which have been prepared by LZ4_loadDict() should
|
||||
* be expected to work.
|
||||
*
|
||||
* Alternatively, the provided dictionaryStream may be NULL,
|
||||
* in which case any existing dictionary stream is unset.
|
||||
*
|
||||
* If a dictionary is provided, it replaces any pre-existing stream history.
|
||||
* The dictionary contents are the only history that can be referenced and
|
||||
* logically immediately precede the data compressed in the first subsequent
|
||||
* compression call.
|
||||
*
|
||||
* The dictionary will only remain attached to the working stream through the
|
||||
* first compression call, at the end of which it is cleared. The dictionary
|
||||
* stream (and source buffer) must remain in-place / accessible / unchanged
|
||||
* through the completion of the first compression call on the stream.
|
||||
*/
|
||||
LZ4LIB_STATIC_API void
|
||||
LZ4_attach_dictionary(LZ4_stream_t* workingStream,
|
||||
const LZ4_stream_t* dictionaryStream);
|
||||
|
||||
|
||||
/*! In-place compression and decompression
|
||||
*
|
||||
* It's possible to have input and output sharing the same buffer,
|
||||
* for highly constrained memory environments.
|
||||
* In both cases, it requires input to lay at the end of the buffer,
|
||||
* and decompression to start at beginning of the buffer.
|
||||
* Buffer size must feature some margin, hence be larger than final size.
|
||||
*
|
||||
* |<------------------------buffer--------------------------------->|
|
||||
* |<-----------compressed data--------->|
|
||||
* |<-----------decompressed size------------------>|
|
||||
* |<----margin---->|
|
||||
*
|
||||
* This technique is more useful for decompression,
|
||||
* since decompressed size is typically larger,
|
||||
* and margin is short.
|
||||
*
|
||||
* In-place decompression will work inside any buffer
|
||||
* which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
|
||||
* This presumes that decompressedSize > compressedSize.
|
||||
* Otherwise, it means compression actually expanded data,
|
||||
* and it would be more efficient to store such data with a flag indicating it's not compressed.
|
||||
* This can happen when data is not compressible (already compressed, or encrypted).
|
||||
*
|
||||
* For in-place compression, margin is larger, as it must be able to cope with both
|
||||
* history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
|
||||
* and data expansion, which can happen when input is not compressible.
|
||||
* As a consequence, buffer size requirements are much higher,
|
||||
* and memory savings offered by in-place compression are more limited.
|
||||
*
|
||||
* There are ways to limit this cost for compression :
|
||||
* - Reduce history size, by modifying LZ4_DISTANCE_MAX.
|
||||
* Note that it is a compile-time constant, so all compressions will apply this limit.
|
||||
* Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
|
||||
* so it's a reasonable trick when inputs are known to be small.
|
||||
* - Require the compressor to deliver a "maximum compressed size".
|
||||
* This is the `dstCapacity` parameter in `LZ4_compress*()`.
|
||||
* When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
|
||||
* in which case, the return code will be 0 (zero).
|
||||
* The caller must be ready for these cases to happen,
|
||||
* and typically design a backup scheme to send data uncompressed.
|
||||
* The combination of both techniques can significantly reduce
|
||||
* the amount of margin required for in-place compression.
|
||||
*
|
||||
* In-place compression can work in any buffer
|
||||
* which size is >= (maxCompressedSize)
|
||||
* with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
|
||||
* LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
|
||||
* so it's possible to reduce memory requirements by playing with them.
|
||||
*/
|
||||
|
||||
#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32)
|
||||
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */
|
||||
|
||||
#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */
|
||||
# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
|
||||
#endif
|
||||
|
||||
#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */
|
||||
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
|
||||
|
||||
#endif /* LZ4_STATIC_3504398509 */
|
||||
#endif /* LZ4_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
|
||||
#ifndef LZ4_H_98237428734687
|
||||
#define LZ4_H_98237428734687
|
||||
|
||||
/*-************************************************************
|
||||
* Private Definitions
|
||||
**************************************************************
|
||||
* Do not use these definitions directly.
|
||||
* They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
|
||||
* Accessing members will expose user code to API and/or ABI break in future versions of the library.
|
||||
**************************************************************/
|
||||
#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
|
||||
#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
|
||||
#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
|
||||
|
||||
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# include <stdint.h>
|
||||
typedef int8_t LZ4_i8;
|
||||
typedef uint8_t LZ4_byte;
|
||||
typedef uint16_t LZ4_u16;
|
||||
typedef uint32_t LZ4_u32;
|
||||
#else
|
||||
typedef signed char LZ4_i8;
|
||||
typedef unsigned char LZ4_byte;
|
||||
typedef unsigned short LZ4_u16;
|
||||
typedef unsigned int LZ4_u32;
|
||||
#endif
|
||||
|
||||
/*! LZ4_stream_t :
|
||||
* Never ever use below internal definitions directly !
|
||||
* These definitions are not API/ABI safe, and may change in future versions.
|
||||
* If you need static allocation, declare or allocate an LZ4_stream_t object.
|
||||
**/
|
||||
|
||||
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
|
||||
struct LZ4_stream_t_internal {
|
||||
LZ4_u32 hashTable[LZ4_HASH_SIZE_U32];
|
||||
const LZ4_byte* dictionary;
|
||||
const LZ4_stream_t_internal* dictCtx;
|
||||
LZ4_u32 currentOffset;
|
||||
LZ4_u32 tableType;
|
||||
LZ4_u32 dictSize;
|
||||
/* Implicit padding to ensure structure is aligned */
|
||||
};
|
||||
|
||||
#define LZ4_STREAM_MINSIZE ((1UL << LZ4_MEMORY_USAGE) + 32) /* static size, for inter-version compatibility */
|
||||
union LZ4_stream_u {
|
||||
char minStateSize[LZ4_STREAM_MINSIZE];
|
||||
LZ4_stream_t_internal internal_donotuse;
|
||||
}; /* previously typedef'd to LZ4_stream_t */
|
||||
|
||||
|
||||
/*! LZ4_initStream() : v1.9.0+
|
||||
* An LZ4_stream_t structure must be initialized at least once.
|
||||
* This is automatically done when invoking LZ4_createStream(),
|
||||
* but it's not when the structure is simply declared on stack (for example).
|
||||
*
|
||||
* Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t.
|
||||
* It can also initialize any arbitrary buffer of sufficient size,
|
||||
* and will @return a pointer of proper type upon initialization.
|
||||
*
|
||||
* Note : initialization fails if size and alignment conditions are not respected.
|
||||
* In which case, the function will @return NULL.
|
||||
* Note2: An LZ4_stream_t structure guarantees correct alignment and size.
|
||||
* Note3: Before v1.9.0, use LZ4_resetStream() instead
|
||||
**/
|
||||
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
|
||||
|
||||
|
||||
/*! LZ4_streamDecode_t :
|
||||
* Never ever use below internal definitions directly !
|
||||
* These definitions are not API/ABI safe, and may change in future versions.
|
||||
* If you need static allocation, declare or allocate an LZ4_streamDecode_t object.
|
||||
**/
|
||||
typedef struct {
|
||||
const LZ4_byte* externalDict;
|
||||
const LZ4_byte* prefixEnd;
|
||||
size_t extDictSize;
|
||||
size_t prefixSize;
|
||||
} LZ4_streamDecode_t_internal;
|
||||
|
||||
#define LZ4_STREAMDECODE_MINSIZE 32
|
||||
union LZ4_streamDecode_u {
|
||||
char minStateSize[LZ4_STREAMDECODE_MINSIZE];
|
||||
LZ4_streamDecode_t_internal internal_donotuse;
|
||||
} ; /* previously typedef'd to LZ4_streamDecode_t */
|
||||
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Obsolete Functions
|
||||
**************************************/
|
||||
|
||||
/*! Deprecation warnings
|
||||
*
|
||||
* Deprecated functions make the compiler generate a warning when invoked.
|
||||
* This is meant to invite users to update their source code.
|
||||
* Should deprecation warnings be a problem, it is generally possible to disable them,
|
||||
* typically with -Wno-deprecated-declarations for gcc
|
||||
* or _CRT_SECURE_NO_WARNINGS in Visual.
|
||||
*
|
||||
* Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS
|
||||
* before including the header file.
|
||||
*/
|
||||
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
|
||||
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
|
||||
#else
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
|
||||
# elif defined(_MSC_VER)
|
||||
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
|
||||
# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45))
|
||||
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
|
||||
# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31)
|
||||
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
|
||||
# else
|
||||
# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler")
|
||||
# define LZ4_DEPRECATED(message) /* disabled */
|
||||
# endif
|
||||
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
|
||||
|
||||
/*! Obsolete compression functions (since v1.7.3) */
|
||||
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
|
||||
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||
|
||||
/*! Obsolete decompression functions (since v1.8.0) */
|
||||
LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
|
||||
LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
|
||||
|
||||
/* Obsolete streaming functions (since v1.7.0)
|
||||
* degraded functionality; do not use!
|
||||
*
|
||||
* In order to perform streaming compression, these functions depended on data
|
||||
* that is no longer tracked in the state. They have been preserved as well as
|
||||
* possible: using them will still produce a correct output. However, they don't
|
||||
* actually retain any history between compression calls. The compression ratio
|
||||
* achieved will therefore be no better than compressing each chunk
|
||||
* independently.
|
||||
*/
|
||||
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
|
||||
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void);
|
||||
LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);
|
||||
LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);
|
||||
|
||||
/*! Obsolete streaming decoding functions (since v1.7.0) */
|
||||
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
|
||||
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
|
||||
|
||||
/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) :
|
||||
* These functions used to be faster than LZ4_decompress_safe(),
|
||||
* but this is no longer the case. They are now slower.
|
||||
* This is because LZ4_decompress_fast() doesn't know the input size,
|
||||
* and therefore must progress more cautiously into the input buffer to not read beyond the end of block.
|
||||
* On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
|
||||
* As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
|
||||
*
|
||||
* The last remaining LZ4_decompress_fast() specificity is that
|
||||
* it can decompress a block without knowing its compressed size.
|
||||
* Such functionality can be achieved in a more secure manner
|
||||
* by employing LZ4_decompress_safe_partial().
|
||||
*
|
||||
* Parameters:
|
||||
* originalSize : is the uncompressed size to regenerate.
|
||||
* `dst` must be already allocated, its size must be >= 'originalSize' bytes.
|
||||
* @return : number of bytes read from source buffer (== compressed size).
|
||||
* The function expects to finish at block's end exactly.
|
||||
* If the source stream is detected malformed, the function stops decoding and returns a negative result.
|
||||
* note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
|
||||
* However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
|
||||
* Also, since match offsets are not validated, match reads from 'src' may underflow too.
|
||||
* These issues never happen if input (compressed) data is correct.
|
||||
* But they may happen if input data is invalid (error or intentional tampering).
|
||||
* As a consequence, use these functions in trusted environments with trusted data **only**.
|
||||
*/
|
||||
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")
|
||||
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
|
||||
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")
|
||||
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
|
||||
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead")
|
||||
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
|
||||
|
||||
/*! LZ4_resetStream() :
|
||||
* An LZ4_stream_t structure must be initialized at least once.
|
||||
* This is done with LZ4_initStream(), or LZ4_resetStream().
|
||||
* Consider switching to LZ4_initStream(),
|
||||
* invoking LZ4_resetStream() will trigger deprecation warnings in the future.
|
||||
*/
|
||||
LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
|
||||
|
||||
|
||||
#endif /* LZ4_H_98237428734687 */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
31
meson.build
31
meson.build
@ -61,6 +61,7 @@ runtime_lib = library('vyrt',
|
||||
'src/runtime/renderer_api.h',
|
||||
'src/runtime/runtime.h',
|
||||
'src/runtime/threading.h',
|
||||
'src/runtime/uidtab.h',
|
||||
|
||||
'src/runtime/aio.c',
|
||||
'src/runtime/app.c',
|
||||
@ -69,12 +70,13 @@ runtime_lib = library('vyrt',
|
||||
'src/runtime/error_report.c',
|
||||
'src/runtime/file_tab.c',
|
||||
'src/runtime/gfx_main.c',
|
||||
'src/runtime/init.c',
|
||||
'src/runtime/jobs.c',
|
||||
'src/runtime/runtime_cvars.c',
|
||||
'src/runtime/text.c',
|
||||
'src/runtime/threading_cond.c',
|
||||
'src/runtime/threading_mutex.c',
|
||||
'src/runtime/threading_thread.c',
|
||||
'src/runtime/uidtab.c',
|
||||
|
||||
# Contrib Sources
|
||||
'contrib/xxhash/xxhash.c',
|
||||
@ -122,9 +124,20 @@ if host_machine.system() == 'windows'
|
||||
shaderc_libdir = meson.project_source_root() / 'contrib/shaderc/build-win/libshaderc/Release'
|
||||
endif
|
||||
|
||||
|
||||
# Handle linking against both runtime and renderer if we build static libs
|
||||
runtime_link_libs = []
|
||||
if get_option('default_library') == 'static'
|
||||
runtime_link_libs = [runtime_lib, static_renderer_lib]
|
||||
else
|
||||
runtime_link_libs = [runtime_lib]
|
||||
endif
|
||||
|
||||
# Asset Compiler Tool
|
||||
executable('assetc',
|
||||
'src/tools/assetc/assetmeta.h',
|
||||
'src/tools/assetc/assetsettings.h',
|
||||
'src/tools/assetc/dependency_tracking.h',
|
||||
'src/tools/assetc/description_parser.h',
|
||||
'src/tools/assetc/options.h',
|
||||
'src/tools/assetc/packages.h',
|
||||
@ -134,6 +147,8 @@ executable('assetc',
|
||||
|
||||
'src/tools/assetc/assetc.c',
|
||||
'src/tools/assetc/assetmeta.c',
|
||||
'src/tools/assetc/assetsettings.c',
|
||||
'src/tools/assetc/dependency_tracking.c',
|
||||
'src/tools/assetc/description_parser.c',
|
||||
'src/tools/assetc/discovery.c',
|
||||
'src/tools/assetc/packages.c',
|
||||
@ -145,30 +160,24 @@ executable('assetc',
|
||||
|
||||
# Contrib sources
|
||||
'contrib/xxhash/xxhash.c',
|
||||
'contrib/lz4/lz4.c',
|
||||
include_directories : [incdir, shaderc_include],
|
||||
dependencies : [],
|
||||
link_with : [runtime_lib],
|
||||
link_with : runtime_link_libs,
|
||||
link_args : ['-L'+shaderc_libdir, '-lshaderc_combined'],
|
||||
win_subsystem : 'console')
|
||||
|
||||
# Game
|
||||
game_link_libs = []
|
||||
if get_option('default_library') == 'static'
|
||||
game_link_libs = [runtime_lib, static_renderer_lib]
|
||||
else
|
||||
game_link_libs = [runtime_lib]
|
||||
endif
|
||||
|
||||
executable('voyage',
|
||||
'src/game/voyage.c',
|
||||
include_directories : incdir,
|
||||
link_with : game_link_libs,
|
||||
link_with : runtime_link_libs,
|
||||
win_subsystem : 'windows')
|
||||
|
||||
# Unit Tests
|
||||
rttest_exe = executable('rttest',
|
||||
'src/tests/rttest.c',
|
||||
link_with : [runtime_lib],
|
||||
link_with : runtime_link_libs,
|
||||
include_directories : incdir,
|
||||
win_subsystem : 'console')
|
||||
test('runtime test', rttest_exe)
|
||||
@ -62,4 +62,5 @@ VY_DLLEXPORT vy_aio_state vyWaitForAIOCompletion(vy_aio_handle handle);
|
||||
*/
|
||||
VY_DLLEXPORT void vyReleaseAIO(vy_aio_handle handle);
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
@ -27,27 +27,11 @@ static LRESULT CALLBACK win32WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM
|
||||
|
||||
VY_DLLEXPORT int
|
||||
vyWin32Entry(HINSTANCE hInstance, HINSTANCE hPrevInstance, PWSTR pCmdLine, int nCmdShow) {
|
||||
if (vyInitRuntime() != VY_SUCCESS)
|
||||
return 1;
|
||||
|
||||
__RegisterRuntimeCVars();
|
||||
vyRegisterRendererCVars();
|
||||
|
||||
/* TODO: Parse the cvar config file */
|
||||
|
||||
if (vyInitBufferManager() != VY_SUCCESS) {
|
||||
vyReportError("BUFFERMGR", "Init failed.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (vyInitFileTab(1024) != VY_SUCCESS) {
|
||||
vyReportError("FTAB", "Init failed.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (vyInitAIO(0) != VY_SUCCESS) {
|
||||
vyReportError("AIO", "Init failed.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
WNDCLASSEXW wndclass = {
|
||||
.cbSize = sizeof(wndclass),
|
||||
.hInstance = hInstance,
|
||||
@ -127,10 +111,7 @@ vyWin32Entry(HINSTANCE hInstance, HINSTANCE hPrevInstance, PWSTR pCmdLine, int n
|
||||
DestroyWindow(wnd);
|
||||
UnregisterClassW(L"vyWndClass", hInstance);
|
||||
|
||||
vyShutdownAIO();
|
||||
vyShutdownAIO();
|
||||
vyShutdownFileTab();
|
||||
vyShutdownBufferManager();
|
||||
vyShutdownRuntime();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -176,23 +157,11 @@ static void xlibSetFullscreen(Display *dpy, int screen, Window window, bool enab
|
||||
}
|
||||
|
||||
VY_DLLEXPORT int vyXlibEntry(int argc, char **argv) {
|
||||
__RegisterRuntimeCVars();
|
||||
if (vyInitRuntime() != VY_SUCCESS)
|
||||
return 1;
|
||||
|
||||
vyRegisterRendererCVars();
|
||||
|
||||
if (vyInitBufferManager() != VY_SUCCESS) {
|
||||
vyReportError("BUFFERMGR", "Init failed.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (vyInitFileTab(1024) != VY_SUCCESS) {
|
||||
vyReportError("FTAB", "Init failed.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (vyInitAIO(0) != VY_SUCCESS) {
|
||||
vyReportError("AIO", "Init failed.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
Display *dpy = XOpenDisplay(NULL);
|
||||
if (!dpy) {
|
||||
@ -262,9 +231,7 @@ VY_DLLEXPORT int vyXlibEntry(int argc, char **argv) {
|
||||
vyShutdownGFX();
|
||||
XDestroyWindow(dpy, window);
|
||||
XCloseDisplay(dpy);
|
||||
vyShutdownAIO();
|
||||
vyShutdownFileTab();
|
||||
vyShutdownBufferManager();
|
||||
vyShutdownRuntime();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "runtime.h"
|
||||
|
||||
/* Unique identifier for an asset. */
|
||||
typedef uint32_t vy_uid;
|
||||
|
||||
@ -17,4 +19,13 @@ enum {
|
||||
};
|
||||
typedef uint8_t vy_renderer_backend_code;
|
||||
|
||||
enum {
|
||||
VY_UNKNOWN_ASSET = VY_SUCCESS + 1,
|
||||
VY_BUFFER_ALLOC_FAILED,
|
||||
VY_LOAD_FAILED,
|
||||
};
|
||||
|
||||
/* Load an asset without using the cache */
|
||||
VY_DLLEXPORT vy_result vyLoadAssetDirect(vy_uid uid, void **buffer, size_t *size);
|
||||
|
||||
#endif
|
||||
@ -198,6 +198,14 @@ VY_DLLEXPORT void *vyAllocBuffer(size_t size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (size_t i = 0; i < dword_count; ++i) {
|
||||
if (region->bitmap[i] == UINT32_MAX) {
|
||||
continue;
|
||||
}
|
||||
/* Check if we can start the allocation here */
|
||||
|
||||
}
|
||||
}
|
||||
vyUnlockMutex(region->guard);
|
||||
return result;
|
||||
|
||||
@ -50,7 +50,7 @@ vy_file_id vyGetFileId(const char *path) {
|
||||
vy_file_id vyGetFileIdFromSpan(vy_text_span path) {
|
||||
/* Randomly choosen, aka finger smash keyboard */
|
||||
XXH64_hash_t seed = 15340978;
|
||||
vy_file_id fid = (vy_file_id)XXH64(path.start, path.length, seed);
|
||||
vy_file_id fid = (vy_file_id)XXH3_64bits_withSeed(path.start, path.length, seed);
|
||||
if (fid == 0)
|
||||
fid = ~fid;
|
||||
return fid;
|
||||
|
||||
56
src/runtime/init.c
Normal file
56
src/runtime/init.c
Normal file
@ -0,0 +1,56 @@
|
||||
#include "runtime.h"
|
||||
#include "config.h"
|
||||
#include "aio.h"
|
||||
#include "file_tab.h"
|
||||
#include "buffer_manager.h"
|
||||
#include "uidtab.h"
|
||||
|
||||
extern vy_cvar rt_Renderer;
|
||||
extern vy_cvar rt_Fullscreen;
|
||||
extern vy_cvar rt_WindowWidth;
|
||||
extern vy_cvar rt_WindowHeight;
|
||||
extern vy_cvar rt_BufferManagerMemory;
|
||||
|
||||
void __RegisterRuntimeCVars(void) {
|
||||
vyRegisterCVAR(&rt_Renderer);
|
||||
vyRegisterCVAR(&rt_Fullscreen);
|
||||
vyRegisterCVAR(&rt_WindowWidth);
|
||||
vyRegisterCVAR(&rt_WindowHeight);
|
||||
vyRegisterCVAR(&rt_BufferManagerMemory);
|
||||
}
|
||||
|
||||
extern void __RTSetMainThreadId(void);
|
||||
|
||||
VY_DLLEXPORT vy_result vyInitRuntime(void) {
|
||||
__RTSetMainThreadId();
|
||||
__RegisterRuntimeCVars();
|
||||
|
||||
vy_result res;
|
||||
if ((res = vyInitBufferManager()) != VY_SUCCESS) {
|
||||
vyReportError("BUFFERMGR", "Init failed.");
|
||||
return res;
|
||||
}
|
||||
|
||||
if ((res = vyInitFileTab(1024)) != VY_SUCCESS) {
|
||||
vyReportError("FTAB", "Init failed.");
|
||||
return res;
|
||||
}
|
||||
|
||||
if ((res = vyInitAIO(0)) != VY_SUCCESS) {
|
||||
vyReportError("AIO", "Init failed.");
|
||||
return res;
|
||||
}
|
||||
|
||||
if ((res = vyLoadUIDTable()) != VY_SUCCESS) {
|
||||
vyLog("CORE", "LoadUIDTable returned result: %u", res);
|
||||
}
|
||||
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
|
||||
VY_DLLEXPORT void vyShutdownRuntime(void) {
|
||||
vyReleaseUIDTable();
|
||||
vyShutdownAIO();
|
||||
vyShutdownFileTab();
|
||||
vyShutdownBufferManager();
|
||||
}
|
||||
@ -87,4 +87,10 @@ static VY_INLINE void vySetRelptr(vy_relptr *ptr, void *target) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Runtime init. Initializes basic systems.
|
||||
* You need to call this, even if you build a CLI only app. */
|
||||
VY_DLLEXPORT vy_result vyInitRuntime(void);
|
||||
|
||||
VY_DLLEXPORT void vyShutdownRuntime(void);
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,15 +0,0 @@
|
||||
#include "config.h"
|
||||
|
||||
extern vy_cvar rt_Renderer;
|
||||
extern vy_cvar rt_Fullscreen;
|
||||
extern vy_cvar rt_WindowWidth;
|
||||
extern vy_cvar rt_WindowHeight;
|
||||
extern vy_cvar rt_BufferManagerMemory;
|
||||
|
||||
void __RegisterRuntimeCVars(void) {
|
||||
vyRegisterCVAR(&rt_Renderer);
|
||||
vyRegisterCVAR(&rt_Fullscreen);
|
||||
vyRegisterCVAR(&rt_WindowWidth);
|
||||
vyRegisterCVAR(&rt_WindowHeight);
|
||||
vyRegisterCVAR(&rt_BufferManagerMemory);
|
||||
}
|
||||
@ -7,6 +7,8 @@
|
||||
|
||||
#include "runtime.h"
|
||||
|
||||
/* Mutexes */
|
||||
|
||||
typedef struct vy_mutex_s vy_mutex;
|
||||
|
||||
VY_DLLEXPORT vy_mutex *vyCreateMutex(void);
|
||||
@ -17,6 +19,8 @@ VY_DLLEXPORT bool vyLockMutex(vy_mutex *mutex);
|
||||
|
||||
VY_DLLEXPORT bool vyUnlockMutex(vy_mutex *mutex);
|
||||
|
||||
/* Condition variables */
|
||||
|
||||
typedef struct vy_condition_var_s vy_condition_var;
|
||||
|
||||
VY_DLLEXPORT vy_condition_var *vyCreateConditionVar(void);
|
||||
@ -30,6 +34,8 @@ VY_DLLEXPORT void vyUnlockConditionVar(vy_condition_var *var, bool signal);
|
||||
/* The condition variable must be locked by the thread! */
|
||||
VY_DLLEXPORT void vyWaitOnConditionVar(vy_condition_var *var);
|
||||
|
||||
/* Threads */
|
||||
|
||||
typedef struct vy_thread_s vy_thread;
|
||||
|
||||
typedef void vy_thread_entry_fn(void *param);
|
||||
@ -40,4 +46,10 @@ VY_DLLEXPORT void vyJoinThread(vy_thread *thread);
|
||||
|
||||
VY_DLLEXPORT unsigned int vyGetCPUCoreCount(void);
|
||||
|
||||
typedef uint32_t vy_thread_id;
|
||||
|
||||
VY_DLLEXPORT vy_thread_id vyGetCurrentThreadId(void);
|
||||
|
||||
VY_DLLEXPORT bool vyIsMainThread(void);
|
||||
|
||||
#endif
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
struct vy_mutex_s {
|
||||
HANDLE handle;
|
||||
ptrdiff_t next_reusable;
|
||||
DWORD owner;
|
||||
};
|
||||
|
||||
#define MAX_MUTEX 1024
|
||||
@ -39,6 +40,7 @@ VY_DLLEXPORT vy_mutex *vyCreateMutex(void) {
|
||||
if (_first_reusable < MAX_MUTEX) {
|
||||
vy_mutex *mtx = &_mutex[_first_reusable];
|
||||
_first_reusable = mtx->next_reusable;
|
||||
mtx->owner = 0;
|
||||
return mtx;
|
||||
} else if (_next < MAX_MUTEX) {
|
||||
vy_mutex *mtx = &_mutex[_next];
|
||||
@ -48,6 +50,7 @@ VY_DLLEXPORT vy_mutex *vyCreateMutex(void) {
|
||||
return NULL;
|
||||
}
|
||||
mtx->next_reusable = MAX_MUTEX;
|
||||
mtx->owner = 0;
|
||||
++_next;
|
||||
return mtx;
|
||||
}
|
||||
@ -62,11 +65,33 @@ VY_DLLEXPORT void vyDestroyMutex(vy_mutex *mutex) {
|
||||
}
|
||||
|
||||
VY_DLLEXPORT bool vyLockMutex(vy_mutex *mutex) {
|
||||
return WaitForSingleObjectEx(mutex->handle, INFINITE, TRUE) == WAIT_OBJECT_0;
|
||||
DWORD caller = GetCurrentThreadId();
|
||||
DWORD result;
|
||||
do {
|
||||
result = WaitForSingleObjectEx(mutex->handle, INFINITE, TRUE);
|
||||
if (result == WAIT_OBJECT_0) {
|
||||
mutex->owner = caller;
|
||||
} else if (result != WAIT_IO_COMPLETION) {
|
||||
vyLog("core",
|
||||
"WaitForSingleObjectEx returned %x (GetLastError(): %u)",
|
||||
result,
|
||||
(result == WAIT_FAILED) ? GetLastError() : 0);
|
||||
}
|
||||
} while (result == WAIT_IO_COMPLETION);
|
||||
return result == WAIT_OBJECT_0;
|
||||
}
|
||||
|
||||
VY_DLLEXPORT bool vyUnlockMutex(vy_mutex *mutex) {
|
||||
return ReleaseMutex(mutex->handle) != 0;
|
||||
DWORD caller = GetCurrentThreadId();
|
||||
if (caller != mutex->owner) {
|
||||
vyReportError("core", "Tried to unlock a mutex held by another thread.");
|
||||
return false;
|
||||
}
|
||||
mutex->owner = 0;
|
||||
bool result = ReleaseMutex(mutex->handle) != 0;
|
||||
if (!result)
|
||||
mutex->owner = caller;
|
||||
return result;
|
||||
}
|
||||
|
||||
#elif defined(__linux__)
|
||||
|
||||
@ -25,6 +25,13 @@ static ptrdiff_t _next = 0;
|
||||
static HANDLE _guard;
|
||||
static INIT_ONCE _guard_init = INIT_ONCE_STATIC_INIT;
|
||||
|
||||
static vy_thread_id _main_thread_id;
|
||||
|
||||
/* Called by the runtime during setup */
|
||||
extern void __RTSetMainThreadId(void) {
|
||||
_main_thread_id = (vy_thread_id)GetCurrentThreadId();
|
||||
}
|
||||
|
||||
static BOOL CALLBACK InitGuardFn(PINIT_ONCE initOnce, PVOID parameter, PVOID *context) {
|
||||
VY_UNUSED(initOnce);
|
||||
VY_UNUSED(parameter);
|
||||
@ -108,6 +115,14 @@ VY_DLLEXPORT unsigned int vyGetCPUCoreCount(void) {
|
||||
return (unsigned int)info.dwNumberOfProcessors;
|
||||
}
|
||||
|
||||
VY_DLLEXPORT vy_thread_id vyGetCurrentThreadId(void) {
|
||||
return (vy_thread_id)GetCurrentThreadId();
|
||||
}
|
||||
|
||||
VY_DLLEXPORT bool vyIsMainThread(void) {
|
||||
return vyGetCurrentThreadId() == _main_thread_id;
|
||||
}
|
||||
|
||||
#elif defined(__linux__)
|
||||
|
||||
#define _GNU_SOURCE
|
||||
@ -185,4 +200,13 @@ VY_DLLEXPORT unsigned int vyGetCPUCoreCount(void) {
|
||||
return (n > 0) ? (unsigned int)n : (unsigned int)sysconf(_SC_NPROCESSORS_CONF);
|
||||
}
|
||||
|
||||
VY_DLLEXPORT vy_thread_id vyGetCurrentThreadId(void) {
|
||||
return (vy_thread_id)GetCurrentThreadId();
|
||||
}
|
||||
|
||||
VY_DLLEXPORT bool vyIsMainThread(void) {
|
||||
return vyGetCurrentThreadId() == _main_thread_id;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
133
src/runtime/uidtab.c
Normal file
133
src/runtime/uidtab.c
Normal file
@ -0,0 +1,133 @@
|
||||
#define VY_DEFINE_UIDTAB_FILE_STRUCTURES
|
||||
#include "uidtab.h"
|
||||
#include "aio.h"
|
||||
|
||||
#include "xxhash/xxhash.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
typedef struct {
|
||||
vy_uid *uids;
|
||||
vy_uid_data *data;
|
||||
uint32_t slots;
|
||||
} vy_uidtab;
|
||||
|
||||
static vy_uidtab _tab;
|
||||
|
||||
/* Portable solution, On x64 using clzl is probably faster. */
|
||||
static uint32_t NextPowerOfTwo(uint32_t v) {
|
||||
v--;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
v++;
|
||||
return v;
|
||||
}
|
||||
|
||||
VY_DLLEXPORT vy_result vyLoadUIDTable(void) {
|
||||
/* We use stdio here, because we cannot load any asset in parallel to this.
|
||||
* This is because the uidtab is what tells us which assets exist.
|
||||
*/
|
||||
FILE *f = fopen("uidtab.bin", "rb");
|
||||
if (!f)
|
||||
return VY_LOAD_FAILED;
|
||||
|
||||
vy_uidtab_header header;
|
||||
if (fread(&header, sizeof(header), 1, f) != 1) {
|
||||
fclose(f);
|
||||
return VY_LOAD_FAILED;
|
||||
}
|
||||
|
||||
/* TODO(Kevin): For some reason, the checksum calculation causes
|
||||
* Memory access errors .
|
||||
XXH3_state_t *checksum = XXH3_createState();
|
||||
if (!checksum) {
|
||||
fclose(f);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
*/
|
||||
|
||||
_tab.slots = NextPowerOfTwo(header.num_entries * 2);
|
||||
void *mem = malloc((sizeof(vy_uid) + sizeof(vy_uid_data)) * _tab.slots);
|
||||
if (!mem) {
|
||||
fclose(f);
|
||||
_tab.slots = 0;
|
||||
return VY_BUFFER_ALLOC_FAILED;
|
||||
}
|
||||
_tab.uids = mem;
|
||||
_tab.data = (vy_uid_data *)(_tab.uids + _tab.slots);
|
||||
memset(mem, 0, (sizeof(vy_uid) + sizeof(vy_uid_data)) * _tab.slots);
|
||||
|
||||
uint32_t mod = _tab.slots - 1;
|
||||
for (uint32_t i = 0; i < header.num_entries; ++i) {
|
||||
vy_uidtab_entry entry;
|
||||
if (fread(&entry, sizeof(entry), 1, f) != 1) {
|
||||
free(mem);
|
||||
_tab.slots = 0;
|
||||
fclose(f);
|
||||
return VY_LOAD_FAILED;
|
||||
}
|
||||
//XXH3_64bits_update(checksum, &entry, sizeof(entry));
|
||||
|
||||
/* Insert into hashtable */
|
||||
bool inserted = false;
|
||||
for (uint32_t j = 0; j < _tab.slots; ++j) {
|
||||
uint32_t at = (entry.uid + j) & mod;
|
||||
if (_tab.uids[at] == VY_INVALID_UID) {
|
||||
_tab.uids[at] = entry.uid;
|
||||
_tab.data[at].pkg_file = entry.file;
|
||||
_tab.data[at].size = entry.size;
|
||||
_tab.data[at].offset = entry.offset;
|
||||
|
||||
inserted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!inserted) {
|
||||
vyReportError("core",
|
||||
"Failed to insert an entry into the uid table. This should not happen.");
|
||||
fclose(f);
|
||||
free(mem);
|
||||
_tab.slots = 0;
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
|
||||
/*
|
||||
XXH64_hash_t checksum_hash = XXH3_64bits_digest(checksum);
|
||||
XXH64_hash_t file_hash = XXH64_hashFromCanonical(&header.checksum);
|
||||
XXH3_freeState(checksum);
|
||||
|
||||
if (checksum_hash != file_hash) {
|
||||
vyLog("core",
|
||||
"WARNING: uidtab.bin checksum does not match calculated checksum of loaded entries.");
|
||||
}
|
||||
*/
|
||||
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
|
||||
VY_DLLEXPORT void vyReleaseUIDTable(void) {
|
||||
free(_tab.uids);
|
||||
_tab.slots = 0;
|
||||
}
|
||||
|
||||
VY_DLLEXPORT const vy_uid_data *vyGetUIDData(vy_uid uid) {
|
||||
uint32_t mod = _tab.slots - 1;
|
||||
for (uint32_t j = 0; j < _tab.slots; ++j) {
|
||||
uint32_t at = (uid + j) & mod;
|
||||
if (_tab.uids[at] == uid) {
|
||||
return &_tab.data[at];
|
||||
} else if (_tab.uids[at] == VY_INVALID_UID) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
38
src/runtime/uidtab.h
Normal file
38
src/runtime/uidtab.h
Normal file
@ -0,0 +1,38 @@
|
||||
#ifndef VY_UIDTAB_H
|
||||
#define VY_UIDTAB_H
|
||||
|
||||
#include "runtime.h"
|
||||
#include "file_tab.h"
|
||||
#include "assets.h"
|
||||
#include "xxhash/xxhash.h"
|
||||
|
||||
#ifdef VY_DEFINE_UIDTAB_FILE_STRUCTURES
|
||||
#pragma pack(push, 1)
|
||||
typedef struct {
|
||||
XXH64_canonical_t checksum;
|
||||
uint32_t num_entries;
|
||||
} vy_uidtab_header;
|
||||
#pragma pack(pop)
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef struct {
|
||||
vy_file_id file;
|
||||
uint64_t offset;
|
||||
uint64_t size;
|
||||
vy_uid uid;
|
||||
} vy_uidtab_entry;
|
||||
#pragma pack(pop)
|
||||
#endif
|
||||
|
||||
/* Data associated with an uid */
|
||||
typedef struct {
|
||||
vy_file_id pkg_file;
|
||||
uint64_t offset;
|
||||
uint64_t size;
|
||||
} vy_uid_data;
|
||||
|
||||
VY_DLLEXPORT vy_result vyLoadUIDTable(void);
|
||||
VY_DLLEXPORT void vyReleaseUIDTable(void);
|
||||
VY_DLLEXPORT const vy_uid_data *vyGetUIDData(vy_uid uid);
|
||||
|
||||
#endif
|
||||
@ -2,12 +2,14 @@
|
||||
#include "processing.h"
|
||||
#include "utils.h"
|
||||
#include "packages.h"
|
||||
#include "dependency_tracking.h"
|
||||
|
||||
#define VY_ASSETC_DONT_DEFINE_OPTIONS_GLOBAL
|
||||
#include "options.h"
|
||||
|
||||
#include "runtime/aio.h"
|
||||
#include "runtime/buffer_manager.h"
|
||||
#include "runtime/uidtab.h"
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdlib.h>
|
||||
@ -91,28 +93,25 @@ static bool ParseCommandLineArgs(int argc, char **argv) {
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
/* Setup */
|
||||
if (vyInitFileTab(4096) != VY_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
if (vyInitAIO(256) != VY_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
if (vyInitBufferManager() != VY_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
vyInitRuntime();
|
||||
|
||||
/* Init assetc */
|
||||
if (!ParseCommandLineArgs(argc, argv)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
vyInitPackages();
|
||||
vySetWorkingDirectory(g_assetc_options.root_directory);
|
||||
|
||||
/* Explictily reload uidtab, because we changed the working directory here */
|
||||
vyLoadUIDTable();
|
||||
|
||||
if (vyLoadAssetMeta() != VY_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
vySetWorkingDirectory(g_assetc_options.root_directory);
|
||||
|
||||
if (vyInitDependencyTracking() != VY_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (vyAddAssetProcessor(".pipeline", vyProcessPipelineFile) != VY_SUCCESS)
|
||||
return 1;
|
||||
@ -123,16 +122,22 @@ int main(int argc, char **argv) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* "Mainloop" */
|
||||
vyDiscoverAssets();
|
||||
|
||||
vyWaitUntilProcessingIsFinished();
|
||||
|
||||
vyStopProcessing();
|
||||
vySaveAssetMeta();
|
||||
|
||||
vyShutdownBufferManager();
|
||||
vyShutdownAIO();
|
||||
vyShutdownFileTab();
|
||||
/* Write result */
|
||||
vySaveAssetMeta();
|
||||
if (vySavePackages() != VY_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
if (vyWriteUIDTab() != VY_SUCCESS) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
vyShutdownRuntime();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -204,6 +204,7 @@ void vyAddUIDMapping(vy_file_id fid, vy_uid uid, const vy_assetmeta *meta) {
|
||||
} else {
|
||||
_map.meta[slot].last_changed = 0;
|
||||
_map.meta[slot].compiled_ts = 0;
|
||||
_map.meta[slot].processing_flags = 0;
|
||||
}
|
||||
++_map.used_slots;
|
||||
break;
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
typedef struct {
|
||||
uint64_t last_changed;
|
||||
uint64_t compiled_ts;
|
||||
uint32_t processing_flags;
|
||||
} vy_assetmeta;
|
||||
|
||||
vy_result vyLoadAssetMeta(void);
|
||||
|
||||
144
src/tools/assetc/assetsettings.c
Normal file
144
src/tools/assetc/assetsettings.c
Normal file
@ -0,0 +1,144 @@
|
||||
#include "assetsettings.h"
|
||||
#include "description_parser.h"
|
||||
#include "packages.h"
|
||||
#include "utils.h"
|
||||
|
||||
#include "runtime/aio.h"
|
||||
#include "runtime/buffer_manager.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
|
||||
vy_result vyParseAssetSettings(const char *text,
|
||||
size_t length,
|
||||
const char *file_path,
|
||||
vy_asset_settings *settings) {
|
||||
unsigned int root_list;
|
||||
vy_parse_state state;
|
||||
vy_result res = vyParseDescription(text, length, file_path, &root_list, &state);
|
||||
if (res != VY_SUCCESS) {
|
||||
vyReportError("ASSETC", "Failed to parse asset settings: %s", file_path);
|
||||
return res;
|
||||
}
|
||||
|
||||
/* Default settings */
|
||||
settings->package = 0;
|
||||
settings->processing_flags = 0;
|
||||
settings->reprocess_on_dependency_change = false;
|
||||
|
||||
const vy_parsed_stmt *package_stmt = vyFindStatement(&state, root_list, "package");
|
||||
if (package_stmt) {
|
||||
if (package_stmt->form != VY_STMT_FORM_VALUE) {
|
||||
vyReportError("ASSETC",
|
||||
"Expected a package name as the value of 'package' in %s.",
|
||||
file_path);
|
||||
res = VY_UNKNOWN_ERROR;
|
||||
goto out;
|
||||
}
|
||||
settings->package = vyAddPackageFile(package_stmt->value);
|
||||
}
|
||||
|
||||
const vy_parsed_stmt *flags_stmt = vyFindStatement(&state, root_list, "processing_flags");
|
||||
if (flags_stmt) {
|
||||
if (flags_stmt->form != VY_STMT_FORM_VALUE) {
|
||||
vyReportError("ASSETC",
|
||||
"Expected a hexadecimal number as the value of 'processing_flags' in %s.",
|
||||
file_path);
|
||||
res = VY_UNKNOWN_ERROR;
|
||||
goto out;
|
||||
}
|
||||
|
||||
sscanf(flags_stmt->value.start, "%x", &settings->processing_flags);
|
||||
}
|
||||
|
||||
const vy_parsed_stmt *reprocess_stmt =
|
||||
vyFindStatement(&state, root_list, "reprocess_on_dependency_change");
|
||||
if (reprocess_stmt) {
|
||||
if (reprocess_stmt->form != VY_STMT_FORM_VALUE) {
|
||||
vyReportError("ASSETC",
|
||||
"Expected either 'true' or 'false' as the value of 'reprocess_on_dependency_change' in %s.",
|
||||
file_path);
|
||||
res = VY_UNKNOWN_ERROR;
|
||||
goto out;
|
||||
}
|
||||
if (vyCompareSpanToString(reprocess_stmt->value, "true") == 0)
|
||||
settings->reprocess_on_dependency_change = true;
|
||||
else if (vyCompareSpanToString(reprocess_stmt->value, "false") == 0)
|
||||
settings->reprocess_on_dependency_change = false;
|
||||
else {
|
||||
vyReportError("ASSETC",
|
||||
"Expected either 'true' or 'false' as the value of "
|
||||
"'reprocess_on_dependency_change' in %s.",
|
||||
file_path);
|
||||
res = VY_UNKNOWN_ERROR;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
out:
|
||||
vyReleaseParseState(&state);
|
||||
return res;
|
||||
}
|
||||
|
||||
vy_result vyLoadAssetSettings(const char *asset_path, vy_asset_settings *settings) {
|
||||
size_t path_len = strlen(asset_path);
|
||||
|
||||
char *as_path = malloc(path_len + 3);
|
||||
if (!as_path) {
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
memcpy(as_path, asset_path, path_len);
|
||||
|
||||
size_t ext_len = 0;
|
||||
while (ext_len < path_len) {
|
||||
if (as_path[path_len - ext_len] == '.')
|
||||
break;
|
||||
++ext_len;
|
||||
}
|
||||
|
||||
strcpy(&as_path[path_len - ext_len], ".as");
|
||||
size_t as_size = vyGetFileSize(as_path);
|
||||
if (as_size == 0) {
|
||||
vyReportError("ASSETC", "Failed to retrieve size of setting file %s", as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
void *as_buffer = vyAllocBuffer(as_size);
|
||||
|
||||
vy_load_batch as_load;
|
||||
as_load.loads[0].file = vyAddFile(as_path);
|
||||
as_load.loads[0].num_bytes = as_size;
|
||||
as_load.loads[0].dest = as_buffer;
|
||||
if (!as_load.loads[0].dest) {
|
||||
vyReportError("ASSETC", "Failed to allocate buffer for setting file %s", as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
as_load.loads[0].offset = 0;
|
||||
as_load.num_loads = 1;
|
||||
|
||||
vy_aio_handle as_handle;
|
||||
if (vySubmitLoadBatch(&as_load, &as_handle) != VY_SUCCESS) {
|
||||
vyReportError("ASSETC", "Failed to submit load of setting file %s", as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
if (vyWaitForAIOCompletion(as_handle) != VY_AIO_STATE_FINISHED) {
|
||||
vyReportError("ASSETC", "Failed to load setting file %s", as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
vyReleaseAIO(as_handle);
|
||||
|
||||
if (vyParseAssetSettings(as_buffer, as_size, as_path, settings) != VY_SUCCESS) {
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
free(as_path);
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
21
src/tools/assetc/assetsettings.h
Normal file
21
src/tools/assetc/assetsettings.h
Normal file
@ -0,0 +1,21 @@
|
||||
#ifndef VY_ASSETC_ASSETSETTINGS_H
|
||||
#define VY_ASSETC_ASSETSETTINGS_H
|
||||
|
||||
#include "runtime/runtime.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
typedef struct {
|
||||
unsigned int package;
|
||||
uint32_t processing_flags;
|
||||
bool reprocess_on_dependency_change;
|
||||
} vy_asset_settings;
|
||||
|
||||
|
||||
vy_result vyParseAssetSettings(const char *text,
|
||||
size_t length,
|
||||
const char *file_path,
|
||||
vy_asset_settings *settings);
|
||||
|
||||
vy_result vyLoadAssetSettings(const char *asset_path, vy_asset_settings *settings);
|
||||
|
||||
#endif
|
||||
127
src/tools/assetc/dependency_tracking.c
Normal file
127
src/tools/assetc/dependency_tracking.c
Normal file
@ -0,0 +1,127 @@
|
||||
#include "dependency_tracking.h"
|
||||
|
||||
#include "runtime/assets.h"
|
||||
#include "runtime/threading.h"
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <limits.h>
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
|
||||
/* Track a list of dependencies per asset.
|
||||
* For the runtime, we only care about dependent -> dependency (so for example material -> texture),
|
||||
* but for assetc we also care about the opposite direction:
|
||||
* If an asset has the reprocess_on_dependency_change setting enabled,
|
||||
* we need to queue a reprocessing, even if the asset itself did not change.
|
||||
*/
|
||||
|
||||
#define END_OF_LIST 0
|
||||
|
||||
/* 64 byte cache line - 8 (next index + count) / 4 (u32) = 14 */
|
||||
#define BUCKET_ENTRY_COUNT 14
|
||||
typedef struct vy_dep_list_bucket_s {
|
||||
uint32_t next;
|
||||
uint32_t count;
|
||||
vy_uid entries[BUCKET_ENTRY_COUNT];
|
||||
} vy_dep_list_bucket;
|
||||
|
||||
typedef union {
|
||||
/* Indices of the first buckets */
|
||||
struct {
|
||||
uint32_t dependencies;
|
||||
uint32_t dependents;
|
||||
};
|
||||
uint32_t lists[2];
|
||||
} vy_dep_list;
|
||||
|
||||
static vy_mutex *_guard;
|
||||
|
||||
static vy_dep_list_bucket *_buckets;
|
||||
static uint32_t _bucket_count;
|
||||
static uint32_t _bucket_capacity;
|
||||
|
||||
|
||||
#define MAP_SIZE 2048
|
||||
static vy_uid _uids[MAP_SIZE];
|
||||
static vy_dep_list _lists[MAP_SIZE];
|
||||
|
||||
vy_result vyInitDependencyTracking(void) {
|
||||
_guard = vyCreateMutex();
|
||||
if (!_guard)
|
||||
return VY_UNKNOWN_ERROR;
|
||||
|
||||
_buckets = malloc(sizeof(vy_dep_list_bucket) * 256);
|
||||
if (!_buckets)
|
||||
return VY_UNKNOWN_ERROR;
|
||||
_bucket_capacity = 256;
|
||||
_bucket_count = 1;
|
||||
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
|
||||
static uint32_t AllocNewBucket(void) {
|
||||
if (_bucket_count == _bucket_capacity) {
|
||||
void *t = realloc(_buckets, (size_t)_bucket_capacity * 2 * sizeof(vy_dep_list_bucket));
|
||||
if (!t)
|
||||
return 0;
|
||||
_buckets = t;
|
||||
_bucket_capacity *= 2;
|
||||
}
|
||||
memset(&_buckets[_bucket_count], 0, sizeof(vy_dep_list_bucket));
|
||||
return _bucket_count++;
|
||||
}
|
||||
|
||||
static vy_result InsertIntoList(vy_uid list_asset, vy_uid uid, int list_index) {
|
||||
vyLockMutex(_guard);
|
||||
bool inserted = false;
|
||||
for (uint32_t i = 0; i < MAP_SIZE; i++) {
|
||||
uint32_t at = (list_asset + i) % MAP_SIZE;
|
||||
if (_uids[at] == list_asset || _uids[at] == 0) {
|
||||
_uids[at] = list_asset;
|
||||
|
||||
/* Alloc a new list, if necessary */
|
||||
if (_lists[at].lists[list_index] == 0) {
|
||||
_lists[at].lists[list_index] = AllocNewBucket();
|
||||
if (!_lists[at].lists[list_index]) {
|
||||
vyUnlockMutex(_guard);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
/* Advance to the end of the list */
|
||||
vy_dep_list_bucket *bucket = &_buckets[_lists[at].lists[list_index]];
|
||||
while (bucket->next != END_OF_LIST) {
|
||||
bucket = &_buckets[bucket->next];
|
||||
}
|
||||
|
||||
/* Grow the list, if necessary */
|
||||
if (bucket->count == BUCKET_ENTRY_COUNT) {
|
||||
bucket->next = AllocNewBucket();
|
||||
if (!bucket->next) {
|
||||
vyUnlockMutex(_guard);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
bucket = &_buckets[bucket->next];
|
||||
}
|
||||
|
||||
assert(bucket->count < BUCKET_ENTRY_COUNT);
|
||||
bucket->entries[bucket->count++] = uid;
|
||||
|
||||
inserted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
vyUnlockMutex(_guard);
|
||||
assert(inserted);
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
|
||||
vy_result vyAddAssetDependency(vy_uid dependent, vy_uid dependency) {
|
||||
vy_result res = InsertIntoList(dependent, dependency, 0);
|
||||
if (res != VY_SUCCESS)
|
||||
return res;
|
||||
res = InsertIntoList(dependency, dependent, 1);
|
||||
return res;
|
||||
}
|
||||
12
src/tools/assetc/dependency_tracking.h
Normal file
12
src/tools/assetc/dependency_tracking.h
Normal file
@ -0,0 +1,12 @@
|
||||
#ifndef VY_ASSETC_DEPENDENCY_TRACKING_H
|
||||
#define VY_ASSETC_DEPENDENCY_TRACKING_H
|
||||
|
||||
#include "runtime/runtime.h"
|
||||
#include "runtime/assets.h"
|
||||
|
||||
|
||||
vy_result vyInitDependencyTracking(void);
|
||||
|
||||
vy_result vyAddAssetDependency(vy_uid dependent, vy_uid dependency);
|
||||
|
||||
#endif
|
||||
@ -1,49 +1,84 @@
|
||||
#include "utils.h"
|
||||
#include "assetmeta.h"
|
||||
#include "processing.h"
|
||||
#include "assetsettings.h"
|
||||
#include "packages.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "runtime/uidtab.h"
|
||||
|
||||
typedef struct {
|
||||
char path_scratch[1024];
|
||||
unsigned int path_end;
|
||||
} vy_discovery_data;
|
||||
|
||||
static vy_result LoadCompressedAsset(vy_uid uid, void **buffer, size_t size) {
|
||||
}
|
||||
|
||||
static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type, void *user) {
|
||||
vy_discovery_data *data = user;
|
||||
|
||||
size_t name_len = strlen(name);
|
||||
|
||||
if (type == VY_DIR_ELEMENT_TYPE_FILE) {
|
||||
/* Don't parse .as files */
|
||||
/* Skip files we don't want to process */
|
||||
if (name_len >= 3) {
|
||||
if (memcmp(&name[name_len - 3], ".as", 3) == 0) {
|
||||
/* Skip */
|
||||
if (memcmp(&name[name_len - 3], ".as", 3) == 0)
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
if (name_len >= 4) {
|
||||
if (memcmp(&name[name_len - 4], ".pkg", 4) == 0)
|
||||
return VY_SUCCESS;
|
||||
else if (memcmp(&name[name_len - 4], ".bin", 4) == 0)
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
/* Check if we know that file */
|
||||
data->path_scratch[data->path_end] = '/';
|
||||
memcpy(data->path_scratch + data->path_end + 1, name, name_len);
|
||||
data->path_scratch[data->path_end + 1 + name_len] = '\0';
|
||||
if (data->path_end > 0) {
|
||||
data->path_scratch[data->path_end] = '/';
|
||||
memcpy(data->path_scratch + data->path_end + 1, name, name_len);
|
||||
data->path_scratch[data->path_end + 1 + name_len] = '\0';
|
||||
} else {
|
||||
memcpy(data->path_scratch, name, name_len);
|
||||
data->path_scratch[name_len] = '\0';
|
||||
}
|
||||
|
||||
vy_file_id fid = vyAddFile(data->path_scratch);
|
||||
if (vyLookupUID(fid) != VY_INVALID_UID) {
|
||||
vy_assetmeta meta;
|
||||
vy_assetmeta meta = {0};
|
||||
if (!vyGetAssetMeta(fid, &meta) || (meta.last_changed >= meta.compiled_ts)) {
|
||||
/* The file (may have) changed */
|
||||
vy_result res = vyAddFileToProcessingQueue(fid, 0);
|
||||
vy_result res = vyAddFileToProcessingQueue(fid, meta.processing_flags);
|
||||
if (res != VY_SUCCESS)
|
||||
return res;
|
||||
}
|
||||
else {
|
||||
/* The file is unchanged, we just need to add it to the output again. */
|
||||
vyLog("ASSETC", "File %s is unchanged.", data->path_scratch);
|
||||
|
||||
vy_asset_settings settings = {0};
|
||||
if (vyLoadAssetSettings(data->path_scratch, &settings) != VY_SUCCESS) {
|
||||
vyLog("ASSETC", "Failed to load settings for %s", data->path_scratch);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
/* We need to load the processed data and add it to the package */
|
||||
vy_uid uid = vyLookupUID(fid);
|
||||
const vy_uid_data *asset_data = vyGetUIDData(uid);
|
||||
if (!data) {
|
||||
vyLog("ASSETC", "No data available for asset %s (%u)", data->path_scratch, uid);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
/* Process it */
|
||||
vy_result res = vyAddFileToProcessingQueue(fid, 0);
|
||||
vy_asset_settings settings = {0};
|
||||
if (vyLoadAssetSettings(data->path_scratch, &settings) != VY_SUCCESS) {
|
||||
vyLog("ASSETC", "Failed to load settings for %s", data->path_scratch);
|
||||
}
|
||||
|
||||
vy_result res = vyAddFileToProcessingQueue(fid, settings.processing_flags);
|
||||
if (res != VY_SUCCESS)
|
||||
return res;
|
||||
}
|
||||
@ -54,10 +89,11 @@ static vy_result DirectoryHandler(const char *name, vyIterateDirElementType type
|
||||
return VY_SUCCESS;
|
||||
|
||||
unsigned int path_end_before = data->path_end;
|
||||
data->path_scratch[data->path_end++] = '/';
|
||||
if (data->path_end > 0)
|
||||
data->path_scratch[data->path_end++] = '/';
|
||||
memcpy(data->path_scratch + data->path_end, name, name_len);
|
||||
data->path_scratch[data->path_end + name_len] = '\0';
|
||||
data->path_end += name_len;
|
||||
data->path_end += (unsigned int)name_len;
|
||||
|
||||
vy_result res = vyIterateDirectory(data->path_scratch, user, DirectoryHandler);
|
||||
if (res != VY_SUCCESS)
|
||||
@ -72,7 +108,7 @@ void vyDiscoverAssets(void) {
|
||||
vy_discovery_data data;
|
||||
data.path_scratch[0] = '.';
|
||||
data.path_scratch[1] = '\0';
|
||||
data.path_end = 1;
|
||||
data.path_end = 0;
|
||||
|
||||
vyIterateDirectory(".", &data, DirectoryHandler);
|
||||
}
|
||||
@ -1,16 +1,25 @@
|
||||
#include "packages.h"
|
||||
#include "processing.h"
|
||||
|
||||
#include "runtime/threading.h"
|
||||
#include "runtime/assets.h"
|
||||
#include "runtime/file_tab.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "xxhash/xxhash.h"
|
||||
#include "lz4/lz4.h"
|
||||
|
||||
typedef struct {
|
||||
vy_uid uid;
|
||||
void *buffer;
|
||||
size_t buffer_size;
|
||||
bool needs_compression;
|
||||
} vy_package_entry;
|
||||
|
||||
typedef struct {
|
||||
@ -62,7 +71,11 @@ void vyInitPackages(void) {
|
||||
vyAddPackageFile((vy_text_span){.start = "default.pkg", .length = 12});
|
||||
}
|
||||
|
||||
void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size) {
|
||||
static void AddAssetToPackageImpl(unsigned int package,
|
||||
vy_uid uid,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool needs_compression) {
|
||||
vyLockMutex(_guard);
|
||||
if (package >= _package_count) {
|
||||
vyReportError("ASSETC", "Trying to add an asset to a non-existing package.");
|
||||
@ -71,6 +84,14 @@ void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t
|
||||
}
|
||||
|
||||
vy_package *pkg = &_packages[package];
|
||||
|
||||
for (unsigned int i = 0; i < pkg->num_entries; ++i) {
|
||||
if (pkg->entries[i].uid == uid) {
|
||||
vyUnlockMutex(_guard);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (pkg->num_entries == pkg->entry_capacity) {
|
||||
unsigned int new_cap = (pkg->entry_capacity > 0) ? 2 * pkg->entry_capacity : 256;
|
||||
vy_package_entry *n = realloc(pkg->entries, new_cap * sizeof(vy_package_entry));
|
||||
@ -81,10 +102,134 @@ void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t
|
||||
pkg->entry_capacity = new_cap;
|
||||
pkg->entries = n;
|
||||
}
|
||||
pkg->entries[pkg->num_entries].buffer = buffer;
|
||||
pkg->entries[pkg->num_entries].buffer_size = size;
|
||||
pkg->entries[pkg->num_entries].uid = uid;
|
||||
pkg->entries[pkg->num_entries].buffer = buffer;
|
||||
pkg->entries[pkg->num_entries].buffer_size = size;
|
||||
pkg->entries[pkg->num_entries].uid = uid;
|
||||
pkg->entries[pkg->num_entries].needs_compression = needs_compression;
|
||||
++pkg->num_entries;
|
||||
|
||||
|
||||
vyUnlockMutex(_guard);
|
||||
}
|
||||
|
||||
void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size) {
|
||||
AddAssetToPackageImpl(package, uid, buffer, size, true);
|
||||
}
|
||||
|
||||
void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size) {
|
||||
AddAssetToPackageImpl(package, uid, buffer, size, false);
|
||||
}
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef struct {
|
||||
XXH64_canonical_t checksum;
|
||||
uint32_t decompressed_size;
|
||||
} vy_package_asset_header;
|
||||
#pragma pack(pop)
|
||||
|
||||
static vy_result SavePackage(vy_package *pkg) {
|
||||
if (pkg->num_entries == 0) {
|
||||
vyLog("ASSETC", "Package %s has no entries.", pkg->name);
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
|
||||
int current_buffer_size = 0;
|
||||
void *buffer = NULL;
|
||||
|
||||
size_t offset_in_file = 0;
|
||||
|
||||
vy_file_id package_fid = vyAddFile(pkg->name);
|
||||
|
||||
FILE *f = fopen(pkg->name, "wb");
|
||||
if (!f) {
|
||||
vyReportError("ASSETC", "Failed to open %s for writing.", pkg->name);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
for (unsigned int i = 0; i < pkg->num_entries; ++i) {
|
||||
|
||||
if (pkg->entries[i].needs_compression) {
|
||||
int required_size = LZ4_compressBound((int)pkg->entries[i].buffer_size);
|
||||
if (required_size > current_buffer_size) {
|
||||
void *tmp = realloc(buffer, (size_t)required_size);
|
||||
if (!tmp)
|
||||
return VY_UNKNOWN_ERROR;
|
||||
buffer = tmp;
|
||||
current_buffer_size = required_size;
|
||||
}
|
||||
int compressed_bytes = LZ4_compress_default(pkg->entries[i].buffer,
|
||||
buffer,
|
||||
(int)pkg->entries[i].buffer_size,
|
||||
current_buffer_size);
|
||||
if (compressed_bytes == 0) {
|
||||
free(buffer);
|
||||
vyReportError("ASSETC", "Failed to compress entry %u of package %s", i, pkg->name);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
vy_package_asset_header header;
|
||||
XXH64_hash_t checksum = XXH3_64bits_withSeed(buffer, (size_t)compressed_bytes, 0);
|
||||
XXH64_canonicalFromHash(&header.checksum, checksum);
|
||||
header.decompressed_size = (uint32_t)pkg->entries[i].buffer_size;
|
||||
|
||||
if (fwrite(&header, sizeof(header), 1, f) != 1) {
|
||||
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
|
||||
free(buffer);
|
||||
fclose(f);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
if (fwrite(buffer, compressed_bytes, 1, f) != 1) {
|
||||
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
|
||||
free(buffer);
|
||||
fclose(f);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
vyAddUIDTabEntry(package_fid,
|
||||
pkg->entries[i].uid,
|
||||
offset_in_file,
|
||||
(size_t)compressed_bytes + sizeof(header));
|
||||
|
||||
offset_in_file += sizeof(header) + (size_t)compressed_bytes;
|
||||
} else {
|
||||
vy_package_asset_header header;
|
||||
XXH64_hash_t checksum = XXH3_64bits_withSeed(pkg->entries[i].buffer, pkg->entries[i].buffer_size, 0);
|
||||
XXH64_canonicalFromHash(&header.checksum, checksum);
|
||||
header.decompressed_size = (uint32_t)pkg->entries[i].buffer_size;
|
||||
|
||||
if (fwrite(&header, sizeof(header), 1, f) != 1) {
|
||||
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
|
||||
free(buffer);
|
||||
fclose(f);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
if (fwrite(pkg->entries[i].buffer, pkg->entries[i].buffer_size, 1, f) != 1) {
|
||||
vyReportError("ASSETC", "Failed to write to %s", pkg->name);
|
||||
free(buffer);
|
||||
fclose(f);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
vyAddUIDTabEntry(package_fid,
|
||||
pkg->entries[i].uid,
|
||||
offset_in_file,
|
||||
pkg->entries[i].buffer_size + sizeof(header));
|
||||
|
||||
offset_in_file += sizeof(header) + pkg->entries[i].buffer_size;
|
||||
}
|
||||
}
|
||||
|
||||
free(buffer);
|
||||
fclose(f);
|
||||
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
|
||||
vy_result vySavePackages(void) {
|
||||
assert(vyIsMainThread());
|
||||
|
||||
for (unsigned int i = 0; i < _package_count; ++i) {
|
||||
vy_result res = SavePackage(&_packages[i]);
|
||||
if (res != VY_SUCCESS)
|
||||
return res;
|
||||
}
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
@ -9,5 +9,8 @@ void vyInitPackages(void);
|
||||
unsigned int vyAddPackageFile(vy_text_span name);
|
||||
|
||||
void vyAddAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size);
|
||||
void vyAddCompressedAssetToPackage(unsigned int package, vy_uid uid, void *buffer, size_t size);
|
||||
|
||||
vy_result vySavePackages(void);
|
||||
|
||||
#endif
|
||||
|
||||
@ -19,6 +19,7 @@
|
||||
#include "processing.h"
|
||||
#include "utils.h"
|
||||
#include "processing_flags.h"
|
||||
#include "dependency_tracking.h"
|
||||
|
||||
typedef struct {
|
||||
vy_attribute_binding *uniform_bindings;
|
||||
@ -409,6 +410,14 @@ vy_result vyProcessPipelineFile(vy_file_id file,
|
||||
output->data = out_buffer;
|
||||
output->size = outbuffer_size;
|
||||
output->asset_uid = vyCalculateUID(vyGetFilePath(file));
|
||||
|
||||
/* Store dependencies to shaders */
|
||||
if (info->vertex_shader != VY_INVALID_UID)
|
||||
result = vyAddAssetDependency(output->asset_uid, info->vertex_shader);
|
||||
if (info->fragment_shader != VY_INVALID_UID)
|
||||
result = vyAddAssetDependency(output->asset_uid, info->fragment_shader);
|
||||
if (info->compute_shader != VY_INVALID_UID)
|
||||
result = vyAddAssetDependency(output->asset_uid, info->compute_shader);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -38,6 +38,8 @@ vy_result vyStartProcessing(void);
|
||||
void vyStopProcessing(void);
|
||||
|
||||
vy_uid vyCalculateUID(const char *name);
|
||||
vy_result vyWriteUIDTab(void);
|
||||
vy_result vyAddUIDTabEntry(vy_file_id package_fid, vy_uid uid, uint64_t offset, uint64_t size);
|
||||
|
||||
void vyWaitUntilProcessingIsFinished(void);
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include "description_parser.h"
|
||||
#include "packages.h"
|
||||
#include "assetmeta.h"
|
||||
#include "assetsettings.h"
|
||||
|
||||
#include "runtime/aio.h"
|
||||
#include "runtime/buffer_manager.h"
|
||||
@ -50,41 +51,6 @@ static vy_thread *_processing_threads[MAX_PROCESSING_THREADS];
|
||||
|
||||
static unsigned int _processing_thread_count = 0;
|
||||
|
||||
typedef struct {
|
||||
unsigned int package;
|
||||
} vy_asset_settings;
|
||||
|
||||
static vy_result ParseAssetSettings(const char *text,
|
||||
size_t length,
|
||||
const char *file_path,
|
||||
vy_asset_settings *settings) {
|
||||
unsigned int root_list;
|
||||
vy_parse_state state;
|
||||
vy_result res = vyParseDescription(text, length, file_path, &root_list, &state);
|
||||
if (res != VY_SUCCESS) {
|
||||
vyReportError("ASSETC", "Failed to parse asset settings: %s", file_path);
|
||||
return res;
|
||||
}
|
||||
|
||||
settings->package = 0;
|
||||
|
||||
const vy_parsed_stmt *package_stmt = vyFindStatement(&state, root_list, "package");
|
||||
if (package_stmt) {
|
||||
if (package_stmt->form != VY_STMT_FORM_VALUE) {
|
||||
vyReportError("ASSETC",
|
||||
"Expected a package name as the value of 'package' in %s.",
|
||||
file_path);
|
||||
res = VY_UNKNOWN_ERROR;
|
||||
goto out;
|
||||
}
|
||||
settings->package = vyAddPackageFile(package_stmt->value);
|
||||
}
|
||||
|
||||
out:
|
||||
vyReleaseParseState(&state);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
static vy_result vyAddFileToProcessingQueueImpl(vy_file_processing_queue *queue,
|
||||
vy_file_id file,
|
||||
@ -166,71 +132,22 @@ static vy_result ProcessLoadedFile(vy_file_processing_queue_entry entry, void *b
|
||||
/* Load the corresponding .as file.
|
||||
* TODO: Using malloc here is probably relatively slow.
|
||||
*/
|
||||
vy_result res;
|
||||
|
||||
vy_asset_settings settings;
|
||||
{
|
||||
char *as_path = malloc(path_len + 3);
|
||||
if (!as_path) {
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
memcpy(as_path, path, path_len);
|
||||
strcpy(&as_path[path_len - ext_len], ".as");
|
||||
size_t as_size = vyGetFileSize(as_path);
|
||||
if (as_size == 0) {
|
||||
vyReportError("ASSETC", "Failed to retrieve size of setting file %s", as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
|
||||
void *as_buffer = vyAllocBuffer(as_size);
|
||||
|
||||
vy_load_batch as_load;
|
||||
as_load.loads[0].file = vyAddFile(as_path);
|
||||
as_load.loads[0].num_bytes = as_size;
|
||||
as_load.loads[0].dest = as_buffer;
|
||||
if (!as_load.loads[0].dest) {
|
||||
vyReportError("ASSETC",
|
||||
"Failed to allocate buffer for setting file %s",
|
||||
as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
as_load.loads[0].offset = 0;
|
||||
as_load.num_loads = 1;
|
||||
|
||||
vy_aio_handle as_handle;
|
||||
if (vySubmitLoadBatch(&as_load, &as_handle) != VY_SUCCESS) {
|
||||
vyReportError("ASSETC",
|
||||
"Failed to submit load of setting file %s",
|
||||
as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
if (vyWaitForAIOCompletion(as_handle) != VY_AIO_STATE_FINISHED) {
|
||||
vyReportError("ASSETC", "Failed to load setting file %s", as_path);
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
vyReleaseAIO(as_handle);
|
||||
|
||||
if (ParseAssetSettings(as_buffer, as_size, as_path, &settings) != VY_SUCCESS) {
|
||||
free(as_path);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
free(as_path);
|
||||
if ((res = vyLoadAssetSettings(path, &settings)) != VY_SUCCESS) {
|
||||
return res;
|
||||
}
|
||||
|
||||
/* Process the asset */
|
||||
vy_processor_output out;
|
||||
vy_result res = _processor_fns[i](entry.fid, buffer, size, entry.flags, &out);
|
||||
res = _processor_fns[i](entry.fid, buffer, size, entry.flags, &out);
|
||||
if (res == VY_SUCCESS) {
|
||||
/* Add the output to the appropriate package file */
|
||||
vy_assetmeta meta;
|
||||
meta.compiled_ts = vyGetCurrentTimestamp();
|
||||
meta.last_changed = vyGetFileModificationTimestamp(entry.fid);
|
||||
meta.processing_flags = entry.flags;
|
||||
vyAddUIDMapping(entry.fid, out.asset_uid, &meta);
|
||||
|
||||
vyAddAssetToPackage(settings.package, out.asset_uid, out.data, out.size);
|
||||
@ -447,7 +364,7 @@ void vyWaitUntilProcessingIsFinished(void) {
|
||||
unsigned int done_counter = 0;
|
||||
while (done_counter < 3) {
|
||||
#ifdef _WIN32
|
||||
Sleep(1000);
|
||||
Sleep(200);
|
||||
#else
|
||||
sleep(1);
|
||||
#endif
|
||||
|
||||
@ -111,6 +111,7 @@ vy_result vyProcessShaderFile(vy_file_id file,
|
||||
}
|
||||
const uint32_t *spv_bytes = (uint32_t *)shaderc_result_get_bytes(result);
|
||||
memcpy(output->data, spv_bytes, output_size);
|
||||
output->size = output_size;
|
||||
|
||||
shaderc_result_release(result);
|
||||
shaderc_compile_options_release(options);
|
||||
|
||||
@ -1,32 +1,69 @@
|
||||
#include "processing.h"
|
||||
#include "utils.h"
|
||||
|
||||
#define VY_DEFINE_UIDTAB_FILE_STRUCTURES
|
||||
#include "runtime/threading.h"
|
||||
#include "runtime/aio.h"
|
||||
#include "runtime/buffer_manager.h"
|
||||
#include "runtime/uidtab.h"
|
||||
|
||||
#include <xxhash/xxhash.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <Windows.h>
|
||||
#endif
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef struct {
|
||||
vy_file_id file;
|
||||
uint64_t offset;
|
||||
uint64_t size;
|
||||
uint64_t last_changed;
|
||||
} vy_uidtab_entry;
|
||||
#pragma pack(pop)
|
||||
|
||||
static vy_uidtab_entry *_entries;
|
||||
static size_t _entry_capacity;
|
||||
static size_t _entry_count;
|
||||
|
||||
vy_uid vyCalculateUID(const char *name) {
|
||||
assert(sizeof(XXH32_hash_t) == sizeof(vy_uid));
|
||||
|
||||
size_t len = strlen(name);
|
||||
return (vy_uid)XXH32(name, len, 0);
|
||||
}
|
||||
|
||||
vy_result vyAddUIDTabEntry(vy_file_id package_fid, vy_uid uid, uint64_t offset, uint64_t size) {
|
||||
assert(vyIsMainThread());
|
||||
|
||||
if (_entry_count == _entry_capacity) {
|
||||
size_t new_cap = (_entry_capacity > 0) ? _entry_capacity * 2 : 256;
|
||||
void *t = realloc(_entries, sizeof(vy_uidtab_entry) * new_cap);
|
||||
if (!t)
|
||||
return VY_UNKNOWN_ERROR;
|
||||
_entry_capacity = new_cap;
|
||||
_entries = t;
|
||||
}
|
||||
_entries[_entry_count].file = package_fid;
|
||||
_entries[_entry_count].offset = offset;
|
||||
_entries[_entry_count].size = size;
|
||||
_entries[_entry_count].uid = uid;
|
||||
_entry_count++;
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
|
||||
vy_result vyWriteUIDTab(void) {
|
||||
vy_uidtab_header header;
|
||||
XXH64_hash_t checksum = XXH3_64bits(_entries, sizeof(vy_uidtab_entry) * _entry_count);
|
||||
XXH64_canonicalFromHash(&header.checksum, checksum);
|
||||
header.num_entries = (uint32_t)_entry_count;
|
||||
|
||||
FILE *f = fopen("uidtab.bin", "wb");
|
||||
if (!f) {
|
||||
vyReportError("ASSETC", "Failed to open 'uidtab.bin' for writing.");
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
if (fwrite(&header, sizeof(header), 1, f) != 1) {
|
||||
vyReportError("ASSETC", "Failed to write header to 'uidtab.bin'");
|
||||
fclose(f);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
if (fwrite(_entries, sizeof(vy_uidtab_entry), _entry_count, f) != _entry_count) {
|
||||
vyReportError("ASSETC", "Failed to write entries to 'uidtab.bin'");
|
||||
fclose(f);
|
||||
return VY_UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
return VY_SUCCESS;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user