Skip to content

Commit 99f9386

Browse files
alichraghialichraghi
committed
not working yet
Signed-off-by: Ali Cheraghi <alichraghi@proton.me>
1 parent 8c70f06 commit 99f9386

File tree

4 files changed

+110
-48
lines changed

4 files changed

+110
-48
lines changed

‎include/nbl/asset/utils/IShaderCompiler.h‎

Lines changed: 11 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
239239
return true;
240240
}
241241

242+
std::string sourceIdentifier;
243+
242244
private:
243245
friend class SCompilerArgs;
244246
friend class SEntry;
@@ -262,7 +264,6 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
262264
// Sort them so equality and hashing are well defined
263265
std::sort(extraDefines.begin(), extraDefines.end(), [](const SMacroDefinition& lhs, const SMacroDefinition& rhs) {return lhs.identifier < rhs.identifier; });
264266
};
265-
std::string sourceIdentifier;
266267
std::vector<SMacroDefinition> extraDefines;
267268
};
268269
// TODO: SPreprocessorArgs could just be folded into `SCompilerArgs` to have less classes and operators
@@ -282,6 +283,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
282283
return retVal;
283284
}
284285

286+
IShader::E_SHADER_STAGE stage;
287+
SPreprocessorArgs preprocessorArgs;
288+
285289
private:
286290
friend class SEntry;
287291
friend void to_json(nlohmann::json&, const SCompilerArgs&);
@@ -306,11 +310,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
306310
}
307311
}
308312

309-
IShader::E_SHADER_STAGE stage;
310313
E_SPIRV_VERSION targetSpirvVersion;
311314
std::vector<ISPIRVOptimizer::E_OPTIMIZER_PASS> optimizerPasses;
312315
core::bitflag<E_DEBUG_INFO_FLAGS> debugInfoFlags;
313-
SPreprocessorArgs preprocessorArgs;
314316
};
315317

316318
// The ordering is important here, the dependencies MUST be added to the array IN THE ORDER THE PREPROCESSOR INCLUDED THEM!
@@ -358,20 +360,23 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
358360
// Making the copy constructor deep-copy everything but the shader
359361
inline SEntry(const SEntry& other)
360362
: mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), lookupHash(other.lookupHash),
361-
dependencies(other.dependencies), cpuShader(other.cpuShader) {}
363+
dependencies(other.dependencies), spirv(other.spirv) {}
362364

363365
inline SEntry& operator=(SEntry& other) = delete;
364366
inline SEntry(SEntry&& other) = default;
365367
// Used for late initialization while looking up a cache, so as not to always initialize an entry even if caching was not requested
366368
inline SEntry& operator=(SEntry&& other) = default;
367369

370+
core::smart_refctd_ptr<ICPUShader> decodeShader() const;
371+
368372
// TODO: make some of these private
369373
std::string mainFileContents;
370374
SCompilerArgs compilerArgs;
371375
core::blake3_hash_t hash;
372376
size_t lookupHash;
373377
dependency_container_t dependencies;
374-
core::smart_refctd_ptr<asset::ICPUShader> cpuShader;
378+
core::smart_refctd_ptr<asset::ICPUBuffer> spirv;
379+
size_t uncompressedSize;
375380
};
376381

377382
inline void insert(SEntry&& entry)
@@ -429,42 +434,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
429434
NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const;
430435
};
431436

432-
inline core::smart_refctd_ptr<ICPUShader> compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const
433-
{
434-
CCache::SEntry entry;
435-
std::vector<CCache::SEntry::SPreprocessingDependency> dependencies;
436-
if (options.readCache || options.writeCache)
437-
entry = std::move(CCache::SEntry(code, options));
438-
439-
if (options.readCache)
440-
{
441-
auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder);
442-
if (found != options.readCache->m_container.end())
443-
{
444-
if (options.writeCache)
445-
{
446-
CCache::SEntry writeEntry = *found;
447-
options.writeCache->insert(std::move(writeEntry));
448-
}
449-
return found->cpuShader;
450-
}
451-
}
452-
453-
auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &dependencies : nullptr);
454-
// compute the SPIR-V shader content hash
455-
{
456-
auto backingBuffer = retVal->getContent();
457-
const_cast<ICPUBuffer*>(backingBuffer)->setContentHash(backingBuffer->computeContentHash());
458-
}
459-
460-
if (options.writeCache)
461-
{
462-
entry.dependencies = std::move(dependencies);
463-
entry.cpuShader = retVal;
464-
options.writeCache->insert(std::move(entry));
465-
}
466-
return retVal;
467-
}
437+
core::smart_refctd_ptr<ICPUShader> compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const;
468438

469439
inline core::smart_refctd_ptr<ICPUShader> compileToSPIRV(const char* code, const SCompilerOptions& options) const
470440
{

‎src/nbl/asset/utils/CHLSLCompiler.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
#include <sstream>
1919
#include <dxc/dxcapi.h>
2020

21+
#include "lzma/C/LzmaEnc.h"
22+
2123
using namespace nbl;
2224
using namespace nbl::asset;
2325
using Microsoft::WRL::ComPtr;
@@ -373,6 +375,9 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE
373375
return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags);
374376
}
375377

378+
static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); }
379+
static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); }
380+
376381
core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV_impl(const std::string_view code, const IShaderCompiler::SCompilerOptions& options, std::vector<CCache::SEntry::SPreprocessingDependency>* dependencies) const
377382
{
378383
auto hlslOptions = option_cast(options);

‎src/nbl/asset/utils/IShaderCompiler.cpp‎

Lines changed: 91 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
#include <regex>
1414
#include <iterator>
1515

16+
#include <lzma/C/LzmaEnc.h>
17+
#include <lzma/C/LzmaDec.h>
18+
1619
using namespace nbl;
1720
using namespace nbl::asset;
1821

@@ -22,6 +25,71 @@ IShaderCompiler::IShaderCompiler(core::smart_refctd_ptr<system::ISystem>&& syste
2225
m_defaultIncludeFinder = core::make_smart_refctd_ptr<CIncludeFinder>(core::smart_refctd_ptr(m_system));
2326
}
2427

28+
static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); }
29+
static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); }
30+
31+
inline core::smart_refctd_ptr<ICPUShader> nbl::asset::IShaderCompiler::compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const
32+
{
33+
CCache::SEntry entry;
34+
std::vector<CCache::SEntry::SPreprocessingDependency> dependencies;
35+
if (options.readCache || options.writeCache)
36+
entry = std::move(CCache::SEntry(code, options));
37+
38+
if (options.readCache)
39+
{
40+
auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder);
41+
if (found != options.readCache->m_container.end())
42+
{
43+
if (options.writeCache)
44+
{
45+
CCache::SEntry writeEntry = *found;
46+
options.writeCache->insert(std::move(writeEntry));
47+
}
48+
return found->decodeShader();
49+
}
50+
}
51+
52+
auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &dependencies : nullptr);
53+
// compute the SPIR-V shader content hash
54+
{
55+
auto backingBuffer = retVal->getContent();
56+
const_cast<ICPUBuffer*>(backingBuffer)->setContentHash(backingBuffer->computeContentHash());
57+
}
58+
59+
std::cout << "writeCache: " << options.writeCache << "\n";
60+
if (options.writeCache)
61+
{
62+
auto* spirvBuffer = retVal->getContent();
63+
size_t propsSize = LZMA_PROPS_SIZE;
64+
size_t destLen = spirvBuffer->getSize() + spirvBuffer->getSize() / 3 + 128;
65+
auto compressedSpirvBuffer = core::make_smart_refctd_ptr<ICPUBuffer>(propsSize + destLen);
66+
67+
CLzmaEncProps props;
68+
LzmaEncProps_Init(&props);
69+
props.dictSize = 1 << 16; // 64 KB
70+
props.writeEndMark = 1; // 0 or 1
71+
72+
ISzAlloc alloc = { SzAlloc, SzFree };
73+
int res = LzmaEncode(
74+
reinterpret_cast<unsigned char*>(compressedSpirvBuffer->getPointer()) + LZMA_PROPS_SIZE, &destLen,
75+
reinterpret_cast<const unsigned char*>(spirvBuffer->getPointer()), spirvBuffer->getSize(),
76+
&props, reinterpret_cast<unsigned char*>(compressedSpirvBuffer->getPointer()), &propsSize, props.writeEndMark,
77+
nullptr, &alloc, &alloc);
78+
79+
assert(propsSize == LZMA_PROPS_SIZE);
80+
assert(res == SZ_OK);
81+
82+
entry.dependencies = std::move(dependencies);
83+
entry.spirv = std::move(compressedSpirvBuffer);
84+
entry.uncompressedSize = spirvBuffer->getSize();
85+
86+
std::cout << "original: " << spirvBuffer->getSize() << ", compressed: " << compressedSpirvBuffer->getSize() << "\n";
87+
88+
options.writeCache->insert(std::move(entry));
89+
}
90+
return retVal;
91+
}
92+
2593
std::string IShaderCompiler::preprocessShader(
2694
system::IFile* sourcefile,
2795
IShader::E_SHADER_STAGE stage,
@@ -224,7 +292,7 @@ auto IShaderCompiler::CIncludeFinder::tryIncludeGenerators(const std::string& in
224292

225293
core::smart_refctd_ptr<asset::ICPUShader> IShaderCompiler::CCache::find(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const
226294
{
227-
return find_impl(mainFile, finder)->cpuShader;
295+
return find_impl(mainFile, finder)->decodeShader();
228296
}
229297

230298
IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_impl(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const
@@ -273,10 +341,10 @@ core::smart_refctd_ptr<ICPUBuffer> IShaderCompiler::CCache::serialize() const
273341
// We keep a copy of the offsets and the sizes of each shader. This is so that later on, when we add the shaders to the buffer after json creation
274342
// (where the params array has been moved) we don't have to read the json to get the offsets again
275343
offsets[i] = shaderBufferSize;
276-
sizes[i] = entry.cpuShader->getContent()->getSize();
344+
sizes[i] = entry.spirv->getSize();
277345

278346
// And add the params to the shader creation parameters array
279-
shaderCreationParams.emplace_back(entry.cpuShader->getStage(), entry.cpuShader->getContentType(), entry.cpuShader->getFilepathHint(), sizes[i], shaderBufferSize);
347+
shaderCreationParams.emplace_back(entry.compilerArgs.stage, IShader::E_CONTENT_TYPE::ECT_SPIRV, entry.compilerArgs.preprocessorArgs.sourceIdentifier.data(), sizes[i], shaderBufferSize);
280348
// Enlarge the shader buffer by the size of the current shader
281349
shaderBufferSize += sizes[i];
282350
i++;
@@ -300,7 +368,7 @@ core::smart_refctd_ptr<ICPUBuffer> IShaderCompiler::CCache::serialize() const
300368
// Loop over entries again, adding each one's shader to the buffer.
301369
i = 0u;
302370
for (auto& entry : m_container) {
303-
memcpy(retVal.data() + SHADER_BUFFER_SIZE_BYTES + offsets[i], entry.cpuShader->getContent()->getPointer(), sizes[i]);
371+
memcpy(retVal.data() + SHADER_BUFFER_SIZE_BYTES + offsets[i], entry.spirv->getPointer(), sizes[i]);
304372
i++;
305373
}
306374

@@ -343,13 +411,30 @@ core::smart_refctd_ptr<IShaderCompiler::CCache> IShaderCompiler::CCache::deseria
343411
// Create buffer to hold the code
344412
auto code = core::make_smart_refctd_ptr<ICPUBuffer>(shaderCreationParams[i].codeByteSize);
345413
// Copy the shader bytecode into the buffer
414+
346415
memcpy(code->getPointer(), serializedCache.data() + SHADER_BUFFER_SIZE_BYTES + shaderCreationParams[i].offset, shaderCreationParams[i].codeByteSize);
347416
code->setContentHash(code->computeContentHash());
348417
// Create the ICPUShader
349-
entries[i].cpuShader = core::make_smart_refctd_ptr<ICPUShader>(std::move(code), shaderCreationParams[i].stage, shaderCreationParams[i].contentType, std::move(shaderCreationParams[i].filepathHint));
418+
entries[i].spirv = std::move(code);
350419

351420
retVal->insert(std::move(entries[i]));
352421
}
353422

354423
return retVal;
355-
}
424+
}
425+
426+
core::smart_refctd_ptr<ICPUShader> nbl::asset::IShaderCompiler::CCache::SEntry::decodeShader() const
427+
{
428+
auto uncompressedBuf = core::make_smart_refctd_ptr<ICPUBuffer>(uncompressedSize);
429+
size_t dstSize = uncompressedBuf->getSize();
430+
size_t srcSize = spirv->getSize() - LZMA_PROPS_SIZE;
431+
ELzmaStatus status;
432+
ISzAlloc alloc = { SzAlloc, SzFree };
433+
SRes res = LzmaDecode(
434+
reinterpret_cast<unsigned char*>(uncompressedBuf->getPointer()), &dstSize,
435+
reinterpret_cast<const unsigned char*>(spirv->getPointer()) + LZMA_PROPS_SIZE, &srcSize,
436+
reinterpret_cast<const unsigned char*>(spirv->getPointer()), LZMA_PROPS_SIZE,
437+
LZMA_FINISH_ANY, &status, &alloc);
438+
assert(res == SZ_OK);
439+
return core::make_smart_refctd_ptr<asset::ICPUShader>(std::move(uncompressedBuf), compilerArgs.stage, IShader::E_CONTENT_TYPE::ECT_SPIRV, compilerArgs.preprocessorArgs.sourceIdentifier.data());
440+
}

‎src/nbl/asset/utils/shaderCompiler_serialization.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ inline void to_json(json& j, const SEntry& entry)
180180
{ "hash", entry.hash.data },
181181
{ "lookupHash", entry.lookupHash },
182182
{ "dependencies", entry.dependencies },
183+
{ "uncompressedSize", entry.uncompressedSize },
183184
};
184185
}
185186

@@ -190,7 +191,8 @@ inline void from_json(const json& j, SEntry& entry)
190191
j.at("hash").get_to(entry.hash.data);
191192
j.at("lookupHash").get_to(entry.lookupHash);
192193
j.at("dependencies").get_to(entry.dependencies);
193-
entry.cpuShader = nullptr;
194+
j.at("uncompressedSize").get_to(entry.uncompressedSize);
195+
entry.spirv = nullptr;
194196
}
195197

196198
}

0 commit comments

Comments
 (0)