From d22e3ebfc98d0a408a63d50f6265c99a384334e9 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Mon, 8 Dec 2025 11:23:09 +0000 Subject: [PATCH] Other cache changes Added index to the cache name Moved the hash out to the torque namespace DX now outputs the compiled shader --- Engine/source/core/util/hashFunction.cpp | 19 +++++++ Engine/source/core/util/hashFunction.h | 2 + Engine/source/gfx/D3D11/gfxD3D11Shader.cpp | 66 +++++++++++++++++++++- Engine/source/materials/shaderData.cpp | 2 + Engine/source/shaderGen/shaderGen.cpp | 23 +------- 5 files changed, 91 insertions(+), 21 deletions(-) diff --git a/Engine/source/core/util/hashFunction.cpp b/Engine/source/core/util/hashFunction.cpp index 69206dc30..dbc30e287 100644 --- a/Engine/source/core/util/hashFunction.cpp +++ b/Engine/source/core/util/hashFunction.cpp @@ -31,6 +31,8 @@ #include "core/util/hashFunction.h" +#include "core/util/endian.h" + namespace Torque { @@ -268,4 +270,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval ) return c; } +// Generate a single 64bit hash from the input string. +// +// Don't get paranoid! This has 1 in 18446744073709551616 +// chance for collision... it won't happen in this lifetime. +// +String getStringHash64(const String& in) +{ + String cacheKey = in; + cacheKey.replace("\n", " "); + U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0); + hash = convertHostToLEndian(hash); + U32 high = (U32)(hash >> 32); + U32 low = (U32)(hash & 0x00000000FFFFFFFF); + cacheKey = String::ToString("%x%x", high, low); + return cacheKey; +} + } // namespace diff --git a/Engine/source/core/util/hashFunction.h b/Engine/source/core/util/hashFunction.h index 02ecd019b..f3a57cc9a 100644 --- a/Engine/source/core/util/hashFunction.h +++ b/Engine/source/core/util/hashFunction.h @@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval); extern U64 hash64(const U8 *k, U32 length, U64 initval); +extern String getStringHash64(const String& in); + } #endif // _HASHFUNCTION_H_ diff --git a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp index 6259be330..4a919cffd 100644 --- a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp +++ b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp @@ -681,6 +681,24 @@ bool GFXD3D11Shader::_init() return true; } +static String buildMacroHash(const D3D_SHADER_MACRO* defines) +{ + String combined; + + if (!defines) + return ""; + + for (const D3D_SHADER_MACRO* m = defines; m->Name != nullptr; ++m) + { + combined += m->Name; + combined += "="; + combined += m->Definition ? m->Definition : ""; + combined += ";"; + } + + return Torque::getStringHash64(combined); +} + bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, GFXShaderStage shaderStage, const D3D_SHADER_MACRO *defines) @@ -706,8 +724,40 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, Con::printf( "Compiling Shader: '%s'", filePath.getFullPath().c_str() ); #endif + String macroHash = buildMacroHash(defines); + + Torque::Path cachePath = filePath; + cachePath.setExtension("tso"); + cachePath.setFileName(cachePath.getFileName() + "_" + macroHash); + + if (Torque::FS::IsFile(cachePath)) + { + Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(filePath); + Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath); + + if (rawFile != NULL && cachedFile != NULL) + { + if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime()) + { + + FileStream fs; + if (fs.open(cachePath, Torque::FS::File::Read)) + { + U32 size = fs.getStreamSize(); + + D3DCreateBlob(size, &code); + fs.read(size, code->GetBufferPointer()); + + res = 1; + } + } + } + } + + bool loadedFromCache = (code != NULL); + // Is it an HLSL shader? - if(filePath.getExtension().equal("hlsl", String::NoCase)) + if(filePath.getExtension().equal("hlsl", String::NoCase) && !loadedFromCache) { // Set this so that the D3DInclude::Open will have this // information for relative paths. @@ -788,6 +838,20 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, AssertISV(SUCCEEDED(res), "Unable to compile shader!"); + // succeeded write out a cache + if (!loadedFromCache) + { + if (SUCCEEDED(res) && code) + { + // Save cache + FileStream out; + if (out.open(cachePath, Torque::FS::File::Write)) + { + out.write(code->GetBufferSize(), code->GetBufferPointer()); + } + } + } + if(code != NULL) { switch (shaderStage) diff --git a/Engine/source/materials/shaderData.cpp b/Engine/source/materials/shaderData.cpp index 253838573..8e987e4dc 100644 --- a/Engine/source/materials/shaderData.cpp +++ b/Engine/source/materials/shaderData.cpp @@ -219,6 +219,8 @@ GFXShader* ShaderData::getShader( const Vector ¯os) String cacheKey; GFXShaderMacro::stringize( macros, &cacheKey ); + cacheKey = Torque::getStringHash64(cacheKey); + // Lookup the shader for this instance. ShaderCache::Iterator iter = mShaders.find( cacheKey ); if ( iter != mShaders.end() ) diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index b212d5b33..d9f69e523 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -63,23 +63,6 @@ static const char* _getStagePostfix(GFXShaderStage stage) return "_U"; // Unknown } -// Generate a single 64bit hash from the input string. -// -// Don't get paranoid! This has 1 in 18446744073709551616 -// chance for collision... it won't happen in this lifetime. -// -String getHashForName(const String& in) -{ - String cacheKey = in; - cacheKey.replace("\n", " "); - U64 hash = Torque::hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0); - hash = convertHostToLEndian(hash); - U32 high = (U32)(hash >> 32); - U32 low = (U32)(hash & 0x00000000FFFFFFFF); - cacheKey = String::ToString("%x%x", high, low); - return cacheKey; -} - MODULE_BEGIN( ShaderGen ) MODULE_INIT_BEFORE( GFX ) @@ -244,11 +227,11 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, const FeatureType& type = features.getAt(i); if (stage & FEATUREMGR->getByType(type)->getShaderStages()) { - stageName += type.getName().c_str(); + stageName += type.getName() + "," + String::ToString(i); } } - stageName = getHashForName(stageName); + stageName = Torque::getStringHash64(stageName); stageName += postfix; FileCacheSet::iterator file = mFileCache.find(stageName); @@ -573,7 +556,7 @@ GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GF // and vertex format combination ( and macros ). String shaderDescription = vertexFormat->getDescription() + features.getDescription(); - String cacheKey = getHashForName(shaderDescription); + String cacheKey = Torque::getStringHash64(shaderDescription); ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey); if (dat != mProcShaderData.end())