Other cache changes

Added index to the cache name
Moved the hash out to the torque namespace
DX now outputs the compiled shader

Engine/source/core/util/hashFunction.cpp

@@ -31,6 +31,8 @@
 
 #include "core/util/hashFunction.h"
 
+#include "core/util/endian.h"
+
 namespace Torque
 {
 
@@ -268,4 +270,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval )
   return c;
 }
 
+// Generate a single 64bit hash from the input string.
+//
+// Don't get paranoid!  This has 1 in 18446744073709551616
+// chance for collision... it won't happen in this lifetime.
+//
+String getStringHash64(const String& in)
+{
+   String cacheKey = in;
+   cacheKey.replace("\n", " ");
+   U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0);
+   hash = convertHostToLEndian(hash);
+   U32 high = (U32)(hash >> 32);
+   U32 low = (U32)(hash & 0x00000000FFFFFFFF);
+   cacheKey = String::ToString("%x%x", high, low);
+   return cacheKey;
+}
+
 } // namespace

Engine/source/core/util/hashFunction.h

@@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval);
 
 extern U64 hash64(const U8 *k, U32 length, U64 initval);
 
+extern String getStringHash64(const String& in);
+
 }
 
 #endif // _HASHFUNCTION_H_

Engine/source/gfx/D3D11/gfxD3D11Shader.cpp

@@ -681,6 +681,24 @@ bool GFXD3D11Shader::_init()
    return true;
 }
 
+static String buildMacroHash(const D3D_SHADER_MACRO* defines)
+{
+   String combined;
+
+   if (!defines)
+      return "";
+
+   for (const D3D_SHADER_MACRO* m = defines; m->Name != nullptr; ++m)
+   {
+      combined += m->Name;
+      combined += "=";
+      combined += m->Definition ? m->Definition : "";
+      combined += ";";
+   }
+
+   return Torque::getStringHash64(combined);
+}
+
 bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
                                     GFXShaderStage shaderStage,
                                     const D3D_SHADER_MACRO *defines)
@@ -706,8 +724,40 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
    Con::printf( "Compiling Shader: '%s'", filePath.getFullPath().c_str() );
 #endif
 
+   String macroHash = buildMacroHash(defines);
+
+   Torque::Path cachePath = filePath;
+   cachePath.setExtension("tso");
+   cachePath.setFileName(cachePath.getFileName() + "_" + macroHash);
+
+   if (Torque::FS::IsFile(cachePath))
+   {
+      Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(filePath);
+      Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath);
+
+      if (rawFile != NULL && cachedFile != NULL)
+      {
+         if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime())
+         {
+
+            FileStream fs;
+            if (fs.open(cachePath, Torque::FS::File::Read))
+            {
+               U32 size = fs.getStreamSize();
+
+               D3DCreateBlob(size, &code);
+               fs.read(size, code->GetBufferPointer());
+
+               res = 1;
+            }
+         }
+      }
+   }
+
+   bool loadedFromCache = (code != NULL);
+
    // Is it an HLSL shader?
-   if(filePath.getExtension().equal("hlsl", String::NoCase))
+   if(filePath.getExtension().equal("hlsl", String::NoCase) && !loadedFromCache)
    {
       // Set this so that the D3DInclude::Open will have this
       // information for relative paths.
@@ -788,6 +838,20 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
 
    AssertISV(SUCCEEDED(res), "Unable to compile shader!");
 
+   // succeeded write out a cache
+   if (!loadedFromCache)
+   {
+      if (SUCCEEDED(res) && code)
+      {
+         // Save cache
+         FileStream out;
+         if (out.open(cachePath, Torque::FS::File::Write))
+         {
+            out.write(code->GetBufferSize(), code->GetBufferPointer());
+         }
+      }
+   }
+
    if(code != NULL)
    {
       switch (shaderStage)

Engine/source/materials/shaderData.cpp

@@ -219,6 +219,8 @@ GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros)
    String cacheKey;
    GFXShaderMacro::stringize( macros, &cacheKey );
 
+   cacheKey = Torque::getStringHash64(cacheKey);
+
    // Lookup the shader for this instance.
    ShaderCache::Iterator iter = mShaders.find( cacheKey );
    if ( iter != mShaders.end() )

Engine/source/shaderGen/shaderGen.cpp

@@ -63,23 +63,6 @@ static const char* _getStagePostfix(GFXShaderStage stage)
    return "_U"; // Unknown
 }
 
-// Generate a single 64bit hash from the input string.
-//
-// Don't get paranoid!  This has 1 in 18446744073709551616
-// chance for collision... it won't happen in this lifetime.
-//
-String getHashForName(const String& in)
-{
-   String cacheKey = in;
-   cacheKey.replace("\n", " ");
-   U64 hash = Torque::hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0);
-   hash = convertHostToLEndian(hash);
-   U32 high = (U32)(hash >> 32);
-   U32 low = (U32)(hash & 0x00000000FFFFFFFF);
-   cacheKey = String::ToString("%x%x", high, low);
-   return cacheKey;
-}
-
 MODULE_BEGIN( ShaderGen )
 
    MODULE_INIT_BEFORE( GFX )
@@ -244,11 +227,11 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData,
          const FeatureType& type = features.getAt(i);
          if (stage & FEATUREMGR->getByType(type)->getShaderStages())
          {
-            stageName += type.getName().c_str();
+            stageName += type.getName() + "," + String::ToString(i);
          }
       }
 
-      stageName = getHashForName(stageName);
+      stageName = Torque::getStringHash64(stageName);
       stageName += postfix;
 
       FileCacheSet::iterator file = mFileCache.find(stageName);
@@ -573,7 +556,7 @@ GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GF
    // and vertex format combination ( and macros ).
    String shaderDescription = vertexFormat->getDescription() + features.getDescription();
 
-   String cacheKey = getHashForName(shaderDescription);
+   String cacheKey = Torque::getStringHash64(shaderDescription);
 
    ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey);
    if (dat != mProcShaderData.end())