Merge pull request #1760 from marauder2k9-torque/ShaderGen-produce-shaderdata

Shader Gen to produce ShaderData

Engine/source/core/stream/fileStream.cpp

@@ -22,23 +22,57 @@
 
 #include "platform/platform.h"
 #include "core/stream/fileStream.h"
-
+#include "platform/threads/threadPool.h"
 
 //-----------------------------------------------------------------------------
 // FileStream methods...
 //-----------------------------------------------------------------------------
 
-//-----------------------------------------------------------------------------
+struct FileCloseWorkItem : public ThreadPool::WorkItem
-FileStream::FileStream()
 {
+   Torque::FS::FileRef mFile;
+
+public:
+   FileCloseWorkItem(Torque::FS::FileRef file)
+      : mFile(file)
+   {
+   }
+
+protected:
+   void execute() override
+   {
+      if (!mFile)
+         return;
+
+      // When platforms free a file they
+      // remove the handle which causes the stall
+      // as they write metadata.
+      mFile->close();
+   }
+};
+
+void FileStream::dispatchAsyncClose()
+{
+   if (!mFile)
+      return;
+
+   FileCloseWorkItem* job = new FileCloseWorkItem(mFile);
+
+   ThreadPool::GLOBAL().queueWorkItem(job);
+}
+
+//-----------------------------------------------------------------------------
+FileStream::FileStream(AsyncMode flushMode)
+{
+   mAsyncMode = flushMode;
    dMemset(mBuffer, 0, sizeof(mBuffer));
    // initialize the file stream
    init();
 }
 
-FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode)
+FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode)
 {
-   FileStream  *newStream = new FileStream;
+   FileStream  *newStream = new FileStream(flushMode);
 
    bool success = newStream->open( inFileName, inMode );
 
@@ -193,7 +227,13 @@ void FileStream::close()
       // and close the file
       mFile->close();
 
-      AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
+      if (mAsyncMode == Background)
+         dispatchAsyncClose();
+      else
+      {
+         mFile->close();
+         AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
+      }
 
       mFile = NULL;
    }

Engine/source/core/stream/fileStream.h

@@ -39,15 +39,23 @@ public:
       BUFFER_INVALID = 0xffffffff   // file offsets must all be less than this
    };
 
+   enum AsyncMode
+   {
+      Blocking,      // current behavior
+      Background     // write-behind
+   };
+
    typedef char Ch;    //!< Character type. Only support char.
 public:
-   FileStream();                       // default constructor
+   AsyncMode mAsyncMode;
+   void dispatchAsyncClose();
+   FileStream(AsyncMode flushMode = Blocking);// default constructor
    virtual ~FileStream();              // destructor
 
    // This function will allocate a new FileStream and open it.
    //  If it fails to allocate or fails to open, it will return NULL.
    //  The caller is responsible for deleting the instance.
-   static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode);
+   static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode = Blocking);
 
    // mandatory methods from Stream base class...
    bool hasCapability(const Capability i_cap) const override;
@@ -64,6 +72,7 @@ public:
    //rjson compatibility
    bool Flush() { return flush(); }
    FileStream* clone() const override;
+   static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
 
 protected:
    // more mandatory methods from Stream base class...
@@ -73,7 +82,6 @@ protected:
    void init();
    bool fillBuffer(const U32 i_startPosition);
    void clearBuffer();
-   static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
    static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail);
    void setStatus();
 

Engine/source/core/util/hashFunction.cpp

@@ -30,6 +30,7 @@
 #include "platform/platform.h"
 
 #include "core/util/hashFunction.h"
+#include "core/util/endian.h"
 
 namespace Torque
 {
@@ -268,4 +269,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval )
   return c;
 }
 
+// Generate a single 64bit hash from the input string.
+//
+// Don't get paranoid!  This has 1 in 18446744073709551616
+// chance for collision... it won't happen in this lifetime.
+//
+String getStringHash64(const String& in)
+{
+   String cacheKey = in;
+   cacheKey.replace("\n", " ");
+   U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0);
+   hash = convertHostToLEndian(hash);
+   U32 high = (U32)(hash >> 32);
+   U32 low = (U32)(hash & 0x00000000FFFFFFFF);
+   cacheKey = String::ToString("%x%x", high, low);
+   return cacheKey;
+}
+
 } // namespace

Engine/source/core/util/hashFunction.h

@@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval);
 
 extern U64 hash64(const U8 *k, U32 length, U64 initval);
 
+extern String getStringHash64(const String& in);
+
 }
 
 #endif // _HASHFUNCTION_H_

Engine/source/gfx/D3D11/gfxD3D11Shader.cpp

@@ -681,6 +681,24 @@ bool GFXD3D11Shader::_init()
    return true;
 }
 
+static String buildMacroHash(const D3D_SHADER_MACRO* defines)
+{
+   String combined;
+
+   if (!defines)
+      return "";
+
+   for (const D3D_SHADER_MACRO* m = defines; m->Name != nullptr; ++m)
+   {
+      combined += m->Name;
+      combined += "=";
+      combined += m->Definition ? m->Definition : "";
+      combined += ";";
+   }
+
+   return Torque::getStringHash64(combined);
+}
+
 bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
                                     GFXShaderStage shaderStage,
                                     const D3D_SHADER_MACRO *defines)
@@ -706,8 +724,40 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
    Con::printf( "Compiling Shader: '%s'", filePath.getFullPath().c_str() );
 #endif
 
+   String macroHash = buildMacroHash(defines);
+
+   Torque::Path cachePath = filePath;
+   cachePath.setExtension("tso");
+   cachePath.setFileName(cachePath.getFileName() + "_" + macroHash);
+
+   if (Torque::FS::IsFile(cachePath))
+   {
+      Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(filePath);
+      Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath);
+
+      if (rawFile != NULL && cachedFile != NULL)
+      {
+         if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime())
+         {
+
+            FileStream fs;
+            if (fs.open(cachePath, Torque::FS::File::Read))
+            {
+               U32 size = fs.getStreamSize();
+
+               D3DCreateBlob(size, &code);
+               fs.read(size, code->GetBufferPointer());
+
+               res = 1;
+            }
+         }
+      }
+   }
+
+   bool loadedFromCache = (code != NULL);
+
    // Is it an HLSL shader?
-   if(filePath.getExtension().equal("hlsl", String::NoCase))
+   if(filePath.getExtension().equal("hlsl", String::NoCase) && !loadedFromCache)
    {
       // Set this so that the D3DInclude::Open will have this
       // information for relative paths.
@@ -788,6 +838,20 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
 
    AssertISV(SUCCEEDED(res), "Unable to compile shader!");
 
+   // succeeded write out a cache
+   if (!loadedFromCache)
+   {
+      if (SUCCEEDED(res) && code)
+      {
+         // Save cache
+         FileStream out(FileStream::AsyncMode::Background);
+         if (out.open(cachePath, Torque::FS::File::Write))
+         {
+            out.write(code->GetBufferSize(), code->GetBufferPointer());
+         }
+      }
+   }
+
    if(code != NULL)
    {
       switch (shaderStage)

Engine/source/gfx/gl/gfxGLShader.cpp

@@ -542,6 +542,77 @@ bool GFXGLShader::_init()
    macros.last().name = "TORQUE_VERTEX_SHADER";
    macros.last().value = "";
 
+   Torque::Path cachePath = mVertexFile;
+   cachePath.setExtension("tso");
+   String macroHash;
+   GFXShaderMacro::stringize(macros, &macroHash);
+   macroHash = Torque::getStringHash64(macroHash);
+
+   String fileName = cachePath.getFileName();
+
+   if (!mVertexFile.isEmpty())
+      fileName += "_" + mVertexFile.getFileName();
+
+   if (!mPixelFile.isEmpty())
+      fileName += "_" + mPixelFile.getFileName();
+
+   if (!mGeometryFile.isEmpty())
+      fileName += "_" + mGeometryFile.getFileName();
+
+   fileName += "_" + macroHash;
+
+   cachePath.setFileName(fileName);
+
+   if (Torque::FS::IsFile(cachePath))
+   {
+      Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(mVertexFile);
+      Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath);
+
+      if (rawFile != NULL && cachedFile != NULL)
+      {
+         if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime())
+         {
+
+            FileStream fs;
+            if (fs.open(cachePath, Torque::FS::File::Read))
+            {
+               U32 size;
+               FrameAllocatorMarker bin;
+               GLenum gl_format;
+               fs.read(&gl_format);
+               fs.read(&size);
+
+               char* bin_data = (char*)bin.alloc(size);
+               fs.read(size, bin_data);
+
+               glProgramBinary(
+                  mProgram,
+                  gl_format,
+                  bin_data,
+                  size);
+
+
+               GLint linked;
+               glGetProgramiv(mProgram, GL_LINK_STATUS, &linked);
+               if (linked == GL_TRUE)
+               {
+                  initConstantDescs();
+                  initHandles();
+
+                  // Notify Buffers we might have changed in size.
+                  // If this was our first init then we won't have any activeBuffers
+                  // to worry about unnecessarily calling.
+                  Vector<GFXShaderConstBuffer*>::iterator biter = mActiveBuffers.begin();
+                  for (; biter != mActiveBuffers.end(); biter++)
+                     ((GFXGLShaderConstBuffer*)(*biter))->onShaderReload(this);
+
+                  return true;
+               }
+            }
+         }
+      }
+   }
+
    // Default to true so we're "successful" if a vertex/pixel shader wasn't specified.
    bool compiledVertexShader = true;
    bool compiledPixelShader = true;
@@ -659,6 +730,27 @@ bool GFXGLShader::_init()
    if (linkStatus == GL_FALSE)
       return false;
 
+   GLint binaryLength;
+   glGetProgramiv(mProgram, GL_PROGRAM_BINARY_LENGTH, &binaryLength);
+
+   if (binaryLength)
+   {
+      FrameAllocatorMarker bin;
+      char* bin_data = (char*)bin.alloc(binaryLength);
+      GLenum binaryFormat;
+      GLint length;
+      glGetProgramBinary(mProgram, binaryLength, &length, &binaryFormat, bin_data);
+      AssertWarn(length == binaryLength, "GFXGLShader: Binary length does not match");
+
+      FileStream out(FileStream::AsyncMode::Background);
+      if (out.open(cachePath, Torque::FS::File::Write))
+      {
+         out.write(binaryFormat);
+         out.write(length);
+         out.write(length, bin_data);
+      }
+   }
+
    initConstantDescs();
    initHandles();
 

Engine/source/materials/shaderData.cpp

@@ -71,6 +71,8 @@ ShaderData::ShaderData()
    for( int i = 0; i < NumTextures; ++i)
       mRTParams[i] = false;
 
+   mInstancingFormat = NULL;
+
    mDXVertexShaderName = StringTable->EmptyString();
    mDXPixelShaderName = StringTable->EmptyString();
    mDXGeometryShaderName = StringTable->EmptyString();
@@ -217,6 +219,8 @@ GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
    String cacheKey;
    GFXShaderMacro::stringize( macros, &cacheKey );
 
+   cacheKey = Torque::getStringHash64(cacheKey);
+
    // Lookup the shader for this instance.
    ShaderCache::Iterator iter = mShaders.find( cacheKey );
    if ( iter != mShaders.end() )
@@ -257,30 +261,32 @@ GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
    {
       case Direct3D11:
       {
-         if (mDXVertexShaderName != String::EmptyString)
+         if (mDXVertexShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mDXVertexShaderName);
-         if (mDXPixelShaderName != String::EmptyString)
+         if (mDXPixelShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mDXPixelShaderName);
-         if (mDXGeometryShaderName != String::EmptyString)
+         if (mDXGeometryShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mDXGeometryShaderName);
          success = shader->init( pixver,
                                  macros,
-                                 samplers);
+                                 samplers,
+                                 mInstancingFormat);
          break;
       }
 
       case OpenGL:
       {
-         if(mOGLVertexShaderName != String::EmptyString)
+         if(mOGLVertexShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mOGLVertexShaderName);
-         if (mOGLPixelShaderName != String::EmptyString)
+         if (mOGLPixelShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mOGLPixelShaderName);
-         if (mOGLGeometryShaderName != String::EmptyString)
+         if (mOGLGeometryShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mOGLGeometryShaderName);
 
          success = shader->init( pixver,
                                  macros,
-                                 samplers);
+                                 samplers,
+                                 mInstancingFormat);
          break;
       }
 
@@ -321,6 +327,33 @@ GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
    return shader;
 }
 
+void ShaderData::setShaderStageFile(GFXShaderStage stage, String fileName)
+{
+   const bool isGL = GFX->getAdapterType() == GFXAdapterType::OpenGL;
+   switch (stage)
+   {
+   case VERTEX_SHADER:
+      isGL ? mOGLVertexShaderName = StringTable->insert(fileName) : mDXVertexShaderName = StringTable->insert(fileName);
+      break;
+   case PIXEL_SHADER:
+      isGL ? mOGLPixelShaderName = StringTable->insert(fileName) : mDXPixelShaderName = StringTable->insert(fileName);
+      break;
+   case GEOMETRY_SHADER:
+      isGL ? mOGLGeometryShaderName = StringTable->insert(fileName) : mDXGeometryShaderName = StringTable->insert(fileName);
+      break;
+   case DOMAIN_SHADER:
+      break;
+   case HULL_SHADER:
+      break;
+   case COMPUTE_SHADER:
+      break;
+   case ALL_STAGES:
+      break;
+   default:
+      break;
+   }
+}
+
 void ShaderData::reloadShaders()
 {
    ShaderCache::Iterator iter = mShaders.begin();

Engine/source/materials/shaderData.h

@@ -80,11 +80,6 @@ protected:
    /// them if the content has changed.
    const Vector<GFXShaderMacro>& _getMacros();
 
-   /// Helper for converting an array of macros
-   /// into a formatted string.
-   void _stringizeMacros(const Vector<GFXShaderMacro>& macros,
-      String* outString);
-
    /// Creates a new shader returning NULL on error.
    GFXShader* _createShader(const Vector<GFXShaderMacro>& macros);
 
@@ -98,6 +93,8 @@ protected:
 
    String mSamplerNames[NumTextures];
    bool mRTParams[NumTextures];
+   // the instancing format.
+   GFXVertexFormat* mInstancingFormat;
 
    bool _checkDefinition(GFXShader* shader);
 
@@ -105,9 +102,11 @@ public:
 
    void setSamplerName(const String& name, int idx) { mSamplerNames[idx] = name; }
    String getSamplerName(int idx) const { return mSamplerNames[idx]; }
+   void setShaderStageFile(GFXShaderStage stage, String fileName);
 
    bool hasSamplerDef(const String& samplerName, int& pos) const;
    bool hasRTParamsDef(const int pos) const { return mRTParams[pos]; }
+   void setInstancingFormat(GFXVertexFormat* instancingFormat) { mInstancingFormat = instancingFormat; }
 
    ShaderData();
 
@@ -122,6 +121,7 @@ public:
    /// all loaded ShaderData objects in the system.
    static void reloadAllShaders();
 
+   void setPixVersion(F32 pixVersion) { mPixVersion = pixVersion; }
    /// Returns the required pixel shader version for this shader.
    F32 getPixVersion() const { return mPixVersion; }
 

Engine/source/shaderGen/shaderFeature.h

@@ -237,6 +237,8 @@ public:
    /// Allows the feature to add macros to vertex shader compiles.
    virtual void processVertMacros( Vector<GFXShaderMacro> &macros, const MaterialFeatureData &fd  ) {};
 
+   virtual U32 getShaderStages() { return (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER); }
+
    /// Identifies what type of blending a feature uses.  This is used to
    /// group features with the same blend operation together in a multipass
    /// situation.

Engine/source/shaderGen/shaderGen.cpp

@@ -30,6 +30,7 @@
 #include "gfx/gfxDevice.h"
 #include "core/memVolume.h"
 #include "core/module.h"
+#include "console/persistenceManager.h"
 
 #ifdef TORQUE_D3D11
 #include "shaderGen/HLSL/customFeatureHLSL.h"
@@ -55,6 +56,32 @@ MODULE_BEGIN( ShaderGen )
 
 MODULE_END;
 
+
+static const U32 gStageOrder[] =
+{
+   GFXShaderStage::VERTEX_SHADER,
+   GFXShaderStage::HULL_SHADER,
+   GFXShaderStage::DOMAIN_SHADER,
+   GFXShaderStage::GEOMETRY_SHADER,
+   GFXShaderStage::PIXEL_SHADER,
+   GFXShaderStage::COMPUTE_SHADER
+};
+
+static const char* _getStagePostfix(GFXShaderStage stage)
+{
+   switch (stage)
+   {
+   case GFXShaderStage::VERTEX_SHADER:   return "_V";
+   case GFXShaderStage::HULL_SHADER:     return "_H";
+   case GFXShaderStage::DOMAIN_SHADER:   return "_D";
+   case GFXShaderStage::GEOMETRY_SHADER: return "_G";
+   case GFXShaderStage::PIXEL_SHADER:    return "_P";
+   case GFXShaderStage::COMPUTE_SHADER:  return "_C";
+   }
+
+   return "_U"; // Unknown
+}
+
 String ShaderGen::smCommonShaderPath("shaders/common");
 
 ShaderGen::ShaderGen()
@@ -68,6 +95,14 @@ ShaderGen::~ShaderGen()
 {
    GFXDevice::getDeviceEventSignal().remove(this, &ShaderGen::_handleGFXEvent);
    _uninit();
+
+   for (ShaderDataMap::Pair data : mProcShaderData)
+   {
+      if (data.value->isProperlyAdded() && !data.value->isDeleted())
+         data.value->unregisterObject();
+   }
+
+   mProcShaderData.clear();
 }
 
 void ShaderGen::registerInitDelegate(GFXAdapterType adapterType, ShaderGenInitDelegate& initDelegate)
@@ -99,6 +134,7 @@ void ShaderGen::initShaderGen()
       return;
 
    const GFXAdapterType adapterType = GFX->getAdapterType();
+   const bool isGl = adapterType == GFXAdapterType::OpenGL;
    if (!mInitDelegates[adapterType])
       return;
 
@@ -133,17 +169,25 @@ void ShaderGen::initShaderGen()
 
    // Delete the auto-generated conditioner include file.
    Torque::FS::Remove( "shadergen:/" + ConditionerFeature::ConditionerIncludeFileName );
+
+   Vector<String> fileList;
+   String pattern = "*.";
+   pattern += isGl ? "glsl" : "hlsl";
+   S32 numShaderFiles = Torque::FS::FindByPattern("shadergen:/", pattern, false, fileList);
+   for (U32 i = 0; i < numShaderFiles; i++)
+   {
+      Torque::Path filePath = fileList[i];
+      mFileCache[filePath.getFileName()] = true;
+   }
 }
 
-void ShaderGen::generateShader( const MaterialFeatureData &featureData,
+void ShaderGen::generateShader(const MaterialFeatureData& featureData,
-                                char *vertFile,
+                               ShaderData* shaderData,
-                                char *pixFile,
+                               const GFXVertexFormat* vertexFormat,
-                                F32 *pixVersion,
+                               const char* cacheName,
-                                const GFXVertexFormat *vertexFormat,
+                               Vector<GFXShaderMacro>& macros)
-                                const char* cacheName,
-                                Vector<GFXShaderMacro> &macros)
 {
-   PROFILE_SCOPE( ShaderGen_GenerateShader );
+   PROFILE_SCOPE(ShaderGen_GenerateShader);
 
    mFeatureData = featureData;
    mVertexFormat = vertexFormat;
@@ -151,65 +195,115 @@ void ShaderGen::generateShader( const MaterialFeatureData &featureData,
    _uninit();
    _init();
 
-   char vertShaderName[256];
+   const FeatureSet& features = mFeatureData.features;
-   char pixShaderName[256];
+   U32 stages = 0;
 
-   // Note:  We use a postfix of _V/_P here so that it sorts the matching
+   // loop through and see which stages this featureset is expecting to make.
-   // vert and pixel shaders together when listed alphabetically.
+   for (U32 i = 0; i < features.getCount(); i++)
-   dSprintf( vertShaderName, sizeof(vertShaderName), "shadergen:/%s_V.%s", cacheName, mFileEnding.c_str() );
-   dSprintf( pixShaderName, sizeof(pixShaderName), "shadergen:/%s_P.%s", cacheName, mFileEnding.c_str() );
-
-   dStrcpy( vertFile, vertShaderName, 256 );
-   dStrcpy( pixFile, pixShaderName, 256 );
-
-   // this needs to change - need to optimize down to ps v.1.1
-   *pixVersion = GFX->getPixelShaderVersion();
-
-   if ( !Con::getBoolVariable( "ShaderGen::GenNewShaders", true ) )
    {
-      // If we are not regenerating the shader we will return here.
-      // But we must fill in the shader macros first!
 
-      _processVertFeatures( macros, true );
+      const FeatureType& type = features.getAt(i);
-      _processPixFeatures( macros, true );
+      ShaderFeature* feat = FEATUREMGR->getByType(type);
+      stages |= feat->getShaderStages();
 
-      return;
    }
 
-   // create vertex shader
+   for (U32 s = 0; s < (sizeof(gStageOrder) / sizeof(U32)); s++)
-   //------------------------
-   FileStream* s = new FileStream();
-   if(!s->open(vertShaderName, Torque::FS::File::Write ))
    {
-      AssertFatal(false, "Failed to open Shader Stream" );
+      U32 stage = gStageOrder[s];
-      return;
+
+      // skip unused stages
+      if (!(stages & stage))
+         continue;
+
+      bool macrosOnly = !Con::getBoolVariable("ShaderGen::GenNewShaders", true);
+      bool skipPrint = false;
+      GFXShaderStage curStage = (GFXShaderStage)stage;
+
+      char fileName[256];
+      const char* postfix = _getStagePostfix(curStage);
+      String stageName;
+
+      if (curStage & GFXShaderStage::VERTEX_SHADER)
+         stageName += vertexFormat->getDescription();
+
+      // build our filename.
+      for (U32 i = 0; i < features.getCount(); i++)
+      {
+         const FeatureType& type = features.getAt(i);
+         if (stage & FEATUREMGR->getByType(type)->getShaderStages())
+         {
+            stageName += type.getName();
+         }
+      }
+
+      stageName = Torque::getStringHash64(stageName);
+      stageName += postfix;
+
+      FileCacheSet::iterator file = mFileCache.find(stageName);
+      if (file != mFileCache.end())
+      {
+         // set the shaderdata file for this stage, shaderdata ptr needs to be passed in here.
+         dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str());
+         shaderData->setShaderStageFile(curStage, fileName);
+         if (!(curStage & GFXShaderStage::VERTEX_SHADER))
+         {
+            continue;
+         }
+
+         skipPrint = true;
+      }
+
+      mFileCache[stageName] = true;
+
+      dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str());
+
+      shaderData->setShaderStageFile(curStage, fileName);
+
+
+      FileStream* stream = new FileStream(FileStream::AsyncMode::Background);
+      if (!skipPrint)
+      {
+         if (!stream->open(fileName, Torque::FS::File::Write))
+         {
+            AssertFatal(false, "Failed to open Shader Stream");
+            return;
+         }
+      }
+
+      switch (curStage)
+      {
+      case VERTEX_SHADER:
+         mOutput = new MultiLine;
+         mInstancingFormat.clear();
+         _processVertFeatures(macros, macrosOnly);
+         if (!skipPrint || macrosOnly) _printVertShader(*stream);
+         delete stream;
+         ((ShaderConnector*)mComponents[C_CONNECTOR])->reset();
+         LangElement::deleteElements();
+         break;
+      case PIXEL_SHADER:
+         mOutput = new MultiLine;
+         _processPixFeatures(macros, macrosOnly);
+         if (!skipPrint || macrosOnly)_printPixShader(*stream);
+         delete stream;
+         LangElement::deleteElements();
+         break;
+      case GEOMETRY_SHADER:
+         break;
+      case DOMAIN_SHADER:
+         break;
+      case HULL_SHADER:
+         break;
+      case COMPUTE_SHADER:
+         break;
+      case ALL_STAGES:
+         break;
+      default:
+         break;
+      }
+
    }
-
-   mOutput = new MultiLine;
-   mInstancingFormat.clear();
-   _processVertFeatures(macros);
-   _printVertShader( *s );
-   delete s;
-
-   ((ShaderConnector*)mComponents[C_CONNECTOR])->reset();
-   LangElement::deleteElements();
-
-   // create pixel shader
-   //------------------------
-   s = new FileStream();
-   if(!s->open(pixShaderName, Torque::FS::File::Write ))
-   {
-      AssertFatal(false, "Failed to open Shader Stream" );
-      delete s;
-      return;
-   }
-
-   mOutput = new MultiLine;
-   _processPixFeatures(macros);
-   _printPixShader( *s );
-
-   delete s;
-   LangElement::deleteElements();
 }
 
 void ShaderGen::_init()
@@ -264,7 +358,7 @@ void ShaderGen::_processVertFeatures( Vector<GFXShaderMacro> &macros, bool macro
       else
          feature = FEATUREMGR->getByType( type );
 
-      if ( feature )
+      if ( feature && (feature->getShaderStages() & GFXShaderStage::VERTEX_SHADER) )
       {
          feature->setProcessIndex( index );
 
@@ -312,7 +406,7 @@ void ShaderGen::_processPixFeatures( Vector<GFXShaderMacro> &macros, bool macros
          feature = FEATUREMGR->createFeature(type, args);
       else
          feature = FEATUREMGR->getByType(type);
-      if ( feature )
+      if ( feature && (feature->getShaderStages() & GFXShaderStage::PIXEL_SHADER) )
       {
          feature->setProcessIndex( index );
 
@@ -466,60 +560,50 @@ void ShaderGen::_printPixShader( Stream &stream )
    mPrinter->printPixelShaderCloser(stream);
 }
 
-GFXShader* ShaderGen::getShader( const MaterialFeatureData &featureData, const GFXVertexFormat *vertexFormat, const Vector<GFXShaderMacro> *macros, const Vector<String> &samplers )
+GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GFXVertexFormat* vertexFormat, const Vector<GFXShaderMacro>* macros, const Vector<String>& samplers)
 {
-   PROFILE_SCOPE( ShaderGen_GetShader );
+   PROFILE_SCOPE(ShaderGen_GetShader);
 
-   const FeatureSet &features = featureData.codify();
+   const FeatureSet& features = featureData.codify();
 
    // Build a description string from the features
    // and vertex format combination ( and macros ).
    String shaderDescription = vertexFormat->getDescription() + features.getDescription();
-   // Generate a single 64bit hash from the description string.
-   //
-   // Don't get paranoid!  This has 1 in 18446744073709551616
-   // chance for collision... it won't happen in this lifetime.
-   //
-   shaderDescription.replace("\n", " ");
-   U64 hash = Torque::hash64( (const U8*)shaderDescription.c_str(), shaderDescription.length(), 0 );
-   hash = convertHostToLEndian(hash);
-   U32 high = (U32)( hash >> 32 );
-   U32 low = (U32)( hash & 0x00000000FFFFFFFF );
-   String cacheKey = String::ToString( "%x%x", high, low );
-   // return shader if exists
-   GFXShader *match = mProcShaders[cacheKey];
-   if ( match )
-      return match;
 
-   // if not, then create it
+   String cacheKey = Torque::getStringHash64(shaderDescription);
-   char vertFile[256];
-   char pixFile[256];
-   F32  pixVersion;
 
    Vector<GFXShaderMacro> shaderMacros;
-   shaderMacros.push_back( GFXShaderMacro( "TORQUE_SHADERGEN" ) );
+   shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN"));
-   if ( macros )
+   if (macros)
-      shaderMacros.merge( *macros );
+      shaderMacros.merge(*macros);
-   generateShader( featureData, vertFile, pixFile, &pixVersion, vertexFormat, cacheKey, shaderMacros );
 
-   GFXShader *shader = GFX->createShader();
+   ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey);
-   shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, vertFile);
+   if (dat != mProcShaderData.end())
-   shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, pixFile);
-
-   if (!shader->init(pixVersion, shaderMacros, samplers, &mInstancingFormat))
    {
-      delete shader;
+      // should we loop vertex shader features to build mInstancingFormat before sending it down to see old hob?
-      return NULL;
+      return dat->value->getShader(shaderMacros);
    }
 
-   mProcShaders[cacheKey] = shader;
+   ShaderData* shaderData = new ShaderData;
 
-   return shader;
+   shaderData->setPixVersion(GFX->getPixelShaderVersion());
+
+   for (U32 samp = 0; samp < samplers.size(); samp++)
+   {
+      shaderData->setSamplerName(samplers[samp], samp);
+   }
+
+   generateShader(featureData, shaderData, vertexFormat, cacheKey, shaderMacros);
+
+   shaderData->setInstancingFormat(&mInstancingFormat);
+
+   mProcShaderData.insert(cacheKey, shaderData);
+   return shaderData->getShader(shaderMacros);
 }
 
 void ShaderGen::flushProceduralShaders()
 {
    // The shaders are reference counted, so we
    // just need to clear the map.
-   mProcShaders.clear();
+   mProcShaderData.clear();
 }

Engine/source/shaderGen/shaderGen.h

@@ -46,6 +46,9 @@
 #ifndef _MATERIALFEATUREDATA_H_
 #include "materials/materialFeatureData.h"
 #endif
+#ifndef _SHADERDATA_H_
+#include "materials/shaderData.h"
+#endif // !_SHADERDATA_H_
 
 /// Base class used by shaderGen to be API agnostic.  Subclasses implement the various methods
 /// in an API specific way.
@@ -146,9 +149,7 @@ public:
    /// this function.
    /// @param assignNum used to assign a specific number as the filename   
    void generateShader( const MaterialFeatureData &featureData,
-                        char *vertFile, 
+                        ShaderData* shaderData,
-                        char *pixFile, 
-                        F32 *pixVersion,
                         const GFXVertexFormat *vertexFormat,
                         const char* cacheName,
                         Vector<GFXShaderMacro> &macros);
@@ -192,9 +193,14 @@ protected:
    bool mRegisteredWithGFX;
    Torque::FS::FileSystemRef mMemFS;
    
-   /// Map of cache string -> shaders
+   /// <summary>
-   typedef Map<String, GFXShaderRef> ShaderMap;
+   /// Map of shaderdata, string should be built up of stage files
-   ShaderMap mProcShaders;
+   /// </summary>
+   typedef HashMap<String, SimObjectPtr<ShaderData>> ShaderDataMap;
+   ShaderDataMap mProcShaderData;
+
+   typedef HashMap<String, bool> FileCacheSet; // we use a hashmap because it is quicker for finding.
+   FileCacheSet mFileCache;
 
    ShaderGen();
 

Engine/source/ts/assimp/assimpShapeLoader.cpp

@@ -771,7 +771,7 @@ void AssimpShapeLoader::extractTexture(U32 index, aiTexture* pTex)
    {  // Compressed format, write the data directly to disc
       texPath.setExtension(pTex->achFormatHint);
       FileStream *outputStream;
-      if ((outputStream = FileStream::createAndOpen(texPath.getFullPath(), Torque::FS::File::Write)) != NULL)
+      if ((outputStream = FileStream::createAndOpen(texPath.getFullPath(), Torque::FS::File::Write, FileStream::AsyncMode::Background)) != NULL)
       {
          outputStream->setPosition(0);
          outputStream->write(pTex->mWidth, pTex->pcData);

Engine/source/ts/collada/colladaShapeLoader.cpp

@@ -720,7 +720,7 @@ static bool sReadCollada(const Torque::Path& path, TSShape*& res_shape)
          // Cache the model to a DTS file for faster loading next time.
          cachedPath.setExtension("cached.dts");
          // Cache the model to a DTS file for faster loading next time.
-         FileStream dtsStream;
+         FileStream dtsStream(FileStream::AsyncMode::Background);
          if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write))
          {
             Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str());