ShaderGen stage dynamics

ShaderGen now generates a ShaderData class to simplify macro switchups
ShaderData caches mInstancingFormat

ShaderGen now creates a cache of the files that already exist, and if it exists it will return and use that file instead of regenning a new one. Vertex files can be used for multiple pixel files and vice versa

Requires partial shadernode setup due to changes on how shader feature parameters are handled

Engine/source/forest/hlsl/windDeformationHLSL.h

@@ -46,6 +46,8 @@ public:
    void processVert( Vector<ShaderComponent*> &componentList,
                              const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; }
+
    String getName() override
    {
       return "Wind Effect";

Engine/source/gfx/gfxAPI.h

@@ -44,6 +44,7 @@ DefineEnumType( GFXTextureFilterType );
 DefineEnumType( GFXCullMode );
 DefineEnumType( GFXStencilOp );
 DefineEnumType( GFXBlendOp );
+DefineEnumType(GFXShaderConstType);
 DefineEnumType( GFXAdapterType );
 
 DECLARE_STRUCT( GFXVideoMode );

Engine/source/gfx/gfxShader.cpp

@@ -42,8 +42,12 @@ GFXShader::GFXShader()
 
 GFXShader::~GFXShader()
 {
-   Torque::FS::RemoveChangeNotification( mVertexFile, this, &GFXShader::_onFileChanged );
+   if (!mVertexFile.isEmpty())
-   Torque::FS::RemoveChangeNotification( mPixelFile, this, &GFXShader::_onFileChanged );
+      Torque::FS::RemoveChangeNotification( mVertexFile, this, &GFXShader::_onFileChanged );
+   if (!mPixelFile.isEmpty())
+      Torque::FS::RemoveChangeNotification( mPixelFile, this, &GFXShader::_onFileChanged );
+   if (!mGeometryFile.isEmpty())
+      Torque::FS::RemoveChangeNotification(mGeometryFile, this, &GFXShader::_onFileChanged);
 
    SAFE_DELETE(mInstancingFormat);
 }

Engine/source/gfx/gfxShader.h

@@ -77,7 +77,9 @@ enum GFXShaderStage
    GEOMETRY_SHADER   = BIT(2),
    DOMAIN_SHADER     = BIT(3),
    HULL_SHADER       = BIT(4),
-   COMPUTE_SHADER    = BIT(5)
+   COMPUTE_SHADER    = BIT(5),
+   ALL_STAGES = VERTEX_SHADER | PIXEL_SHADER | GEOMETRY_SHADER |
+   DOMAIN_SHADER | HULL_SHADER | COMPUTE_SHADER
 };
 
 /// Instances of this struct are returned GFXShaderConstBuffer

Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h

@@ -138,6 +138,8 @@ public:
    void processPix(   Vector<ShaderComponent*> &componentList, 
                               const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override
    {
       return "Sub-Surface Approximation [Deferred]";

Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h

@@ -57,6 +57,8 @@ public:
       const MaterialFeatureData &fd ) override;
 
    U32 getOutputTargets(const MaterialFeatureData& fd) const override;
+
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
 };
 
 class ORMConfigVarsHLSL : public ShaderFeatureHLSL
@@ -68,6 +70,8 @@ public:
 
    void processPix( Vector<ShaderComponent*> &componentList, 
       const MaterialFeatureData &fd ) override;
+
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
 };
 
 class GlowMapHLSL : public ShaderFeatureHLSL
@@ -78,6 +82,8 @@ public:
    void processPix(Vector<ShaderComponent*> &componentList,
       const MaterialFeatureData &fd) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    U32 getOutputTargets(const MaterialFeatureData& fd) const override;
 
    Resources getResources(const MaterialFeatureData& fd) override;

Engine/source/materials/shaderData.cpp

@@ -78,6 +78,8 @@ ShaderData::ShaderData()
    mOGLVertexShaderName = StringTable->EmptyString();
    mOGLPixelShaderName = StringTable->EmptyString();
    mOGLGeometryShaderName = StringTable->EmptyString();
+
+   mInstancingFormat = NULL;
 }
 
 void ShaderData::initPersistFields()
@@ -204,7 +206,7 @@ const Vector<GFXShaderMacro>& ShaderData::_getMacros()
    return mShaderMacros;
 }
 
-GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
+GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros)
 {
    PROFILE_SCOPE( ShaderData_GetShader );
 
@@ -224,7 +226,7 @@ GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
 
    // Create the shader instance... if it fails then
    // bail out and return nothing to the caller.
-   GFXShader *shader = _createShader( finalMacros );
+   GFXShader *shader = _createShader( finalMacros);
    if ( !shader )
       return NULL;
 
@@ -235,7 +237,7 @@ GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
    return shader;
 }
 
-GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
+GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros)
 {
    F32 pixver = mPixVersion;
    if ( mUseDevicePixVersion )
@@ -257,30 +259,32 @@ GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
    {
       case Direct3D11:
       {
-         if (mDXVertexShaderName != String::EmptyString)
+         if (mDXVertexShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mDXVertexShaderName);
-         if (mDXPixelShaderName != String::EmptyString)
+         if (mDXPixelShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mDXPixelShaderName);
-         if (mDXGeometryShaderName != String::EmptyString)
+         if (mDXGeometryShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mDXGeometryShaderName);
          success = shader->init( pixver,
                                  macros,
-                                 samplers);
+                                 samplers,
+                                 mInstancingFormat);
          break;
       }
 
       case OpenGL:
       {
-         if(mOGLVertexShaderName != String::EmptyString)
+         if(mOGLVertexShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mOGLVertexShaderName);
-         if (mOGLPixelShaderName != String::EmptyString)
+         if (mOGLPixelShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mOGLPixelShaderName);
-         if (mOGLGeometryShaderName != String::EmptyString)
+         if (mOGLGeometryShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mOGLGeometryShaderName);
 
          success = shader->init( pixver,
                                  macros,
-                                 samplers);
+                                 samplers,
+                                 mInstancingFormat);
          break;
       }
 
@@ -348,6 +352,33 @@ void ShaderData::_onLMActivate( const char *lm, bool activate )
    reloadAllShaders();
 }
 
+void ShaderData::setShaderStageFile(GFXShaderStage stage, String fileName)
+{
+   const bool isGL = GFX->getAdapterType() == GFXAdapterType::OpenGL;
+   switch (stage)
+   {
+   case VERTEX_SHADER:
+      isGL ? mOGLVertexShaderName = StringTable->insert(fileName) : mDXVertexShaderName = StringTable->insert(fileName);
+      break;
+   case PIXEL_SHADER:
+      isGL ? mOGLPixelShaderName = StringTable->insert(fileName) : mDXPixelShaderName = StringTable->insert(fileName);
+      break;
+   case GEOMETRY_SHADER:
+      isGL ? mOGLGeometryShaderName = StringTable->insert(fileName) : mDXGeometryShaderName = StringTable->insert(fileName);
+      break;
+   case DOMAIN_SHADER:
+      break;
+   case HULL_SHADER:
+      break;
+   case COMPUTE_SHADER:
+      break;
+   case ALL_STAGES:
+      break;
+   default:
+      break;
+   }
+}
+
 bool ShaderData::hasSamplerDef(const String &_samplerName, int &pos) const
 {
    String samplerName = _samplerName.startsWith("$") ? _samplerName : "$"+_samplerName;

Engine/source/materials/shaderData.h

@@ -80,10 +80,8 @@ protected:
    /// them if the content has changed.
    const Vector<GFXShaderMacro>& _getMacros();
 
-   /// Helper for converting an array of macros
+   // the instancing format.
-   /// into a formatted string.
+   GFXVertexFormat* mInstancingFormat;
-   void _stringizeMacros(const Vector<GFXShaderMacro>& macros,
-      String* outString);
 
    /// Creates a new shader returning NULL on error.
    GFXShader* _createShader(const Vector<GFXShaderMacro>& macros);
@@ -106,8 +104,11 @@ public:
    void setSamplerName(const String& name, int idx) { mSamplerNames[idx] = name; }
    String getSamplerName(int idx) const { return mSamplerNames[idx]; }
 
+   void setShaderStageFile(GFXShaderStage stage, String fileName);
+
    bool hasSamplerDef(const String& samplerName, int& pos) const;
    bool hasRTParamsDef(const int pos) const { return mRTParams[pos]; }
+   void setInstancingFormat(GFXVertexFormat* instancingFormat) { mInstancingFormat = instancingFormat; }
 
    ShaderData();
 
@@ -122,6 +123,7 @@ public:
    /// all loaded ShaderData objects in the system.
    static void reloadAllShaders();
 
+   void setPixVersion(F32 pixVersion) { mPixVersion = pixVersion; }
    /// Returns the required pixel shader version for this shader.
    F32 getPixVersion() const { return mPixVersion; }
 

Engine/source/shaderGen/HLSL/accuFeatureHLSL.h

@@ -98,6 +98,8 @@ public:
       }
    }
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override { return "Accu Scale"; }
 };
 
@@ -118,6 +120,8 @@ public:
       }
    }
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override { return "Accu Direction"; }
 };
 
@@ -138,6 +142,8 @@ public:
       }
    }
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override { return "Accu Strength"; }
 };
 
@@ -158,6 +164,8 @@ public:
       }
    }
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override { return "Accu Coverage"; }
 };
 
@@ -179,6 +187,8 @@ public:
       }
    }
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override { return "Accu Specular"; }
 };
 

Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h

@@ -37,5 +37,7 @@ public:
    void processPix(Vector<ShaderComponent*>& componentList,
       const MaterialFeatureData& fd) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override { return "Debug Viz"; }
 };

Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h

@@ -181,6 +181,8 @@ public:
    void processPix( Vector<ShaderComponent*> &componentList, 
       const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+   
    U32 getOutputTargets( const MaterialFeatureData &fd ) const override { return mOutputTargetMask; }
 };
 
@@ -307,6 +309,8 @@ public:
    void processPix(   Vector<ShaderComponent*> &componentList, 
                               const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    Material::BlendOp getBlendOp() override{ return Material::None; }
 
    U32 getOutputTargets(const MaterialFeatureData &fd) const override;
@@ -558,6 +562,9 @@ public:
 
    void processPix(   Vector<ShaderComponent*> &componentList, 
                               const MaterialFeatureData &fd ) override;
+
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    String getName() override
    {
       return "Glow Mask";
@@ -582,6 +589,8 @@ public:
    void processPix(   Vector<ShaderComponent*> &componentList, 
                               const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+
    Material::BlendOp getBlendOp() override { return Material::None; }
 
    String getName() override { return "HDR Output"; }
@@ -626,6 +635,8 @@ public:
    void processVert( Vector<ShaderComponent*> &componentList,
       const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; }
+
    String getName() override
    {
       return "Particle Normal Generation Feature";
@@ -672,6 +683,8 @@ public:
    void processVert(  Vector<ShaderComponent*> &componentList,
                               const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; }
+
    String getName() override { return "Hardware Skinning"; }
 };
 

Engine/source/shaderGen/conditionerFeature.cpp

@@ -227,7 +227,7 @@ void ConditionerFeature::printFooterComment( MethodType methodType, const String
    meta->addStatement( new GenOp( "\r\n\r\n" ) );
 }
 
-void ConditionerMethodDependency::print( Stream &s ) const
+void ConditionerMethodDependency::print( Stream &s, GFXShaderStage stage) const
 {
    mConditioner->_printMethod(mMethodType, mConditioner->getShaderMethodName(mMethodType), s);
 }

Engine/source/shaderGen/conditionerFeature.h

@@ -126,9 +126,11 @@ protected:
 
 public:
    ConditionerMethodDependency( ConditionerFeature *conditioner, const ConditionerFeature::MethodType methodType ) :
-      mConditioner(conditioner), mMethodType(methodType) {}
+      mConditioner(conditioner), mMethodType(methodType) {
+      stages = (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER);
+   }
 
-   void print( Stream &s ) const override;
+   void print( Stream &s, GFXShaderStage stage ) const override;
 
    // Auto insert information into a macro
    virtual void createMethodMacro( const String &methodName, Vector<GFXShaderMacro> &macros );

Engine/source/shaderGen/langElement.h

@@ -124,6 +124,8 @@ enum ConstantSortPosition
    cspPotentialPrimitive,
    /// Updated one per pass
    cspPass,
+   /// Set once per scene
+   cspScene,
    /// Count var, do not use
    csp_Count
 };

Engine/source/shaderGen/shaderDependency.cpp

@@ -30,6 +30,7 @@
 ShaderIncludeDependency::ShaderIncludeDependency( const Torque::Path &pathToInclude ) 
    : mIncludePath( pathToInclude )
 {
+   stages = (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER);
 }
 
 bool ShaderIncludeDependency::operator==( const ShaderDependency &cmpTo ) const
@@ -39,9 +40,12 @@ bool ShaderIncludeDependency::operator==( const ShaderDependency &cmpTo ) const
                static_cast<const ShaderIncludeDependency*>( &cmpTo )->mIncludePath == mIncludePath );
 }
 
-void ShaderIncludeDependency::print( Stream &s ) const
+void ShaderIncludeDependency::print( Stream &s , GFXShaderStage stage) const
 {
    // Print the include... all shaders support #includes.
-   String include = String::ToString( "#include \"%s\"\r\n", mIncludePath.getFullPath().c_str() );
+   if (stages & stage)
-   s.write( include.length(), include.c_str() );
+   {
+      String include = String::ToString("#include \"%s\"\r\n", mIncludePath.getFullPath().c_str());
+      s.write(include.length(), include.c_str());
+   }
 }

Engine/source/shaderGen/shaderDependency.h

@@ -26,7 +26,9 @@
 #ifndef _PATH_H_
 #include "core/util/path.h"
 #endif
-
+#ifndef _GFXSHADER_H_
+#include "gfx/gfxShader.h"
+#endif
 
 class Stream;
 
@@ -35,6 +37,7 @@ class Stream;
 class ShaderDependency
 {
 public:
+   U32 stages;
    virtual ~ShaderDependency() {}
    
    /// Compare this dependency to another one.
@@ -44,7 +47,7 @@ public:
    }
 
    /// Print the dependency into the header of a shader.
-   virtual void print( Stream &s ) const = 0;
+   virtual void print( Stream &s, GFXShaderStage stage) const = 0;
 };
 
 
@@ -60,7 +63,7 @@ public:
    ShaderIncludeDependency( const Torque::Path &pathToInclude );
 
    bool operator==( const ShaderDependency &cmpTo ) const override;
-   void print( Stream &s ) const override;
+   void print( Stream &s, GFXShaderStage stage) const override;
 };
 
 #endif // _SHADER_DEPENDENCY_H_

Engine/source/shaderGen/shaderFeature.h

@@ -237,6 +237,8 @@ public:
    /// Allows the feature to add macros to vertex shader compiles.
    virtual void processVertMacros( Vector<GFXShaderMacro> &macros, const MaterialFeatureData &fd  ) {};
 
+   virtual U32 getShaderStages() { return (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER); }
+
    /// Identifies what type of blending a feature uses.  This is used to
    /// group features with the same blend operation together in a multipass
    /// situation.

Engine/source/shaderGen/shaderGen.cpp

@@ -38,6 +38,48 @@
 #include "shaderGen/GLSL/customFeatureGLSL.h"
 #endif
 
+static const U32 gStageOrder[] =
+{
+   GFXShaderStage::VERTEX_SHADER,
+   GFXShaderStage::HULL_SHADER,
+   GFXShaderStage::DOMAIN_SHADER,
+   GFXShaderStage::GEOMETRY_SHADER,
+   GFXShaderStage::PIXEL_SHADER,
+   GFXShaderStage::COMPUTE_SHADER
+};
+
+static const char* _getStagePostfix(GFXShaderStage stage)
+{
+   switch (stage)
+   {
+   case GFXShaderStage::VERTEX_SHADER:   return "_V";
+   case GFXShaderStage::HULL_SHADER:     return "_H";
+   case GFXShaderStage::DOMAIN_SHADER:   return "_D";
+   case GFXShaderStage::GEOMETRY_SHADER: return "_G";
+   case GFXShaderStage::PIXEL_SHADER:    return "_P";
+   case GFXShaderStage::COMPUTE_SHADER:  return "_C";
+   }
+
+   return "_U"; // Unknown
+}
+
+// Generate a single 64bit hash from the input string.
+//
+// Don't get paranoid!  This has 1 in 18446744073709551616
+// chance for collision... it won't happen in this lifetime.
+//
+String getHashForName(const String& in)
+{
+   String cacheKey = in;
+   cacheKey.replace("\n", " ");
+   U64 hash = Torque::hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0);
+   hash = convertHostToLEndian(hash);
+   U32 high = (U32)(hash >> 32);
+   U32 low = (U32)(hash & 0x00000000FFFFFFFF);
+   cacheKey = String::ToString("%x%x", high, low);
+   return cacheKey;
+}
+
 MODULE_BEGIN( ShaderGen )
 
    MODULE_INIT_BEFORE( GFX )
@@ -99,6 +141,8 @@ void ShaderGen::initShaderGen()
       return;
 
    const GFXAdapterType adapterType = GFX->getAdapterType();
+   const bool isGl = adapterType == GFXAdapterType::OpenGL;
+
    if (!mInitDelegates[adapterType])
       return;
 
@@ -134,14 +178,22 @@ void ShaderGen::initShaderGen()
    // Delete the auto-generated conditioner include file.
    Torque::FS::Remove( "shadergen:/" + ConditionerFeature::ConditionerIncludeFileName );
 
+   Vector<String> fileList;
+   String pattern = "*.";
+   pattern += isGl ? "glsl" : "hlsl";
+   S32 numShaderFiles = Torque::FS::FindByPattern("shadergen:/", pattern, false, fileList);
+   for (U32 i = 0; i < numShaderFiles; i++)
+   {
+      Torque::Path filePath = fileList[i];
+      mFileCache[filePath.getFileName()] = true;
+   }
+
    // build our type maps.
    LangElement::buildTypeMaps();
 }
 
-void ShaderGen::generateShader( const MaterialFeatureData &featureData,
+void ShaderGen::generateShader( const MaterialFeatureData& featureData,
-                                char *vertFile,
+                                ShaderData* shaderData,
-                                char *pixFile,
-                                F32 *pixVersion,
                                 const GFXVertexFormat *vertexFormat,
                                 const char* cacheName,
                                 Vector<GFXShaderMacro> &macros)
@@ -150,69 +202,111 @@ void ShaderGen::generateShader( const MaterialFeatureData &featureData,
 
    mFeatureData = featureData;
    mVertexFormat = vertexFormat;
+   mInstancingFormat.clear();
 
    _uninit();
    _init();
 
-   char vertShaderName[256];
+   const bool skipRegen = !Con::getBoolVariable("ShaderGen::GenNewShaders", true);
-   char pixShaderName[256];
+   const FeatureSet& features = mFeatureData.features;
+   U32 stages = 0;
 
-   // Note:  We use a postfix of _V/_P here so that it sorts the matching
+   // loop through and see which stages this featureset is expecting to make.
-   // vert and pixel shaders together when listed alphabetically.
+   for (U32 i = 0; i < features.getCount(); i++)
-   dSprintf( vertShaderName, sizeof(vertShaderName), "shadergen:/%s_V.%s", cacheName, mFileEnding.c_str() );
-   dSprintf( pixShaderName, sizeof(pixShaderName), "shadergen:/%s_P.%s", cacheName, mFileEnding.c_str() );
-
-   dStrcpy( vertFile, vertShaderName, 256 );
-   dStrcpy( pixFile, pixShaderName, 256 );
-
-   // this needs to change - need to optimize down to ps v.1.1
-   *pixVersion = GFX->getPixelShaderVersion();
-
-   if ( !Con::getBoolVariable( "ShaderGen::GenNewShaders", true ) )
    {
-      // If we are not regenerating the shader we will return here.
-      // But we must fill in the shader macros first!
 
-      _processVertFeatures( macros, true );
+      const FeatureType& type = features.getAt(i);
-      _processPixFeatures( macros, true );
+      ShaderFeature* feat = FEATUREMGR->getByType(type);
+      stages |= feat->getShaderStages();
 
-      return;
    }
 
-   // create vertex shader
+   for (U32 s = 0; s < (sizeof(gStageOrder) / sizeof(U32)); s++)
-   //------------------------
-   FileStream* s = new FileStream();
-   if(!s->open(vertShaderName, Torque::FS::File::Write ))
    {
-      AssertFatal(false, "Failed to open Shader Stream" );
+      U32 stage = gStageOrder[s];
-      return;
+
+      // skip unused stages
+      if (!(stages & stage))
+         continue;
+
+      GFXShaderStage curStage = (GFXShaderStage)stage;
+
+      char fileName[256];
+      const char* postfix = _getStagePostfix(curStage);
+      String stageName;
+
+      if (curStage & GFXShaderStage::VERTEX_SHADER)
+         stageName += vertexFormat->getDescription();
+
+      // build our filename.
+      for (U32 i = 0; i < features.getCount(); i++)
+      {
+         const FeatureType& type = features.getAt(i);
+         if (stage & FEATUREMGR->getByType(type)->getShaderStages())
+         {
+            stageName += type.getName().c_str();
+         }
+      }
+
+      stageName = getHashForName(stageName);
+      stageName += postfix;
+
+      FileCacheSet::iterator file = mFileCache.find(stageName);
+      if (file != mFileCache.end())
+      {
+         // set the shaderdata file for this stage, shaderdata ptr needs to be passed in here.
+         dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str());
+         shaderData->setShaderStageFile(curStage, fileName);
+         continue;
+      }
+
+      mFileCache[stageName] = true;
+
+      dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str());
+
+      shaderData->setShaderStageFile(curStage, fileName);
+
+      mOutput = new MultiLine;
+      FileStream* stream = new FileStream();
+      if (!stream->open(fileName, Torque::FS::File::Write))
+      {
+         AssertFatal(false, "Failed to open Shader Stream");
+         return;
+      }
+
+      switch (curStage)
+      {
+      case VERTEX_SHADER:
+         _processVertFeatures(macros, skipRegen);
+         if (skipRegen)
+            continue;
+         _printVertShader(*stream);
+         ((ShaderConnector*)mComponents[C_CONNECTOR])->reset();
+         break;
+      case PIXEL_SHADER:
+         _processPixFeatures(macros, skipRegen);
+         if (skipRegen)
+            continue;
+         _printPixShader(*stream);
+         break;
+      case GEOMETRY_SHADER:
+         break;
+      case DOMAIN_SHADER:
+         break;
+      case HULL_SHADER:
+         break;
+      case COMPUTE_SHADER:
+         break;
+      case ALL_STAGES:
+         break;
+      default:
+         break;
+      }
+
+      delete stream;
+      LangElement::deleteElements();
+
    }
-
-   mOutput = new MultiLine;
-   mInstancingFormat.clear();
-   _processVertFeatures(macros);
-   _printVertShader( *s );
-   delete s;
-
-   ((ShaderConnector*)mComponents[C_CONNECTOR])->reset();
-   LangElement::deleteElements();
-
-   // create pixel shader
-   //------------------------
-   s = new FileStream();
-   if(!s->open(pixShaderName, Torque::FS::File::Write ))
-   {
-      AssertFatal(false, "Failed to open Shader Stream" );
-      delete s;
-      return;
-   }
-
-   mOutput = new MultiLine;
-   _processPixFeatures(macros);
-   _printPixShader( *s );
-
-   delete s;
-   LangElement::deleteElements();
 }
 
 void ShaderGen::_init()
@@ -267,7 +361,7 @@ void ShaderGen::_processVertFeatures( Vector<GFXShaderMacro> &macros, bool macro
       else
          feature = FEATUREMGR->getByType( type );
 
-      if ( feature )
+      if ( feature && (feature->getShaderStages() & GFXShaderStage::VERTEX_SHADER))
       {
          feature->setProcessIndex( index );
 
@@ -315,7 +409,7 @@ void ShaderGen::_processPixFeatures( Vector<GFXShaderMacro> &macros, bool macros
          feature = FEATUREMGR->createFeature(type, args);
       else
          feature = FEATUREMGR->getByType(type);
-      if ( feature )
+      if ( feature && (feature->getShaderStages() & GFXShaderStage::PIXEL_SHADER))
       {
          feature->setProcessIndex( index );
 
@@ -377,7 +471,7 @@ void ShaderGen::_printFeatureList(Stream &stream)
    mPrinter->printLine(stream, "");
 }
 
-void ShaderGen::_printDependencies(Stream &stream)
+void ShaderGen::_printDependencies(Stream &stream, GFXShaderStage stage)
 {
    Vector<const ShaderDependency *> dependencies;
 
@@ -415,7 +509,7 @@ void ShaderGen::_printDependencies(Stream &stream)
       mPrinter->printLine(stream, "// Dependencies:");
 
       for( S32 i = 0; i < dependencies.size(); i++ )
-         dependencies[i]->print( stream );
+         dependencies[i]->print( stream, stage);
 
       mPrinter->printLine(stream, "");
    }
@@ -430,7 +524,7 @@ void ShaderGen::_printVertShader( Stream &stream )
 {
    mPrinter->printShaderHeader(stream);
 
-   _printDependencies(stream); // TODO: Split into vert and pix dependencies?
+   _printDependencies(stream, GFXShaderStage::VERTEX_SHADER); // TODO: Split into vert and pix dependencies?
    _printFeatureList(stream);
 
    // print out structures
@@ -452,7 +546,7 @@ void ShaderGen::_printPixShader( Stream &stream )
 {
    mPrinter->printShaderHeader(stream);
 
-   _printDependencies(stream); // TODO: Split into vert and pix dependencies?
+   _printDependencies(stream, GFXShaderStage::PIXEL_SHADER); // TODO: Split into vert and pix dependencies?
    _printFeatureList(stream);
 
    mComponents[C_CONNECTOR]->print( stream, false );
@@ -469,60 +563,60 @@ void ShaderGen::_printPixShader( Stream &stream )
    mPrinter->printPixelShaderCloser(stream);
 }
 
-GFXShader* ShaderGen::getShader( const MaterialFeatureData &featureData, const GFXVertexFormat *vertexFormat, const Vector<GFXShaderMacro> *macros, const Vector<String> &samplers )
+GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GFXVertexFormat* vertexFormat, const Vector<GFXShaderMacro>* macros, const Vector<String>& samplers)
 {
-   PROFILE_SCOPE( ShaderGen_GetShader );
+   PROFILE_SCOPE(ShaderGen_GetShader);
 
-   const FeatureSet &features = featureData.codify();
+   const FeatureSet& features = featureData.codify();
 
    // Build a description string from the features
    // and vertex format combination ( and macros ).
    String shaderDescription = vertexFormat->getDescription() + features.getDescription();
-   // Generate a single 64bit hash from the description string.
-   //
-   // Don't get paranoid!  This has 1 in 18446744073709551616
-   // chance for collision... it won't happen in this lifetime.
-   //
-   shaderDescription.replace("\n", " ");
-   U64 hash = Torque::hash64( (const U8*)shaderDescription.c_str(), shaderDescription.length(), 0 );
-   hash = convertHostToLEndian(hash);
-   U32 high = (U32)( hash >> 32 );
-   U32 low = (U32)( hash & 0x00000000FFFFFFFF );
-   String cacheKey = String::ToString( "%x%x", high, low );
-   // return shader if exists
-   GFXShader *match = mProcShaders[cacheKey];
-   if ( match )
-      return match;
 
-   // if not, then create it
+   String cacheKey = getHashForName(shaderDescription);
-   char vertFile[256];
-   char pixFile[256];
-   F32  pixVersion;
 
-   Vector<GFXShaderMacro> shaderMacros;
+   ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey);
-   shaderMacros.push_back( GFXShaderMacro( "TORQUE_SHADERGEN" ) );
+   if (dat != mProcShaderData.end())
-   if ( macros )
-      shaderMacros.merge( *macros );
-   generateShader( featureData, vertFile, pixFile, &pixVersion, vertexFormat, cacheKey, shaderMacros );
-
-   GFXShader *shader = GFX->createShader();
-   shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, vertFile);
-   shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, pixFile);
-
-   if (!shader->init(pixVersion, shaderMacros, samplers, &mInstancingFormat))
    {
-      delete shader;
+      Vector<GFXShaderMacro> shaderMacros;
-      return NULL;
+      shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN"));
+      if (macros)
+         shaderMacros.merge(*macros);
+
+      // should we loop vertex shader features to build mInstancingFormat before sending it down to see old hob?
+      return dat->value->getShader(shaderMacros);
    }
 
-   mProcShaders[cacheKey] = shader;
+   ShaderData* shaderData = new ShaderData;
 
-   return shader;
+   shaderData->setPixVersion(GFX->getPixelShaderVersion());
+
+   for (U32 samp = 0; samp < samplers.size(); samp++)
+   {
+      shaderData->setSamplerName(samplers[samp], samp);
+   }
+
+   Vector<GFXShaderMacro> shaderMacros;
+   shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN"));
+   if (macros)
+      shaderMacros.merge(*macros);
+
+   generateShader(featureData, shaderData, vertexFormat, cacheKey, shaderMacros);
+
+   shaderData->setInstancingFormat(&mInstancingFormat);
+   shaderData->registerObject();
+
+   mProcShaderData[cacheKey] = shaderData;
+   return shaderData->getShader(shaderMacros);
 }
 
 void ShaderGen::flushProceduralShaders()
 {
-   // The shaders are reference counted, so we
+   for (auto data : mProcShaderData)
-   // just need to clear the map.
+   {
-   mProcShaders.clear();
+      data.value->deleteObject();
+   }
+
+   mProcShaderData.clear();
+   mFileCache.clear();
 }

Engine/source/shaderGen/shaderGen.h

@@ -46,6 +46,10 @@
 #ifndef _MATERIALFEATUREDATA_H_
 #include "materials/materialFeatureData.h"
 #endif
+#ifndef _SHADERDATA_H_
+#include "materials/shaderData.h"
+#endif // !_SHADERDATA_H_
+
 
 /// Base class used by shaderGen to be API agnostic.  Subclasses implement the various methods
 /// in an API specific way.
@@ -145,13 +149,11 @@ public:
    /// the vertex and pixel shader files.  pixVersion is also filled in by
    /// this function.
    /// @param assignNum used to assign a specific number as the filename   
-   void generateShader( const MaterialFeatureData &featureData,
+   void generateShader( const MaterialFeatureData& featureData,
-                        char *vertFile, 
+                        ShaderData* shaderData,
-                        char *pixFile, 
+                        const GFXVertexFormat* vertexFormat,
-                        F32 *pixVersion,
-                        const GFXVertexFormat *vertexFormat,
                         const char* cacheName,
-                        Vector<GFXShaderMacro> &macros);
+                        Vector<GFXShaderMacro>& macros);
 
    // Returns a shader that implements the features listed by dat.
    GFXShader* getShader( const MaterialFeatureData &dat, const GFXVertexFormat *vertexFormat, const Vector<GFXShaderMacro> *macros, const Vector<String> &samplers );
@@ -192,9 +194,14 @@ protected:
    bool mRegisteredWithGFX;
    Torque::FS::FileSystemRef mMemFS;
 
-   /// Map of cache string -> shaders
+   /// <summary>
-   typedef Map<String, GFXShaderRef> ShaderMap;
+   /// Map of shaderdata, string should be built up of stage files
-   ShaderMap mProcShaders;
+   /// </summary>
+   typedef HashMap<String, SimObjectPtr<ShaderData>> ShaderDataMap;
+   ShaderDataMap mProcShaderData;
+
+   typedef HashMap<String, bool> FileCacheSet; // we use a hashmap because it is quicker for finding.
+   FileCacheSet mFileCache;
 
    ShaderGen();
 
@@ -215,7 +222,7 @@ protected:
    /// print out the processed features to the file stream
    void _printFeatures( Stream &stream );
 
-   void _printDependencies( Stream &stream );
+   void _printDependencies( Stream &stream, GFXShaderStage stage);
 
    void _processPixFeatures( Vector<GFXShaderMacro> &macros, bool macrosOnly = false );
    void _printPixShader( Stream &stream );

Engine/source/terrain/hlsl/terrFeatureHLSL.h

@@ -154,6 +154,8 @@ public:
    void processPix( Vector<ShaderComponent*> &componentList, 
                             const MaterialFeatureData &fd ) override;
 
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
+          
    Resources getResources( const MaterialFeatureData &fd ) override;
 
    String getName() override { return "Terrain Lightmap Texture"; }
@@ -182,6 +184,7 @@ public:
    void processPix(Vector<ShaderComponent*> &componentList,
       const MaterialFeatureData &fd) override;
    String getName() override { return "Blank Matinfo map"; }
+   U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; }
 };
 
 class TerrainHeightMapBlendHLSL : public TerrainFeatHLSL