From 7d3b4d8ac9f1b567ad9a15b85760969500966d98 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Sat, 8 Nov 2025 20:14:51 +0000 Subject: [PATCH 01/17] shader nodes Shader gen unification Shader node features can now be initialized with parameters set through FEATUREMGR This is to facilitate creation of node features from script. ShaderNode parameters are now no longer sent around as a void* and now use base type FeatureParamsBase --- Engine/source/gfx/gfxAPI.cpp | 34 ++ Engine/source/gfx/gfxAPI.h | 1 + .../source/shaderGen/NODE/shaderGenNodes.cpp | 302 ++++++++++++++++++ Engine/source/shaderGen/NODE/shaderGenNodes.h | 140 ++++++++ Engine/source/shaderGen/featureMgr.cpp | 61 +++- Engine/source/shaderGen/featureMgr.h | 71 +++- Engine/source/shaderGen/featureSet.cpp | 4 +- Engine/source/shaderGen/featureSet.h | 9 +- Engine/source/shaderGen/langElement.cpp | 33 ++ Engine/source/shaderGen/langElement.h | 1 + Engine/source/shaderGen/shaderGen.cpp | 6 +- 11 files changed, 650 insertions(+), 12 deletions(-) create mode 100644 Engine/source/shaderGen/NODE/shaderGenNodes.cpp create mode 100644 Engine/source/shaderGen/NODE/shaderGenNodes.h diff --git a/Engine/source/gfx/gfxAPI.cpp b/Engine/source/gfx/gfxAPI.cpp index be9564732..be05be46f 100644 --- a/Engine/source/gfx/gfxAPI.cpp +++ b/Engine/source/gfx/gfxAPI.cpp @@ -180,3 +180,37 @@ ImplementEnumType( GFXBlendOp, { GFXBlendOpMax, "GFXBlendOpMax" } EndImplementEnumType; + +ImplementEnumType(GFXShaderConstType, + "The shader const types.\n" + "@ingroup GFX") + + { GFXSCT_Uknown, "GFXSCT_Uknown" }, + { GFXSCT_ConstBuffer, "GFXSCT_ConstBuffer" }, + { GFXSCT_Float, "GFXSCT_Float" }, + { GFXSCT_Float2, "GFXSCT_Float2" }, + { GFXSCT_Float3, "GFXSCT_Float3" }, + { GFXSCT_Float4, "GFXSCT_Float4" }, + { GFXSCT_Float2x2, "GFXSCT_Float2x2" }, + { GFXSCT_Float3x3, "GFXSCT_Float3x3" }, + { GFXSCT_Float3x4, "GFXSCT_Float3x4" }, + { GFXSCT_Float4x3, "GFXSCT_Float4x3" }, + { GFXSCT_Float4x4, "GFXSCT_Float4x4" }, + { GFXSCT_Int, "GFXSCT_Int" }, + { GFXSCT_Int2, "GFXSCT_Int2" }, + { GFXSCT_Int3, "GFXSCT_Int3" }, + { GFXSCT_Int4, "GFXSCT_Int4" }, + { GFXSCT_UInt, "GFXSCT_UInt" }, + { GFXSCT_UInt2, "GFXSCT_UInt2" }, + { GFXSCT_UInt3, "GFXSCT_UInt3" }, + { GFXSCT_UInt4, "GFXSCT_UInt4" }, + { GFXSCT_Bool, "GFXSCT_Bool" }, + { GFXSCT_Bool2, "GFXSCT_Bool2" }, + { GFXSCT_Bool3, "GFXSCT_Bool3" }, + { GFXSCT_Bool4, "GFXSCT_Bool4" }, + { GFXSCT_Sampler, "GFXSCT_Sampler" }, + { GFXSCT_SamplerCube, "GFXSCT_SamplerCube" }, + { GFXSCT_SamplerCubeArray, "GFXSCT_SamplerCubeArray" }, + { GFXSCT_SamplerTextureArray, "GFXSCT_SamplerTextureArray" } + +EndImplementEnumType; diff --git a/Engine/source/gfx/gfxAPI.h b/Engine/source/gfx/gfxAPI.h index 2c7f420d0..a3610c3c4 100644 --- a/Engine/source/gfx/gfxAPI.h +++ b/Engine/source/gfx/gfxAPI.h @@ -57,5 +57,6 @@ DefineConsoleType( TypeGFXTextureFilterType, GFXTextureFilterType ); DefineConsoleType( TypeGFXCullMode, GFXCullMode ); DefineConsoleType( TypeGFXStencilOp, GFXStencilOp ); DefineConsoleType( TypeGFXBlendOp, GFXBlendOp ); +DefineConsoleType( TypeGFXShaderConstType, GFXShaderConstType); #endif // !_GFXAPI_H_ diff --git a/Engine/source/shaderGen/NODE/shaderGenNodes.cpp b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp new file mode 100644 index 000000000..011d29e38 --- /dev/null +++ b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp @@ -0,0 +1,302 @@ +#include "platform/platform.h" +#include "shaderGen/shaderGen.h" + +#include "shaderGen/NODE/shaderGenNodes.h" +#include "shaderGen/langElement.h" +#include "shaderGen/shaderOp.h" +#include "shaderGen/shaderGenVars.h" +#include "gfx/gfxDevice.h" +#include "materials/matInstance.h" +#include "materials/processedMaterial.h" +#include "materials/materialFeatureTypes.h" +#include "core/util/autoPtr.h" + +#include "core/module.h" +#include "materials/materialFeatureTypes.h" + + +ImplementFeatureType(SNF_DefaultTexCoord, U32(-1), -1, false); +ImplementFeatureType(SNF_TextureFeature, U32(-1), -1, false); + +ImplementEnumType(ShaderNodeFeature_enum, "Shader node features. Each of thes relates to a specific node for generating a shader.\n\n") + { ShaderNodeFeature_enum::eSNF_DefaultTexCoord, "SNF_DefaultTexCoord", "Setup the default texcoord." }, + { ShaderNodeFeature_enum::eSNF_TextureFeature, "SNF_TextureFeature", "Sample a Texture - Params: (string,string,GFXSamplerStateData,bool)." }, +EndImplementEnumType; + +namespace +{ + void register_node_features(GFXAdapterType type) + { + FEATUREMGR->registerFeature(SNF_DefaultTexCoord, new DefaultTexcoordFeature); + FEATUREMGR->registerFeature(SNF_TextureFeature, new TextureFeature, TextureFeature::createFunction); + } + +}; + +MODULE_BEGIN(ShaderGenNodes) + +MODULE_INIT_AFTER(ShaderGen) +MODULE_INIT_AFTER(ShaderGenFeatureMgr) + +MODULE_INIT +{ + SHADERGEN->getFeatureInitSignal().notify(®ister_node_features); +} + +MODULE_END; + +void ShaderFeatureNode::setupTextureSample( const String& samplerName, + GFXShaderConstType samplerType, + Vector& componentList, + MultiLine* meta, + LangElement* texCoord, + LangElement* compareValue, + bool useGather) +{ + const bool isGL = (GFX->getAdapterType() == OpenGL); + const bool isComparison = (compareValue != NULL); + + // ---- Create or find texture/sampler vars ---- + String texVarName = samplerName + "_tex"; + String sampVarName = samplerName + "_sampler"; + + Var* textureVar = dynamic_cast(LangElement::find(texVarName)); + Var* samplerVar = dynamic_cast(LangElement::find(sampVarName)); + + if (!isGL) + { + // HLSL requires both Texture + SamplerState + if (!samplerVar) + { + samplerVar = new Var; + samplerVar->setType(isComparison ? "SamplerComparisonState" : "SamplerState"); + samplerVar->setName(sampVarName); + samplerVar->uniform = true; + samplerVar->sampler = true; + samplerVar->constNum = Var::getTexUnitNum(); + } + + if (!textureVar) + { + textureVar = new Var; + textureVar->setType(LangElement::samplerTypeToString(samplerType)); // Texture2D, TextureCube, etc. + textureVar->setName(texVarName); + textureVar->uniform = true; + textureVar->texture = true; + textureVar->constNum = samplerVar->constNum; + } + } + else + { + // GLSL uses a single sampler uniform + if (!textureVar) + { + textureVar = new Var; + textureVar->setType(LangElement::samplerTypeToString(samplerType)); + textureVar->setName(texVarName); + textureVar->uniform = true; + textureVar->sampler = true; + textureVar->constNum = Var::getTexUnitNum(); + } + } + + // ---- Emit sampling code ---- + String sampleFunc; + if (isComparison) + { + if (useGather) + sampleFunc = isGL ? "textureGather" : "SampleCmpGather"; + else + sampleFunc = isGL ? "texture" : "SampleCmp"; + } + else + { + sampleFunc = isGL ? "texture" : "Sample"; + } + + // The sampled color variable (e.g. "diffuseColor") + Var* sampledColor = new Var; + sampledColor->setType("float4"); + sampledColor->setName(samplerName); // The result var will be named like the sampler + + if (isGL) + { + if (isComparison) + { + meta->addStatement(new GenOp(" @ = %s(@, @, @);\r\n", + sampledColor, sampleFunc.c_str(), textureVar, texCoord, compareValue)); + } + else + { + meta->addStatement(new GenOp(" @ = %s(@, @);\r\n", + sampledColor, sampleFunc.c_str(), textureVar, texCoord)); + } + } + else + { + if (isComparison) + { + if (useGather) + meta->addStatement(new GenOp(" @ = @.%s(@, @, @);\r\n", + sampledColor, textureVar, sampleFunc.c_str(), samplerVar, texCoord, compareValue)); + else + meta->addStatement(new GenOp(" @ = @.%s(@, @, @);\r\n", + sampledColor, textureVar, sampleFunc.c_str(), samplerVar, texCoord, compareValue)); + } + else + { + meta->addStatement(new GenOp(" @ = @.%s(@, @);\r\n", + sampledColor, textureVar, sampleFunc.c_str(), samplerVar, texCoord)); + } + } +} + +Var* ShaderFeatureNode::getOutTexCoord(const char* name, GFXShaderConstType type, bool useTexAnim, MultiLine* meta, Vector& componentList) +{ + String outTexName = String::ToString("out_%s", name); + Var* texCoord = (Var*)LangElement::find(outTexName); + if (!texCoord) + { + Var* inTex = getVertTexCoord(name); + AssertFatal(inTex, "ShaderFeatureNode::getOutTexCoord - Unknown vertex input coord!"); + + ShaderConnector* connectComp = dynamic_cast(componentList[C_CONNECTOR]); + + texCoord = connectComp->getElement(RT_TEXCOORD); + texCoord->setName(outTexName); + texCoord->setStructName("OUT"); + texCoord->setType(type); + + // Statement allows for casting of different types which + // eliminates vector truncation problems. + String statement = String::ToString(" @ = (%s)@;\r\n", type); + meta->addStatement(new GenOp(statement, texCoord, inTex)); + + } + + return texCoord; +} + +LangElement* ShaderFeatureNode::setupTexSpaceMat(Vector& componentList, Var** texSpaceMat) +{ + return nullptr; +} + +LangElement* ShaderFeatureNode::assignColor(LangElement* elem, Material::BlendOp blend, LangElement* lerpElem, ShaderFeature::OutputTarget outputTarget) +{ + // search for color var + Var* color = (Var*)LangElement::find(getOutputTargetVarName(outputTarget)); + + if (!color) + { + // create color var + color = new Var; + color->setType("fragout"); + color->setName(getOutputTargetVarName(outputTarget)); + color->setStructName("OUT"); + + return new GenOp("@ = @", color, elem); + } + + switch (blend) + { + case Material::Add: + return new GenOp("@ += @", color, elem); + break; + + case Material::Sub: + return new GenOp("@ -= @", color, elem); + break; + + case Material::Mul: + return new GenOp("@ *= @", color, elem); + break; + + case Material::PreMul: + return new GenOp("@.rgb = @.rgb + (@.rgb*(1.0-@.a))", color, elem, color, elem); + break; + + case Material::AddAlpha: + return new GenOp("@ += @ * @.a", color, elem, elem); + break; + + case Material::LerpAlpha: + if (!lerpElem) + lerpElem = elem; + return new GenOp("@.rgb = lerp( @.rgb, (@).rgb, (@).a )", color, color, elem, lerpElem); + break; + + case Material::ToneMap: + return new GenOp("@ = 1.0 - exp(-1.0 * @ * @)", color, color, elem); + break; + + case Material::None: + return new GenOp("@ = @", color, elem); + break; + + default: + AssertFatal(false, "Unrecognized color blendOp"); + // Fallthru + } + + return NULL; +} + +//-------------------------------------------------------- +// Setup the default texcoord +//-------------------------------------------------------- + +void DefaultTexcoordFeature::processVert(Vector& componentList, const MaterialFeatureData& fd) +{ + MultiLine* meta = new MultiLine; + getOutTexCoord("texCoord", + GFXSCT_Float2, + fd.features[MFT_TexAnim], + meta, + componentList); + output = meta; +} + +void DefaultTexcoordFeature::processPix(Vector& componentList, const MaterialFeatureData& fd) +{ + // grab connector texcoord register + Var* inTex = getInTexCoord("texCoord", GFXSCT_Float2, componentList); + if (!inTex) + return; +} + +//-------------------------------------------------------- +// TEXTURE SAMPLER FEATURE +//-------------------------------------------------------- + +void TextureFeature::processPix(Vector& componentList, const MaterialFeatureData& fd) +{ + // find the uv var. + Var* inTex = (Var*)LangElement::find(params->uvName); + if (!inTex) + return; + + MultiLine* meta = new MultiLine; + + // Sample texture + setupTextureSample( + params->samplerName, // name of the output variable. + params->samplerType, // or GFXSCT_SamplerCube, etc. + componentList, + meta, + inTex, + NULL, // compareValue (for SampleCmp) + params->useGather // enable gather if desired + ); + + output = meta; +} + +ShaderFeature::Resources TextureFeature::getResources(const MaterialFeatureData& fd) +{ + Resources res; + res.numTex = 1; + res.numTexReg = 1; + + return res; +} diff --git a/Engine/source/shaderGen/NODE/shaderGenNodes.h b/Engine/source/shaderGen/NODE/shaderGenNodes.h new file mode 100644 index 000000000..8cef15786 --- /dev/null +++ b/Engine/source/shaderGen/NODE/shaderGenNodes.h @@ -0,0 +1,140 @@ +#pragma once + +#ifndef _SHADERFEATURE_H_ +#include "shaderGen/shaderFeature.h" +#endif + +#ifndef _FEATUREMGR_H_ +#include "shaderGen/featureMgr.h" +#endif + +#ifndef _FEATURETYPE_H_ +#include "shaderGen/featureType.h" +#endif + +#ifndef __GFXSTATEBLOCKDATA_H_ +#include "gfx/sim/gfxStateBlockData.h" +#endif + +#ifndef _GFXAPI_H_ +#include "gfx/gfxAPI.h" +#endif + +DeclareFeatureType(SNF_DefaultTexCoord); +DeclareFeatureType(SNF_TextureFeature); + +/// +/// This enum is so we can map to nodes in script. +/// +enum ShaderNodeFeature_enum +{ + eSNF_DefaultTexCoord, + eSNF_TextureFeature, +}; + +DefineEnumType(ShaderNodeFeature_enum); + +class ShaderFeatureNode : public ShaderFeature +{ +public: + void setupTextureSample(const String& samplerName, + GFXShaderConstType samplerType, + Vector& componentList, + MultiLine* meta, + LangElement* texCoord, + LangElement* compareValue, + bool useGather); + + Var* getOutTexCoord( const char* name, + GFXShaderConstType type, + bool useTexAnim, + MultiLine* meta, + Vector& componentList); + + LangElement* setupTexSpaceMat(Vector& componentList, Var** texSpaceMat) override; + + LangElement* assignColor(LangElement* elem, Material::BlendOp blend, LangElement* lerpElem = NULL, ShaderFeature::OutputTarget outputTarget = ShaderFeature::DefaultTarget) override; +}; + +/// +/// Parameters for the TextureFeature +/// +struct TextureFeatureParams : public FeatureParamsBase +{ + String samplerName; + GFXShaderConstType samplerType; + GFXSamplerStateData samplerState; + String uvName; + bool useGather; + + TextureFeatureParams() + { + samplerName = "defaultSampler"; + samplerType = GFXSCT_Sampler; + uvName = "texCoord"; + useGather = false; + + } + + const char* getFeatureParamTypeName() const override { return "TextureFeatureParams"; } + + static void persistedFields(Vector& fields) + { + addParam(fields, "sampler", Offset(samplerName, TextureFeatureParams), TypeString); + addParam(fields, "samplerType", Offset(samplerType, TextureFeatureParams), TypeGFXShaderConstType); + addParam(fields, "samplerState", Offset(samplerState, TextureFeatureParams), TYPEID()); + addParam(fields, "uvName", Offset(uvName, TextureFeatureParams), TypeString); + addParam(fields, "useGather", Offset(useGather, TextureFeatureParams), TypeBool); + } +}; + +REGISTER_FEATURE_PARAMS(SNF_TextureFeature, TextureFeatureParams) + +class DefaultTexcoordFeature : public ShaderFeatureNode +{ + void processVert(Vector& componentList, + const MaterialFeatureData& fd) override; + + void processPix(Vector& componentList, + const MaterialFeatureData& fd) override; + + String getName() override + { + return "Default TexCoord"; + } +}; + +class TextureFeature : public ShaderFeatureNode +{ +private: + /// Parameters that this feature can use to change the shadergen output. + TextureFeatureParams* params; +public: + /// default constructor + TextureFeature() + { + params = new TextureFeatureParams(); + } + /// Constructor that takes params as an argument + TextureFeature(TextureFeatureParams* inParams) + { + params = inParams; + } + + void processPix(Vector& componentList, + const MaterialFeatureData& fd) override; + + ShaderFeature::Resources getResources(const MaterialFeatureData& fd); + + // create a static function on the feature class + static ShaderFeature* createFunction(FeatureParamsBase* args) + { + TextureFeatureParams* params = static_cast(args); + return new TextureFeature(params); + } + + String getName() override + { + return "Texture Sampler"; + } +}; diff --git a/Engine/source/shaderGen/featureMgr.cpp b/Engine/source/shaderGen/featureMgr.cpp index 7e742bf58..560c8ab6b 100644 --- a/Engine/source/shaderGen/featureMgr.cpp +++ b/Engine/source/shaderGen/featureMgr.cpp @@ -51,6 +51,7 @@ FeatureMgr::FeatureMgr() : mNeedsSort( false ) { VECTOR_SET_ASSOCIATION( mFeatures ); + VECTOR_SET_ASSOCIATION( mParamInfos ); } FeatureMgr::~FeatureMgr() @@ -96,7 +97,7 @@ ShaderFeature* FeatureMgr::getByType( const FeatureType &type ) return NULL; } -ShaderFeature* FeatureMgr::createFeature(const FeatureType& type, void* argStruct) +ShaderFeature* FeatureMgr::createFeature(const FeatureType& type, FeatureParamsBase* argStruct) { FeatureInfoVector::iterator iter = mFeatures.begin(); @@ -114,6 +115,64 @@ ShaderFeature* FeatureMgr::createFeature(const FeatureType& type, void* argStruc return nullptr; } +void FeatureMgr::registerFeatureParams(const FeatureType& type, const FeatureParamField* fields, U32 fieldCount, CreateFeatureParams createFn) +{ + // Replace if already exists + for (U32 i = 0; i < mParamInfos.size(); ++i) + { + if (*mParamInfos[i].type == type) + { + mParamInfos.erase(i); + break; + } + } + + mParamInfos.increment(); + mParamInfos.last().type = &type; + mParamInfos.last().fields = fields; + mParamInfos.last().fieldCount = fieldCount; + mParamInfos.last().createFn = createFn; + +} + +FeatureParamsBase* FeatureMgr::createFeatureParams(const FeatureType& type) const +{ + for (U32 i = 0; i < mParamInfos.size(); ++i) + { + if (*mParamInfos[i].type == type) + return mParamInfos[i].createFn(); + } + return nullptr; +} + +void FeatureMgr::applyFeatureParams(const FeatureType& type, + FeatureParamsBase* params, + const Vector& args) const +{ + const FeatureParamInfo* info = nullptr; + for (U32 i = 0; i < mParamInfos.size(); ++i) + { + if (*mParamInfos[i].type == type) + { + info = &mParamInfos[i]; + break; + } + } + + if (!info || !params) + return; + + for (U32 i = 0; i < info->fieldCount; ++i) + { + const FeatureParamField& f = info->fields[i]; + const char* val = args[i].c_str(); + + void* fieldPtr = (U8*)params + f.offset; + // no array support yet. + Con::setData(f.type, fieldPtr, 0, 1, &val); + } +} + void FeatureMgr::registerFeature( const FeatureType &type, ShaderFeature *feature, CreateShaderFeatureDelegate featureDelegate) diff --git a/Engine/source/shaderGen/featureMgr.h b/Engine/source/shaderGen/featureMgr.h index 0b4eeecf1..51daf6667 100644 --- a/Engine/source/shaderGen/featureMgr.h +++ b/Engine/source/shaderGen/featureMgr.h @@ -36,7 +36,48 @@ class FeatureType; class ShaderFeature; -typedef Delegate CreateShaderFeatureDelegate; +struct FeatureParamField +{ + StringTableEntry paramName; + S32 offset; + S32 type; + U32 arraySize; +}; + +inline void addParam(Vector& list, + const char* name, + S32 offset, + S32 consoleType, + U32 arraySize = 1) +{ + FeatureParamField f = { name, offset, consoleType, arraySize }; + list.push_back(f); +} + +/// +/// Base class for all shader feature parameter structs. +/// +class FeatureParamsBase +{ +public: + virtual ~FeatureParamsBase() {} + + // For debug or script reflection, you can override to serialize/print parameters + virtual const char* getFeatureParamTypeName() const { return "FeatureParamsBase"; } +}; + +typedef Delegate CreateFeatureParams; + +/// Metadata for a parameter struct type. +struct FeatureParamInfo +{ + const FeatureType* type; // Matches feature + const FeatureParamField* fields; + U32 fieldCount; + CreateFeatureParams createFn; // makes a new param struct +}; + +typedef Delegate CreateShaderFeatureDelegate; /// /// Used by the feature manager. @@ -60,9 +101,11 @@ protected: bool mNeedsSort; typedef Vector FeatureInfoVector; - FeatureInfoVector mFeatures; + typedef Vector FeatureParamInfoVector; + FeatureParamInfoVector mParamInfos; + static S32 QSORT_CALLBACK _featureInfoCompare( const FeatureInfo *a, const FeatureInfo *b ); public: @@ -87,7 +130,13 @@ public: /// An instance of the shader feature using its static createFunction taking in the /// argument struct. /// - ShaderFeature* createFeature(const FeatureType& type, void* argStruct); + ShaderFeature* createFeature(const FeatureType& type, FeatureParamsBase* argStruct); + + void registerFeatureParams(const FeatureType& type, const FeatureParamField* fields, U32 fieldCount, CreateFeatureParams createFn); + + FeatureParamsBase* createFeatureParams(const FeatureType& type) const; + + void applyFeatureParams(const FeatureType& type, FeatureParamsBase* params, const Vector& args) const; /// /// Allows other systems to add features. index is @@ -114,4 +163,20 @@ public: // Helper for accessing the feature manager singleton. #define FEATUREMGR ManagedSingleton::instance() +#define REGISTER_FEATURE_PARAMS(TYPE, STRUCT_TYPE) \ + struct STRUCT_TYPE##_AutoRegister \ + { \ + STRUCT_TYPE##_AutoRegister() \ + { \ + Vector fieldList; \ + STRUCT_TYPE::persistedFields(fieldList); \ + FEATUREMGR->registerFeatureParams( \ + TYPE, \ + fieldList.address(), \ + fieldList.size(), \ + Delegate([]() -> FeatureParamsBase* { return new STRUCT_TYPE(); })\ + ); \ + } \ + } STRUCT_TYPE##_AutoRegisterInstance; + #endif // FEATUREMGR diff --git a/Engine/source/shaderGen/featureSet.cpp b/Engine/source/shaderGen/featureSet.cpp index f295536be..a6af95c55 100644 --- a/Engine/source/shaderGen/featureSet.cpp +++ b/Engine/source/shaderGen/featureSet.cpp @@ -82,7 +82,7 @@ const FeatureType& FeatureSet::getAt( U32 index, S32 *outIndex ) const return *mFeatures[index].type; } -void* FeatureSet::getArguments(U32 index) const +FeatureParamsBase* FeatureSet::getArguments(U32 index) const { if (mFeatures[index].argStruct) return mFeatures[index].argStruct; @@ -146,7 +146,7 @@ void FeatureSet::setFeature( const FeatureType &type, bool set, S32 index ) mDescription.clear(); } -void FeatureSet::addFeature( const FeatureType &type, S32 index, void* argStruct ) +void FeatureSet::addFeature( const FeatureType &type, S32 index, FeatureParamsBase* argStruct ) { if (!argStruct) { diff --git a/Engine/source/shaderGen/featureSet.h b/Engine/source/shaderGen/featureSet.h index b494492fd..299fbe200 100644 --- a/Engine/source/shaderGen/featureSet.h +++ b/Engine/source/shaderGen/featureSet.h @@ -29,6 +29,9 @@ #ifndef _TVECTOR_H_ #include "core/util/tVector.h" #endif +#ifndef _FEATUREMGR_H_ +#include "shaderGen/featureMgr.h" +#endif class FeatureType; @@ -42,7 +45,7 @@ protected: { const FeatureType* type; S32 index; - void* argStruct; + FeatureParamsBase* argStruct; }; /// The list of featurs. @@ -94,7 +97,7 @@ public: /// the feature index when it was added. const FeatureType& getAt( U32 index, S32 *outIndex = NULL ) const; - void* getArguments(U32 index) const; + FeatureParamsBase* getArguments(U32 index) const; /// Returns true if this handle has this feature. bool hasFeature( const FeatureType &type, S32 index = -1 ) const; @@ -108,7 +111,7 @@ public: /// The shader feature type. /// The inedx the shader feature will be sorted in the set. /// A struct representing arguments for a shader feature. - void addFeature( const FeatureType &type, S32 index = -1, void* argStruct = nullptr ); + void addFeature( const FeatureType &type, S32 index = -1, FeatureParamsBase* argStruct = nullptr ); /// void removeFeature( const FeatureType &type ); diff --git a/Engine/source/shaderGen/langElement.cpp b/Engine/source/shaderGen/langElement.cpp index 461a6d43c..ce74e303e 100644 --- a/Engine/source/shaderGen/langElement.cpp +++ b/Engine/source/shaderGen/langElement.cpp @@ -93,6 +93,39 @@ const char* LangElement::constTypeToString(GFXShaderConstType constType) return ""; } +const char* LangElement::samplerTypeToString(GFXShaderConstType constType) +{ + if (constType < GFXSCT_Sampler) + return ""; + + // Determine shader language based on GFXAdapterAPI + if (GFX->getAdapterType() == OpenGL) + { + switch (constType) + { + case GFXSCT_Sampler: return "sampler2D"; break; + case GFXSCT_SamplerCube: return "samplerCube"; break; + case GFXSCT_SamplerTextureArray: return "sampler2DArray"; break; + case GFXSCT_SamplerCubeArray: return "samplerCubeArray"; break; + default: return "unknown"; break; + } + } + else // Assume DirectX/HLSL + { + switch (constType) + { + case GFXSCT_Sampler: return "Texture2D"; break; + case GFXSCT_SamplerCube: return "TextureCube"; break; + case GFXSCT_SamplerTextureArray: return "Texture2DArray"; break; + case GFXSCT_SamplerCubeArray: return "TextureCubeArray"; break; + default: return "unknown"; break; + } + } + + return ""; +} + + //-------------------------------------------------------------------------- // Constructor //-------------------------------------------------------------------------- diff --git a/Engine/source/shaderGen/langElement.h b/Engine/source/shaderGen/langElement.h index c6f25b77a..89897f5a8 100644 --- a/Engine/source/shaderGen/langElement.h +++ b/Engine/source/shaderGen/langElement.h @@ -60,6 +60,7 @@ struct LangElement U8 name[32]; static const char* constTypeToString(GFXShaderConstType constType); + static const char* samplerTypeToString(GFXShaderConstType constType); LangElement(); virtual ~LangElement() {}; virtual void print( Stream &stream ){}; diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index 07781aea0..1f30654c6 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -257,7 +257,7 @@ void ShaderGen::_processVertFeatures( Vector ¯os, bool macro { S32 index; const FeatureType &type = features.getAt( i, &index ); - void* args = features.getArguments(i); + FeatureParamsBase* args = features.getArguments(i); ShaderFeature* feature = nullptr; if(args) feature = FEATUREMGR->createFeature(type, args); @@ -306,7 +306,7 @@ void ShaderGen::_processPixFeatures( Vector ¯os, bool macros { S32 index; const FeatureType &type = features.getAt( i, &index ); - void* args = features.getArguments(i); + FeatureParamsBase* args = features.getArguments(i); ShaderFeature* feature = nullptr; if (args) feature = FEATUREMGR->createFeature(type, args); @@ -353,7 +353,7 @@ void ShaderGen::_printFeatureList(Stream &stream) { S32 index; const FeatureType &type = features.getAt( i, &index ); - void* args = features.getArguments(i); + FeatureParamsBase* args = features.getArguments(i); ShaderFeature* feature = nullptr; if (args) feature = FEATUREMGR->createFeature(type, args); From dfbea4540d1df151f7735e4b9d4788a935c858ec Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Wed, 26 Nov 2025 05:16:26 +0000 Subject: [PATCH 02/17] output vars add output vars to the param types fixed gl bug need samplertype --- .../source/shaderGen/NODE/shaderGenNodes.cpp | 165 ++++++++++++------ Engine/source/shaderGen/NODE/shaderGenNodes.h | 110 ++++++++++-- 2 files changed, 206 insertions(+), 69 deletions(-) diff --git a/Engine/source/shaderGen/NODE/shaderGenNodes.cpp b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp index 011d29e38..8634bc32d 100644 --- a/Engine/source/shaderGen/NODE/shaderGenNodes.cpp +++ b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp @@ -15,12 +15,14 @@ #include "materials/materialFeatureTypes.h" -ImplementFeatureType(SNF_DefaultTexCoord, U32(-1), -1, false); -ImplementFeatureType(SNF_TextureFeature, U32(-1), -1, false); +ImplementFeatureType(SNF_DefaultTexCoord, U32(-1), -1, false); +ImplementFeatureType(SNF_TextureFeature, U32(-1), -1, false); +ImplementFeatureType(SNF_NormalMapFeature, U32(-1), -1, false); ImplementEnumType(ShaderNodeFeature_enum, "Shader node features. Each of thes relates to a specific node for generating a shader.\n\n") { ShaderNodeFeature_enum::eSNF_DefaultTexCoord, "SNF_DefaultTexCoord", "Setup the default texcoord." }, - { ShaderNodeFeature_enum::eSNF_TextureFeature, "SNF_TextureFeature", "Sample a Texture - Params: (string,string,GFXSamplerStateData,bool)." }, + { ShaderNodeFeature_enum::eSNF_TextureFeature, "SNF_TextureFeature", "Sample a Texture - Params: (string,string,GFXSamplerStateData,bool)." }, + { ShaderNodeFeature_enum::eSNF_NormalMapFeature,"SNF_NormalMapFeature", "Convert a texture to a normalmap - Params: (string,float,bool,bool)." }, EndImplementEnumType; namespace @@ -28,7 +30,11 @@ namespace void register_node_features(GFXAdapterType type) { FEATUREMGR->registerFeature(SNF_DefaultTexCoord, new DefaultTexcoordFeature); - FEATUREMGR->registerFeature(SNF_TextureFeature, new TextureFeature, TextureFeature::createFunction); + FEATUREMGR->registerFeature(SNF_TextureFeature, new TextureFeature, TextureFeature::createFunction); + FEATUREMGR->registerFeature(SNF_NormalMapFeature, new NormalMapFeature, NormalMapFeature::createFunction); + + REGISTER_FEATURE_PARAMS(SNF_TextureFeature, TextureFeatureParams); + REGISTER_FEATURE_PARAMS(SNF_NormalMapFeature, NormalMapFeatureParams); } }; @@ -58,14 +64,15 @@ void ShaderFeatureNode::setupTextureSample( const String& samplerName, // ---- Create or find texture/sampler vars ---- String texVarName = samplerName + "_tex"; - String sampVarName = samplerName + "_sampler"; + String sampVarName = samplerName; + String resultVarName = samplerName + "_col"; - Var* textureVar = dynamic_cast(LangElement::find(texVarName)); + Var* samplerVar = dynamic_cast(LangElement::find(sampVarName)); - - if (!isGL) + Var* textureVar; + + if (!isGL) // HLSL requires both Texture + SamplerState { - // HLSL requires both Texture + SamplerState if (!samplerVar) { samplerVar = new Var; @@ -76,6 +83,7 @@ void ShaderFeatureNode::setupTextureSample( const String& samplerName, samplerVar->constNum = Var::getTexUnitNum(); } + textureVar = dynamic_cast(LangElement::find(texVarName)); if (!textureVar) { textureVar = new Var; @@ -88,65 +96,80 @@ void ShaderFeatureNode::setupTextureSample( const String& samplerName, } else { - // GLSL uses a single sampler uniform - if (!textureVar) + if (!samplerVar) { - textureVar = new Var; - textureVar->setType(LangElement::samplerTypeToString(samplerType)); - textureVar->setName(texVarName); - textureVar->uniform = true; - textureVar->sampler = true; - textureVar->constNum = Var::getTexUnitNum(); + samplerVar = new Var; + samplerVar->setType(LangElement::samplerTypeToString(samplerType)); + samplerVar->setName(sampVarName); + samplerVar->uniform = true; + samplerVar->sampler = true; + samplerVar->constNum = Var::getTexUnitNum(); } } - // ---- Emit sampling code ---- - String sampleFunc; - if (isComparison) + // The sampled color variable (e.g. "samplerName_col") should always be new but just in case + Var* sampledColor = (Var*)LangElement::find(resultVarName); + if (!sampledColor) { - if (useGather) - sampleFunc = isGL ? "textureGather" : "SampleCmpGather"; - else - sampleFunc = isGL ? "texture" : "SampleCmp"; + sampledColor->setType(GFXSCT_Float4); + sampledColor->setName(resultVarName); // The result var will be named like the sampler + meta->addStatement(new GenOp(" @", new DecOp(sampledColor))); } else { - sampleFunc = isGL ? "texture" : "Sample"; + meta->addStatement(new GenOp(" @",sampledColor)); } - - // The sampled color variable (e.g. "diffuseColor") - Var* sampledColor = new Var; - sampledColor->setType("float4"); - sampledColor->setName(samplerName); // The result var will be named like the sampler - + if (isGL) { - if (isComparison) - { - meta->addStatement(new GenOp(" @ = %s(@, @, @);\r\n", - sampledColor, sampleFunc.c_str(), textureVar, texCoord, compareValue)); - } - else - { - meta->addStatement(new GenOp(" @ = %s(@, @);\r\n", - sampledColor, sampleFunc.c_str(), textureVar, texCoord)); - } - } - else - { + // ---------------- GLSL Sampling ---------------- if (isComparison) { if (useGather) - meta->addStatement(new GenOp(" @ = @.%s(@, @, @);\r\n", - sampledColor, textureVar, sampleFunc.c_str(), samplerVar, texCoord, compareValue)); + meta->addStatement(new GenOp( + " = textureGather(@, @, @);\r\n", + samplerVar, texCoord, compareValue)); else - meta->addStatement(new GenOp(" @ = @.%s(@, @, @);\r\n", - sampledColor, textureVar, sampleFunc.c_str(), samplerVar, texCoord, compareValue)); + meta->addStatement(new GenOp( + " = texture(@, @, @);\r\n", + samplerVar, texCoord, compareValue)); } else { - meta->addStatement(new GenOp(" @ = @.%s(@, @);\r\n", - sampledColor, textureVar, sampleFunc.c_str(), samplerVar, texCoord)); + if (useGather) + meta->addStatement(new GenOp( + " = textureGather(@, @);\r\n", + samplerVar, texCoord)); + else + meta->addStatement(new GenOp( + " = texture(@, @);\r\n", + samplerVar, texCoord)); + } + } + else + { + // ---------------- HLSL Sampling ---------------- + if (isComparison) + { + if (useGather) + meta->addStatement(new GenOp( + " = @.SampleCmpGather(@, @, @);\r\n", + textureVar, samplerVar, texCoord, compareValue)); + else + meta->addStatement(new GenOp( + " = @.SampleCmp(@, @, @);\r\n", + textureVar, samplerVar, texCoord, compareValue)); + } + else + { + if (useGather) + meta->addStatement(new GenOp( + " = @.Gather(@, @);\r\n", + textureVar, samplerVar, texCoord)); + else + meta->addStatement(new GenOp( + " = @.Sample(@, @);\r\n", + textureVar, samplerVar, texCoord)); } } } @@ -272,7 +295,7 @@ void DefaultTexcoordFeature::processPix(Vector& componentList, void TextureFeature::processPix(Vector& componentList, const MaterialFeatureData& fd) { // find the uv var. - Var* inTex = (Var*)LangElement::find(params->uvName); + Var* inTex = getInTexCoord(params->uvName, GFXSCT_Float2, componentList); if (!inTex) return; @@ -300,3 +323,43 @@ ShaderFeature::Resources TextureFeature::getResources(const MaterialFeatureData& return res; } + +//-------------------------------------------------------- +// NORMAL MAPPING FEATURE +//-------------------------------------------------------- + +void NormalMapFeature::processPix(Vector& componentList, const MaterialFeatureData& fd) +{ + String colorVarName = params->inputName; + Var* sampledColor = (Var*)LangElement::find(colorVarName); + + if (!sampledColor) + { + Con::warnf("NormalMapFeature: sampler %s not sampled yet!", params->inputName.c_str()); + return; // TextureFeature must run first + } + + + MultiLine* meta = new MultiLine; + + // TEMP float3 for base decoded normal + Var* tempNorm = new Var; + tempNorm->setName(params->inputName + "_normTemp"); + tempNorm->setType(GFXSCT_Float4); + LangElement* tempNormDecl = new DecOp(tempNorm); + + // sampledColor is the result of the textureFeature. + meta->addStatement(expandNormalMap(sampledColor, tempNormDecl, tempNorm, fd)); + + meta->addStatement( + new GenOp(" @.xy *= float2(@, @);\r\n", tempNorm, params->flipX ? -1.0f : 1.0f, params->flipY ? -1.0f : 1.0f)); + + meta->addStatement(new GenOp(" @.xyz = normalize( float3( @.xy * @, @.z ) );\r\n", + tempNorm, tempNorm, params->strength, tempNorm)); + + // write back into our known variable. + meta->addStatement(new GenOp(" @ = @;\r\n", sampledColor, tempNorm)); + + output = meta; + +} diff --git a/Engine/source/shaderGen/NODE/shaderGenNodes.h b/Engine/source/shaderGen/NODE/shaderGenNodes.h index 8cef15786..b1082f3d8 100644 --- a/Engine/source/shaderGen/NODE/shaderGenNodes.h +++ b/Engine/source/shaderGen/NODE/shaderGenNodes.h @@ -22,6 +22,7 @@ DeclareFeatureType(SNF_DefaultTexCoord); DeclareFeatureType(SNF_TextureFeature); +DeclareFeatureType(SNF_NormalMapFeature); /// /// This enum is so we can map to nodes in script. @@ -30,6 +31,7 @@ enum ShaderNodeFeature_enum { eSNF_DefaultTexCoord, eSNF_TextureFeature, + eSNF_NormalMapFeature, }; DefineEnumType(ShaderNodeFeature_enum); @@ -56,6 +58,25 @@ public: LangElement* assignColor(LangElement* elem, Material::BlendOp blend, LangElement* lerpElem = NULL, ShaderFeature::OutputTarget outputTarget = ShaderFeature::DefaultTarget) override; }; + +class DefaultTexcoordFeature : public ShaderFeatureNode +{ + void processVert(Vector& componentList, + const MaterialFeatureData& fd) override; + + void processPix(Vector& componentList, + const MaterialFeatureData& fd) override; + + String getName() override + { + return "DefaultTexCoord"; + } +}; + +//-------------------------------------------------------- +// TEXTURE SAMPLER FEATURE +//-------------------------------------------------------- + /// /// Parameters for the TextureFeature /// @@ -78,6 +99,12 @@ struct TextureFeatureParams : public FeatureParamsBase const char* getFeatureParamTypeName() const override { return "TextureFeatureParams"; } + /// + /// Texture features output variable is samplerName + "_col" + /// + /// The output variable for a texture feature. + const char* getOutputVar() const override { return samplerName + "_col"; } + static void persistedFields(Vector& fields) { addParam(fields, "sampler", Offset(samplerName, TextureFeatureParams), TypeString); @@ -88,22 +115,6 @@ struct TextureFeatureParams : public FeatureParamsBase } }; -REGISTER_FEATURE_PARAMS(SNF_TextureFeature, TextureFeatureParams) - -class DefaultTexcoordFeature : public ShaderFeatureNode -{ - void processVert(Vector& componentList, - const MaterialFeatureData& fd) override; - - void processPix(Vector& componentList, - const MaterialFeatureData& fd) override; - - String getName() override - { - return "Default TexCoord"; - } -}; - class TextureFeature : public ShaderFeatureNode { private: @@ -124,7 +135,7 @@ public: void processPix(Vector& componentList, const MaterialFeatureData& fd) override; - ShaderFeature::Resources getResources(const MaterialFeatureData& fd); + ShaderFeature::Resources getResources(const MaterialFeatureData& fd) override; // create a static function on the feature class static ShaderFeature* createFunction(FeatureParamsBase* args) @@ -135,6 +146,69 @@ public: String getName() override { - return "Texture Sampler"; + return "TextureSampler_" + params->samplerName + "_" + params->uvName; + } +}; + +//-------------------------------------------------------- +// NORMAL MAPPING FEATURE +//-------------------------------------------------------- + +/// +/// Parameters for the NormalMapFeature +/// +struct NormalMapFeatureParams : public FeatureParamsBase +{ + String inputName; // name of the sampled normal texture (Var* from TextureFeature) + F32 strength; // normal strength multiplier + bool flipX; + bool flipY; + + NormalMapFeatureParams() + { + inputName = "normalSampler"; + strength = 1.0f; + flipX = false; + flipY = false; + } + + const char* getFeatureParamTypeName() const override { return "NormalMapFeatureParams"; } + + /// + /// Normal map feature writes the result into the input name + /// + /// The output variable for a normal map feature. + const char* getOutputVar() const override { return inputName; } + + static void persistedFields(Vector& fields) + { + addParam(fields, "input", Offset(inputName, NormalMapFeatureParams), TypeString); + addParam(fields, "strength", Offset(strength, NormalMapFeatureParams), TypeF32); + addParam(fields, "flipX", Offset(flipX, NormalMapFeatureParams), TypeBool); + addParam(fields, "flipY", Offset(flipY, NormalMapFeatureParams), TypeBool); + } +}; + +class NormalMapFeature : public ShaderFeatureNode +{ +private: + NormalMapFeatureParams* params; + +public: + NormalMapFeature() { params = new NormalMapFeatureParams(); } + NormalMapFeature(NormalMapFeatureParams* p) { params = p; } + + void processPix(Vector& componentList, const MaterialFeatureData& fd) override; + + // normal map feature does not need any resources as these were created from the texture feature. + + static ShaderFeature* createFunction(FeatureParamsBase* args) + { + return new NormalMapFeature(static_cast(args)); + } + + String getName() override + { + return "NormalMap_" + params->inputName; } }; From cff7a8060d17e904cc309c425e69b7cafb81a532 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Fri, 28 Nov 2025 18:05:03 +0000 Subject: [PATCH 03/17] shadergen op classes Added CastOp logic, prints out a cast operation MatrixInitialization operation, takes input langelements and initializes a matrix Other functions added to langelement --- Engine/source/shaderGen/featureMgr.h | 1 + Engine/source/shaderGen/langElement.cpp | 182 ++++++++------- Engine/source/shaderGen/langElement.h | 57 ++++- Engine/source/shaderGen/shaderGen.cpp | 3 + Engine/source/shaderGen/shaderOp.cpp | 295 +++++++++++++++++++++++- Engine/source/shaderGen/shaderOp.h | 80 ++++++- 6 files changed, 514 insertions(+), 104 deletions(-) diff --git a/Engine/source/shaderGen/featureMgr.h b/Engine/source/shaderGen/featureMgr.h index 51daf6667..239b92828 100644 --- a/Engine/source/shaderGen/featureMgr.h +++ b/Engine/source/shaderGen/featureMgr.h @@ -62,6 +62,7 @@ class FeatureParamsBase public: virtual ~FeatureParamsBase() {} + virtual const char* getOutputVar() const { return "default"; } // For debug or script reflection, you can override to serialize/print parameters virtual const char* getFeatureParamTypeName() const { return "FeatureParamsBase"; } }; diff --git a/Engine/source/shaderGen/langElement.cpp b/Engine/source/shaderGen/langElement.cpp index ce74e303e..e0e197c9b 100644 --- a/Engine/source/shaderGen/langElement.cpp +++ b/Engine/source/shaderGen/langElement.cpp @@ -30,101 +30,107 @@ //************************************************************************** Vector LangElement::elementList( __FILE__, __LINE__ ); -const char* LangElement::constTypeToString(GFXShaderConstType constType) +static const ShaderTypeInfo ShaderTypes[] = { - // Determine shader language based on GFXAdapterAPI - if (GFX->getAdapterType() == OpenGL) - { - switch (constType) - { - case GFXSCT_Float: return "float"; break; - case GFXSCT_Float2: return "vec2"; break; - case GFXSCT_Float3: return "vec3"; break; - case GFXSCT_Float4: return "vec4"; break; - case GFXSCT_Float2x2: return "mat2"; break; - case GFXSCT_Float3x3: return "mat3"; break; - case GFXSCT_Float3x4: return "mat3x4"; break; - case GFXSCT_Float4x3: return "mat4x3"; break; - case GFXSCT_Float4x4: return "mat4"; break; - case GFXSCT_Int: return "int"; break; - case GFXSCT_Int2: return "ivec2"; break; - case GFXSCT_Int3: return "ivec3"; break; - case GFXSCT_Int4: return "ivec4"; break; - case GFXSCT_UInt: return "uint"; break; - case GFXSCT_UInt2: return "uvec2"; break; - case GFXSCT_UInt3: return "uvec3"; break; - case GFXSCT_UInt4: return "uvec4"; break; - case GFXSCT_Bool: return "bool"; break; - case GFXSCT_Bool2: return "bvec2"; break; - case GFXSCT_Bool3: return "bvec3"; break; - case GFXSCT_Bool4: return "bvec4"; break; - default: return "unknown"; break; - } - } - else // Assume DirectX/HLSL - { - switch (constType) - { - case GFXSCT_Float: return "float"; break; - case GFXSCT_Float2: return "float2"; break; - case GFXSCT_Float3: return "float3"; break; - case GFXSCT_Float4: return "float4"; break; - case GFXSCT_Float2x2: return "float2x2"; break; - case GFXSCT_Float3x3: return "float3x3"; break; - case GFXSCT_Float3x4: return "float3x4"; break; - case GFXSCT_Float4x3: return "float4x3"; break; - case GFXSCT_Float4x4: return "float4x4"; break; - case GFXSCT_Int: return "int"; break; - case GFXSCT_Int2: return "int2"; break; - case GFXSCT_Int3: return "int3"; break; - case GFXSCT_Int4: return "int4"; break; - case GFXSCT_UInt: return "uint"; break; - case GFXSCT_UInt2: return "uint2"; break; - case GFXSCT_UInt3: return "uint3"; break; - case GFXSCT_UInt4: return "uint4"; break; - case GFXSCT_Bool: return "bool"; break; - case GFXSCT_Bool2: return "bool2"; break; - case GFXSCT_Bool3: return "bool3"; break; - case GFXSCT_Bool4: return "bool4"; break; - default: return "unknown"; break; - } - } + // ---- FLOATS ---- + { GFXSCT_Float, "float", "float", STC_Scalar, 1, 1 }, + { GFXSCT_Float2, "vec2", "float2", STC_Vector, 1, 2 }, + { GFXSCT_Float3, "vec3", "float3", STC_Vector, 1, 3 }, + { GFXSCT_Float4, "vec4", "float4", STC_Vector, 1, 4 }, - return ""; + // ---- MATRICES ---- + { GFXSCT_Float2x2, "mat2", "float2x2", STC_Matrix, 2, 2 }, + { GFXSCT_Float3x3, "mat3", "float3x3", STC_Matrix, 3, 3 }, + { GFXSCT_Float3x4, "mat3x4", "float3x4", STC_Matrix, 3, 4 }, + { GFXSCT_Float4x3, "mat4x3", "float4x3", STC_Matrix, 4, 3 }, + { GFXSCT_Float4x4, "mat4", "float4x4", STC_Matrix, 4, 4 }, + + // ---- INT ---- + { GFXSCT_Int, "int", "int", STC_Scalar, 1, 1 }, + { GFXSCT_Int2, "ivec2", "int2", STC_Vector, 1, 2 }, + { GFXSCT_Int3, "ivec3", "int3", STC_Vector, 1, 3 }, + { GFXSCT_Int4, "ivec4", "int4", STC_Vector, 1, 4 }, + + // ---- UINT ---- + { GFXSCT_UInt, "uint", "uint", STC_Scalar, 1, 1 }, + { GFXSCT_UInt2, "uvec2", "uint2", STC_Vector, 1, 2 }, + { GFXSCT_UInt3, "uvec3", "uint3", STC_Vector, 1, 3 }, + { GFXSCT_UInt4, "uvec4", "uint4", STC_Vector, 1, 4 }, + + // ---- BOOL ---- + { GFXSCT_Bool, "bool", "bool", STC_Scalar, 1, 1 }, + { GFXSCT_Bool2, "bvec2", "bool2", STC_Vector, 1, 2 }, + { GFXSCT_Bool3, "bvec3", "bool3", STC_Vector, 1, 3 }, + { GFXSCT_Bool4, "bvec4", "bool4", STC_Vector, 1, 4 }, + + // ---- SAMPLERS ---- + { GFXSCT_Sampler, "sampler2D", "Texture2D", STC_Sampler, 0, 0 }, + { GFXSCT_SamplerCube, "samplerCube", "TextureCube", STC_Sampler, 0, 0 }, + { GFXSCT_SamplerTextureArray, "sampler2DArray", "Texture2DArray", STC_Sampler, 0, 0 }, + { GFXSCT_SamplerCubeArray, "samplerCubeArray", "TextureCubeArray", STC_Sampler, 0, 0 }, +}; + +static HashMap glslToType; +static HashMap hlslToType; + +void LangElement::buildTypeMaps() +{ + for (auto& info : ShaderTypes) + { + glslToType[info.glslName] = info.type; + hlslToType[info.hlslName] = info.type; + } } -const char* LangElement::samplerTypeToString(GFXShaderConstType constType) +const ShaderTypeInfo* LangElement::getTypeInfo(GFXShaderConstType type) { - if (constType < GFXSCT_Sampler) - return ""; - - // Determine shader language based on GFXAdapterAPI - if (GFX->getAdapterType() == OpenGL) - { - switch (constType) - { - case GFXSCT_Sampler: return "sampler2D"; break; - case GFXSCT_SamplerCube: return "samplerCube"; break; - case GFXSCT_SamplerTextureArray: return "sampler2DArray"; break; - case GFXSCT_SamplerCubeArray: return "samplerCubeArray"; break; - default: return "unknown"; break; - } - } - else // Assume DirectX/HLSL - { - switch (constType) - { - case GFXSCT_Sampler: return "Texture2D"; break; - case GFXSCT_SamplerCube: return "TextureCube"; break; - case GFXSCT_SamplerTextureArray: return "Texture2DArray"; break; - case GFXSCT_SamplerCubeArray: return "TextureCubeArray"; break; - default: return "unknown"; break; - } - } - - return ""; + for (auto& info : ShaderTypes) + if (info.type == type) + return &info; + return nullptr; } +const char* LangElement::constTypeToString(GFXShaderConstType constType, bool sampler, bool matrix) +{ + const ShaderTypeInfo* info = getTypeInfo(constType); + if (!info) + return "unknown"; + + if (sampler) + { + if (!info->isSampler()) + { + Con::warnf("LangElement::Requested sampler but input const type is not a sampler"); + return "unknown"; + } + } + + if (matrix) + { + if (!info->isMatrix()) + { + Con::warnf("LangElement::Requested matrix but input const type is not a matrix"); + return "unknown"; + } + } + + return (GFX->getAdapterType() == OpenGL) + ? info->glslName + : info->hlslName; +} + +GFXShaderConstType LangElement::stringToConstType(const char* name) +{ + bool glsl = (GFX->getAdapterType() == OpenGL); + + auto& map = glsl ? glslToType : hlslToType; + auto it = map.find(name); + + if (it != map.end()) + return it->value; + + return GFXSCT_Uknown; +} //-------------------------------------------------------------------------- // Constructor diff --git a/Engine/source/shaderGen/langElement.h b/Engine/source/shaderGen/langElement.h index 89897f5a8..48c6afcb2 100644 --- a/Engine/source/shaderGen/langElement.h +++ b/Engine/source/shaderGen/langElement.h @@ -36,6 +36,46 @@ #define WRITESTR( a ){ stream.write( dStrlen(a), a ); } +//************************************************************************** +/*! + These structs are helpers for unifying both sides of shadergen, the setup + allows us to create other shaderops such as constructors for vars, cast + operations and also checks to make sure mathops can be executed cleanly. +*/ +//************************************************************************** + +enum ShaderTypeCategory +{ + STC_Scalar, + STC_Vector, + STC_Matrix, + STC_Sampler +}; + +/// +/// ShaderTypeInfo type helper for casts and other ops +/// +/// GFXShaderConstType enum type +/// const char* type name for glsl +/// const char* type name for hlsl +/// ShaderTypeCategory enum for category eg STC_Scalar +struct ShaderTypeInfo +{ + GFXShaderConstType type; + + const char* glslName; + const char* hlslName; + + ShaderTypeCategory category; + + U32 rows; // for matrices (otherwise 1) + U32 cols; // vector size for scalars/vectors, column count for matrices + + bool isSampler() const { return category == STC_Sampler; } + bool isVector() const { return category == STC_Vector; } + bool isMatrix() const { return category == STC_Matrix; } + bool isScalar() const { return category == STC_Scalar; } +}; //************************************************************************** /*! @@ -54,13 +94,17 @@ //************************************************************************** struct LangElement { + static void buildTypeMaps(); static Vector elementList; static LangElement * find( const char *name ); static void deleteElements(); + static const ShaderTypeInfo* getTypeInfo(GFXShaderConstType type); U8 name[32]; - static const char* constTypeToString(GFXShaderConstType constType); - static const char* samplerTypeToString(GFXShaderConstType constType); + + static const char* constTypeToString(GFXShaderConstType constType, bool sampler = false, bool matrix = false); + static GFXShaderConstType stringToConstType(const char* name); + LangElement(); virtual ~LangElement() {}; virtual void print( Stream &stream ){}; @@ -186,6 +230,13 @@ public: void print( Stream &stream ) override; }; - +class LiteralStr : public LangElement { +public: + LiteralStr(const char* s) : mStr(s) {} + void print(Stream& stream) override { WRITESTR(mStr.c_str()); } + String mStr; +}; #endif // _LANG_ELEMENT_H_ + + diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index 1f30654c6..16db02946 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -133,6 +133,9 @@ void ShaderGen::initShaderGen() // Delete the auto-generated conditioner include file. Torque::FS::Remove( "shadergen:/" + ConditionerFeature::ConditionerIncludeFileName ); + + // build our type maps. + LangElement::buildTypeMaps(); } void ShaderGen::generateShader( const MaterialFeatureData &featureData, diff --git a/Engine/source/shaderGen/shaderOp.cpp b/Engine/source/shaderGen/shaderOp.cpp index b160615aa..80e467080 100644 --- a/Engine/source/shaderGen/shaderOp.cpp +++ b/Engine/source/shaderGen/shaderOp.cpp @@ -22,10 +22,44 @@ #include "core/strings/stringFunctions.h" #include - +#include "gfx/gfxDevice.h" #include "shaderOp.h" +bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo) +{ + outVar = nullptr; + outInfo = nullptr; + + // DIRECT VAR + if (Var* v = dynamic_cast(elem)) + { + outVar = v; + outInfo = LangElement::getTypeInfo(LangElement::stringToConstType((const char*)v->type)); + return outInfo != nullptr; + } + + // INDEX OP: arrVar[index] + if (IndexOp* idx = dynamic_cast(elem)) + { + Var* arr = idx->arrVar; + if (!arr) + return false; + + const ShaderTypeInfo* arrInfo = LangElement::getTypeInfo(LangElement::stringToConstType((const char*)arr->type)); + if (!arrInfo) + return false; + + // array element type = same as var type but no array dimension + outVar = arr; + outInfo = arrInfo; + return true; + } + + return false; +} + + //************************************************************************** // Shader Operations //************************************************************************** @@ -85,6 +119,7 @@ void EchoOp::print( Stream &stream ) //************************************************************************** IndexOp::IndexOp( Var* var, U32 index ) : Parent( NULL, NULL ) { + arrVar = var; // need to keep hold of it for casts. mInput[0] = var; mIndex = index; } @@ -180,18 +215,260 @@ void GenOp::print( Stream &stream ) } } -CastOp::CastOp(Var* in1, GFXShaderConstType type) : Parent(in1, NULL) +//---------------------------------------------------------------------------- +// TYPE OPERATION +//---------------------------------------------------------------------------- + +TypeOp::TypeOp(GFXShaderConstType type) : Parent(NULL, NULL) { - mInput[0] = in1; - mConstType = constTypeToString(type); + mType = type; +} + +TypeOp::~TypeOp() +{ +} + +//---------------------------------------------------------------------------- +// Print +//---------------------------------------------------------------------------- + +void TypeOp::print(Stream& stream) +{ + WRITESTR(LangElement::constTypeToString(mType)); +} + +//---------------------------------------------------------------------------- +// CAST OPERATION +//---------------------------------------------------------------------------- + +CastOp::CastOp(LangElement* srcVar, GFXShaderConstType type, const char* swizzleStr, const char* fillStr) : Parent(srcVar, NULL) +{ + mInput[0] = srcVar; + mTargetType = type; + parseStringList(swizzleStr, mSwizzle); + parseStringList(fillStr, mFillValues); } void CastOp::print(Stream& stream) { - Var* var = dynamic_cast(mInput[0]); + LangElement* srcElem = mInput[0]; - WRITESTR(mConstType); - WRITESTR("( "); - mInput[0]->print(stream); - WRITESTR(" )"); + Var* srcVar = nullptr; + const ShaderTypeInfo* srcInfo = nullptr; + + if (!resolveSourceType(srcElem, srcVar, srcInfo)) + { + // fallback: at least print something + srcElem->print(stream); + return; + } + + const ShaderTypeInfo* dstInfo = getTypeInfo(mTargetType); + + // no info? types match? nothing to do. + if (!srcInfo || !dstInfo || (srcInfo->type == dstInfo->type)) + { + srcElem->print(stream); // print something.... + return; + } + + bool glsl = (GFX->getAdapterType() == OpenGL); + const char* dstName = glsl ? dstInfo->glslName : dstInfo->hlslName; + + U32 srcSize = srcInfo->cols; + U32 dstSize = dstInfo->cols; + + // scalar -> vector + if (srcSize == 1 && dstSize > 1) + { + WRITESTR(dstName); + WRITESTR("("); + srcElem->print(stream); + + for (U32 i = 1; i < dstSize; i++) + { + WRITESTR(", "); + WRITESTR(mFillValues[i].c_str()); + } + + WRITESTR(")"); + return; + } + + // vector -> scalar + if (srcSize > 1 && dstSize == 1) + { + srcElem->print(stream); + WRITESTR("."); + WRITESTR(mSwizzle[0].c_str()); + return; + } + + // vector -> vector narrowing + if (srcSize > dstSize) + { + WRITESTR(dstName); + WRITESTR("("); + srcElem->print(stream); + WRITESTR("."); + + for (U32 i = 0; i < dstSize; i++) + { + WRITESTR(mSwizzle[i].c_str()); + } + + WRITESTR(")"); + return; + } + + // vector -> vector widening + if (srcSize < dstSize) + { + WRITESTR(dstName); + WRITESTR("("); + srcElem->print(stream); + if (mSwizzle.size() < srcSize) + { + WRITESTR("."); + + for (U32 i = 0; i < mSwizzle.size(); i++) + { + WRITESTR(mSwizzle[i].c_str()); + } + } + + for (U32 i = getMin((U32)mSwizzle.size(), srcSize); i < dstSize; i++) + { + WRITESTR(", "); + WRITESTR(mFillValues[i].c_str()); + } + + WRITESTR(")"); + return; + } + + // fallback + srcElem->print(stream); } + +//---------------------------------------------------------------------------- +// MATRIX INITIALIZE OPERATION +//---------------------------------------------------------------------------- + +void MatrixInitializeOp::print(Stream& stream) +{ + Var* matVar = dynamic_cast(mInput[0]); + if (!matVar) + return; + + const ShaderTypeInfo* matInfo = getTypeInfo(stringToConstType((const char*)matVar->type)); + if (!matInfo || !matInfo->isMatrix()) + return; + + // full size of the mat. + const bool glsl = (GFX->getAdapterType() == OpenGL); + const U32 rows = matInfo->rows; + const U32 cols = matInfo->cols; + const U32 matSize = rows * cols; + + if (glsl) + { + WRITESTR(matInfo->glslName); + WRITESTR("(\r\n"); + } + else + { + WRITESTR("{\r\n"); + } + + U32 count = 0; + for (U32 elem = 0; elem < mInitialVals.size(); elem++) + { + LangElement* writeOut = NULL; + Var* curVar = dynamic_cast(mInitialVals[elem]); + if (curVar) // is a var + { + const ShaderTypeInfo* curInfo = getTypeInfo(stringToConstType((const char*)curVar->type)); + if (!curInfo) // no info, cant do it cleanly. + return; + + const U32 curSize = curInfo->cols; + + // if we are an array + if (curVar->arraySize > 1) + { + for (U32 arr = 0; arr < curVar->arraySize; arr++) + { + writeOut = new IndexOp(curVar, arr); + if (curSize != cols) + { + CastOp* cast = new CastOp(writeOut, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); + cast->print(stream); + count += cols; + } + else + { + writeOut->print(stream); + count += curSize; + } + + if (count < matSize) + { + WRITESTR(",\r\n"); + } + } + } + else + { + if (curSize != cols) + { + CastOp* cast = new CastOp(curVar, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); + cast->print(stream); + count += cols; + } + else + { + curVar->print(stream); + count += curSize; + } + + if (count < matSize) + { + WRITESTR(",\r\n"); + } + } + } + else + { + // Non-var LangElement, assume it produces correct vector + mInitialVals[elem]->print(stream); + count += cols; + + if (count < matSize) + WRITESTR(",\r\n"); + } + } + + // If not enough elements → pad with zeros + while (count < matSize) + { + WRITESTR("0"); + count++; + + if (count < matSize) + { + WRITESTR(",\r\n"); + } + } + + + if (glsl) + { + WRITESTR(")\r\n"); + } + else + { + WRITESTR("}\r\n"); + } +} + diff --git a/Engine/source/shaderGen/shaderOp.h b/Engine/source/shaderGen/shaderOp.h index 6b8966250..8a42168dd 100644 --- a/Engine/source/shaderGen/shaderOp.h +++ b/Engine/source/shaderGen/shaderOp.h @@ -45,7 +45,7 @@ */ //************************************************************************** - +bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo); ///************************************************************************** /// Shader operation base class @@ -117,8 +117,8 @@ class IndexOp : public ShaderOp { typedef ShaderOp Parent; U32 mIndex; - public: + Var* arrVar; IndexOp( Var* var, U32 index ); void print( Stream &stream ) override; }; @@ -161,14 +161,86 @@ public: }; +//---------------------------------------------------------------------------- +/*! + Like the name suggests, prints out the type as a string, for working between + glsl and hlsl. +*/ +//---------------------------------------------------------------------------- +class TypeOp : public ShaderOp +{ + typedef ShaderOp Parent; + GFXShaderConstType mType; +public: + TypeOp(GFXShaderConstType type); + ~TypeOp(); + void print(Stream& stream) override; +}; + +//---------------------------------------------------------------------------- +/*! + Casting operation to cast a var from one type to another. +*/ +//---------------------------------------------------------------------------- class CastOp : public ShaderOp { typedef ShaderOp Parent; - const char* mConstType; + GFXShaderConstType mTargetType; + Vector mSwizzle; // "x", "y", "z", "w" + Vector mFillValues; // "0", "0", "0", "1" public: - CastOp(Var* in1, GFXShaderConstType type); + CastOp( LangElement* srcVar, + GFXShaderConstType type, + const char* swizzleStr = "x;y;z;w", + const char* fillStr = "0;0;0;1"); + + void print(Stream& stream) override; + + void parseStringList(const char* src, Vector& out) + { + out.clear(); + const char* p = src; + + while (*p) + { + const char* start = p; + while (*p && *p != ';') + p++; + + out.push_back(String(start, p - start)); + + if (*p == ';') + p++; + } + } +}; + +//---------------------------------------------------------------------------- +/*! + Matrix initialize operation, initializes a matrix with the input + vars as a vector. +*/ +//---------------------------------------------------------------------------- + +class MatrixInitializeOp : public ShaderOp +{ + typedef ShaderOp Parent; + Vector mInitialVals; +public: + MatrixInitializeOp(Var* matrixVar, const Vector& inputs) + : Parent(matrixVar, nullptr) + { + mInitialVals = inputs; + mInput[0] = matrixVar; + } + void print(Stream& stream) override; }; +//---------------------------------------------------------------------------- +/*! + Matrix multiplication operation. +*/ +//---------------------------------------------------------------------------- #endif // _SHADEROP_H_ From a92ea1ffab177f1f21214fa37da61e28e512a4ba Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Sat, 29 Nov 2025 08:15:55 +0000 Subject: [PATCH 04/17] vertex position node Cross api support for vertex positions matrixmultiplyop added for multiplying matrix with other types automatically expands or shortens vec types to suit the matrix Added identity to the matrix initialization op. --- .../source/shaderGen/NODE/shaderGenNodes.cpp | 252 +++++++++++++++--- Engine/source/shaderGen/NODE/shaderGenNodes.h | 27 ++ Engine/source/shaderGen/shaderFeature.cpp | 67 +++++ Engine/source/shaderGen/shaderOp.cpp | 113 +++++++- Engine/source/shaderGen/shaderOp.h | 15 +- 5 files changed, 418 insertions(+), 56 deletions(-) diff --git a/Engine/source/shaderGen/NODE/shaderGenNodes.cpp b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp index 8634bc32d..2ef78ab47 100644 --- a/Engine/source/shaderGen/NODE/shaderGenNodes.cpp +++ b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp @@ -15,11 +15,13 @@ #include "materials/materialFeatureTypes.h" +ImplementFeatureType(SNF_VertexPosition, U32(-1), -1, false); ImplementFeatureType(SNF_DefaultTexCoord, U32(-1), -1, false); ImplementFeatureType(SNF_TextureFeature, U32(-1), -1, false); ImplementFeatureType(SNF_NormalMapFeature, U32(-1), -1, false); ImplementEnumType(ShaderNodeFeature_enum, "Shader node features. Each of thes relates to a specific node for generating a shader.\n\n") + { ShaderNodeFeature_enum::eSNF_VertexPosition, "SNF_VertexPosition", "Setup vertex position." }, { ShaderNodeFeature_enum::eSNF_DefaultTexCoord, "SNF_DefaultTexCoord", "Setup the default texcoord." }, { ShaderNodeFeature_enum::eSNF_TextureFeature, "SNF_TextureFeature", "Sample a Texture - Params: (string,string,GFXSamplerStateData,bool)." }, { ShaderNodeFeature_enum::eSNF_NormalMapFeature,"SNF_NormalMapFeature", "Convert a texture to a normalmap - Params: (string,float,bool,bool)." }, @@ -29,6 +31,7 @@ namespace { void register_node_features(GFXAdapterType type) { + FEATUREMGR->registerFeature(SNF_VertexPosition, new NodeVertexPositionFeature); FEATUREMGR->registerFeature(SNF_DefaultTexCoord, new DefaultTexcoordFeature); FEATUREMGR->registerFeature(SNF_TextureFeature, new TextureFeature, TextureFeature::createFunction); FEATUREMGR->registerFeature(SNF_NormalMapFeature, new NormalMapFeature, NormalMapFeature::createFunction); @@ -51,6 +54,125 @@ MODULE_INIT MODULE_END; +Var* ShaderFeatureNode::getObjTrans(Vector& componentList, + bool useInstancing, + MultiLine* meta) +{ + Var* objTrans = (Var*)LangElement::find("objTrans"); + if (objTrans) + return objTrans; + + if (useInstancing) + { + ShaderConnector* vertStruct = dynamic_cast(componentList[C_VERT_STRUCT]); + Var* instObjTrans = vertStruct->getElement(RT_TEXCOORD, 4, 4); + instObjTrans->setStructName("IN"); + instObjTrans->setName("inst_objectTrans"); + + mInstancingFormat->addElement("objTrans", GFXDeclType_Float4, instObjTrans->constNum + 0); + mInstancingFormat->addElement("objTrans", GFXDeclType_Float4, instObjTrans->constNum + 1); + mInstancingFormat->addElement("objTrans", GFXDeclType_Float4, instObjTrans->constNum + 2); + mInstancingFormat->addElement("objTrans", GFXDeclType_Float4, instObjTrans->constNum + 3); + + objTrans = new Var; + objTrans->setType(GFXSCT_Float4x4); + objTrans->setName("objTrans"); + + Vector matrixVars; + matrixVars.push_back(instObjTrans); + meta->addStatement(new GenOp(" @ = @;", new DecOp(objTrans), new MatrixInitializeOp(objTrans, matrixVars))); + } + else + { + objTrans = new Var; + objTrans->setType(GFXSCT_Float4x4); + objTrans->setName("objTrans"); + objTrans->uniform = true; + objTrans->constSortPos = cspPrimitive; + } + + return objTrans; +} + +Var* ShaderFeatureNode::getModelView( Vector& componentList, + bool useInstancing, + MultiLine* meta) +{ + Var* modelview = (Var*)LangElement::find("modelview"); + if (modelview) + return modelview; + + if (useInstancing) + { + Var* objTrans = getObjTrans(componentList, useInstancing, meta); + + Var* viewProj = (Var*)LangElement::find("viewProj"); + if (!viewProj) + { + viewProj = new Var; + viewProj->setType(GFXSCT_Float4x4); + viewProj->setName("viewProj"); + viewProj->uniform = true; + viewProj->constSortPos = cspPass; + } + + modelview = new Var; + modelview->setType(GFXSCT_Float4x4); + modelview->setName("modelview"); + meta->addStatement(new GenOp(" @ = @; // Instancing!\r\n", new DecOp(modelview), new MatrixMultiplyOp(viewProj, objTrans))); + } + else + { + modelview = new Var; + modelview->setType(GFXSCT_Float4x4); + modelview->setName("modelview"); + modelview->uniform = true; + modelview->constSortPos = cspPrimitive; + } + + return modelview; +} + +Var* ShaderFeatureNode::getWorldView( Vector& componentList, + bool useInstancing, + MultiLine* meta) +{ + Var* worldView = (Var*)LangElement::find("worldViewOnly"); + if (worldView) + return worldView; + + if (useInstancing) + { + Var* objTrans = getObjTrans(componentList, useInstancing, meta); + + Var* worldToCamera = (Var*)LangElement::find("worldToCamera"); + if (!worldToCamera) + { + worldToCamera = new Var; + worldToCamera->setType(GFXSCT_Float4x4); + worldToCamera->setName("worldToCamera"); + worldToCamera->uniform = true; + worldToCamera->constSortPos = cspPass; + } + + worldView = new Var; + worldView->setType(GFXSCT_Float4x4); + worldView->setName("worldViewOnly"); + + meta->addStatement(new GenOp(" @ = @; // Instancing!\r\n", new DecOp(worldView), new MatrixMultiplyOp(worldToCamera, objTrans) )); + } + else + { + worldView = new Var; + worldView->setType(GFXSCT_Float4x4); + worldView->setName("worldViewOnly"); + worldView->uniform = true; + worldView->constSortPos = cspPrimitive; + } + + return worldView; +} + void ShaderFeatureNode::setupTextureSample( const String& samplerName, GFXShaderConstType samplerType, Vector& componentList, @@ -69,43 +191,7 @@ void ShaderFeatureNode::setupTextureSample( const String& samplerName, Var* samplerVar = dynamic_cast(LangElement::find(sampVarName)); - Var* textureVar; - - if (!isGL) // HLSL requires both Texture + SamplerState - { - if (!samplerVar) - { - samplerVar = new Var; - samplerVar->setType(isComparison ? "SamplerComparisonState" : "SamplerState"); - samplerVar->setName(sampVarName); - samplerVar->uniform = true; - samplerVar->sampler = true; - samplerVar->constNum = Var::getTexUnitNum(); - } - - textureVar = dynamic_cast(LangElement::find(texVarName)); - if (!textureVar) - { - textureVar = new Var; - textureVar->setType(LangElement::samplerTypeToString(samplerType)); // Texture2D, TextureCube, etc. - textureVar->setName(texVarName); - textureVar->uniform = true; - textureVar->texture = true; - textureVar->constNum = samplerVar->constNum; - } - } - else - { - if (!samplerVar) - { - samplerVar = new Var; - samplerVar->setType(LangElement::samplerTypeToString(samplerType)); - samplerVar->setName(sampVarName); - samplerVar->uniform = true; - samplerVar->sampler = true; - samplerVar->constNum = Var::getTexUnitNum(); - } - } + Var* textureVar = NULL; // The sampled color variable (e.g. "samplerName_col") should always be new but just in case Var* sampledColor = (Var*)LangElement::find(resultVarName); @@ -123,6 +209,16 @@ void ShaderFeatureNode::setupTextureSample( const String& samplerName, if (isGL) { // ---------------- GLSL Sampling ---------------- + if (!samplerVar) + { + samplerVar = new Var; + samplerVar->setType(LangElement::constTypeToString(samplerType, true)); + samplerVar->setName(sampVarName); + samplerVar->uniform = true; + samplerVar->sampler = true; + samplerVar->constNum = Var::getTexUnitNum(); + } + if (isComparison) { if (useGather) @@ -149,6 +245,27 @@ void ShaderFeatureNode::setupTextureSample( const String& samplerName, else { // ---------------- HLSL Sampling ---------------- + if (!samplerVar) + { + samplerVar = new Var; + samplerVar->setType(isComparison ? "SamplerComparisonState" : "SamplerState"); + samplerVar->setName(sampVarName); + samplerVar->uniform = true; + samplerVar->sampler = true; + samplerVar->constNum = Var::getTexUnitNum(); + } + + textureVar = dynamic_cast(LangElement::find(texVarName)); + if (!textureVar) + { + textureVar = new Var; + textureVar->setType(LangElement::constTypeToString(samplerType, true)); // Texture2D, TextureCube, etc. + textureVar->setName(texVarName); + textureVar->uniform = true; + textureVar->texture = true; + textureVar->constNum = samplerVar->constNum; + } + if (isComparison) { if (useGather) @@ -192,8 +309,7 @@ Var* ShaderFeatureNode::getOutTexCoord(const char* name, GFXShaderConstType type // Statement allows for casting of different types which // eliminates vector truncation problems. - String statement = String::ToString(" @ = (%s)@;\r\n", type); - meta->addStatement(new GenOp(statement, texCoord, inTex)); + meta->addStatement(new GenOp(" @ = (@)@;\r\n", texCoord, new TypeOp(type), inTex)); } @@ -265,6 +381,57 @@ LangElement* ShaderFeatureNode::assignColor(LangElement* elem, Material::BlendOp return NULL; } +//-------------------------------------------------------- +// Vertex position. +//-------------------------------------------------------- + +void NodeVertexPositionFeature::processVert(Vector& componentList, const MaterialFeatureData& fd) +{ + // First check for an input position from a previous feature + // then look for the default vertex position. + Var* inPosition = (Var*)LangElement::find("inPosition"); + if (!inPosition) + inPosition = (Var*)LangElement::find("position"); + + const bool glsl = (GFX->getAdapterType() == OpenGL); + + // grab connector position + ShaderConnector* connectComp = dynamic_cast(componentList[C_CONNECTOR]); + Var* outPosition = connectComp->getElement( glsl ? RT_POSITION : RT_SVPOSITION); + + if (glsl) + { + outPosition->setName("gl_Position"); + } + else + { + outPosition->setName("hpos"); + outPosition->setStructName("OUT"); + } + + MultiLine* meta = new MultiLine; + + Var* modelview = getModelView(componentList, fd.features[MFT_UseInstancing], meta); + + meta->addStatement(new GenOp(" @ = @;\r\n", outPosition, new MatrixMultiplyOp( modelview, new CastOp(inPosition, GFXSCT_Float4, "x;y;z")))); + + output = meta; +} + +void NodeVertexPositionFeature::processPix(Vector& componentList, const MaterialFeatureData& fd) +{ + const bool glsl = (GFX->getAdapterType() == OpenGL); + + if (!glsl) + { + // grab connector position + ShaderConnector* connectComp = dynamic_cast(componentList[C_CONNECTOR]); + Var* outPosition = connectComp->getElement(RT_SVPOSITION); + outPosition->setName("vpos"); + outPosition->setStructName("IN"); + } +} + //-------------------------------------------------------- // Setup the default texcoord //-------------------------------------------------------- @@ -352,10 +519,10 @@ void NormalMapFeature::processPix(Vector& componentList, const meta->addStatement(expandNormalMap(sampledColor, tempNormDecl, tempNorm, fd)); meta->addStatement( - new GenOp(" @.xy *= float2(@, @);\r\n", tempNorm, params->flipX ? -1.0f : 1.0f, params->flipY ? -1.0f : 1.0f)); + new GenOp(" @.xy *= @(@, @);\r\n", tempNorm, new TypeOp(GFXSCT_Float2), params->flipX ? -1.0f : 1.0f, params->flipY ? -1.0f : 1.0f)); - meta->addStatement(new GenOp(" @.xyz = normalize( float3( @.xy * @, @.z ) );\r\n", - tempNorm, tempNorm, params->strength, tempNorm)); + meta->addStatement(new GenOp(" @.xyz = normalize( @( @.xy * @, @.z ) );\r\n", + tempNorm, new TypeOp(GFXSCT_Float3), tempNorm, params->strength, tempNorm)); // write back into our known variable. meta->addStatement(new GenOp(" @ = @;\r\n", sampledColor, tempNorm)); @@ -363,3 +530,4 @@ void NormalMapFeature::processPix(Vector& componentList, const output = meta; } + diff --git a/Engine/source/shaderGen/NODE/shaderGenNodes.h b/Engine/source/shaderGen/NODE/shaderGenNodes.h index b1082f3d8..d93f3ca80 100644 --- a/Engine/source/shaderGen/NODE/shaderGenNodes.h +++ b/Engine/source/shaderGen/NODE/shaderGenNodes.h @@ -20,6 +20,7 @@ #include "gfx/gfxAPI.h" #endif +DeclareFeatureType(SNF_VertexPosition); DeclareFeatureType(SNF_DefaultTexCoord); DeclareFeatureType(SNF_TextureFeature); DeclareFeatureType(SNF_NormalMapFeature); @@ -29,6 +30,7 @@ DeclareFeatureType(SNF_NormalMapFeature); /// enum ShaderNodeFeature_enum { + eSNF_VertexPosition, eSNF_DefaultTexCoord, eSNF_TextureFeature, eSNF_NormalMapFeature, @@ -39,6 +41,18 @@ DefineEnumType(ShaderNodeFeature_enum); class ShaderFeatureNode : public ShaderFeature { public: + + /// + Var* getObjTrans( Vector& componentList, + bool useInstancing, + MultiLine* meta); + + Var* getModelView(Vector& componentList, + bool useInstancing, + MultiLine* meta); + + Var* getWorldView(Vector& componentList, bool useInstancing, MultiLine* meta); + void setupTextureSample(const String& samplerName, GFXShaderConstType samplerType, Vector& componentList, @@ -58,6 +72,19 @@ public: LangElement* assignColor(LangElement* elem, Material::BlendOp blend, LangElement* lerpElem = NULL, ShaderFeature::OutputTarget outputTarget = ShaderFeature::DefaultTarget) override; }; +class NodeVertexPositionFeature : public ShaderFeatureNode +{ + void processVert(Vector& componentList, + const MaterialFeatureData& fd) override; + + void processPix(Vector& componentList, + const MaterialFeatureData& fd) override; + + String getName() override + { + return "NodeVertexPositionFeature"; + } +}; class DefaultTexcoordFeature : public ShaderFeatureNode { diff --git a/Engine/source/shaderGen/shaderFeature.cpp b/Engine/source/shaderGen/shaderFeature.cpp index 00bbd7c9d..6dfd7b553 100644 --- a/Engine/source/shaderGen/shaderFeature.cpp +++ b/Engine/source/shaderGen/shaderFeature.cpp @@ -38,6 +38,73 @@ void ShaderFeature::addDependency( const ShaderDependency *dependsOn ) mDependencies.push_back( dependsOn ); } +Var* ShaderFeature::getVertTexCoord(const String& name) +{ + Var* inTex = NULL; + + for (U32 i = 0; i < LangElement::elementList.size(); i++) + { + if (!String::compare((char*)LangElement::elementList[i]->name, name.c_str())) + { + inTex = dynamic_cast(LangElement::elementList[i]); + if (inTex) + { + // NOTE: This used to do this check... + // + // String::compare( (char*)inTex->structName, "IN" ) + // + // ... to ensure that the var was from the input + // vertex structure, but this kept some features + // ( ie. imposter vert ) from decoding their own + // coords for other features to use. + // + // If we run into issues with collisions between + // IN vars and local vars we may need to revise. + + break; + } + } + } + + return inTex; +} + +LangElement* ShaderFeature::expandNormalMap(LangElement* sampleNormalOp, LangElement* normalDecl, LangElement* normalVar, const MaterialFeatureData& fd) +{ + MultiLine* meta = new MultiLine; + const bool hasBc3 = fd.features.hasFeature(MFT_IsBC3nm, getProcessIndex()); + const bool hasBc5 = fd.features.hasFeature(MFT_IsBC5nm, getProcessIndex()); + + if (hasBc3 || hasBc5) + { + if (fd.features[MFT_ImposterVert]) + { + // The imposter system uses object space normals and + // encodes them with the z axis in the alpha component. + meta->addStatement(new GenOp(" @ = @( normalize( @.xyw * 2.0 - 1.0 ), 0.0 ); // Obj DXTnm\r\n", normalDecl, new TypeOp(GFXSCT_Float4), sampleNormalOp)); + } + else if (hasBc3) + { + // BC3 Swizzle trick + meta->addStatement(new GenOp(" @ = @( @.ag * 2.0 - 1.0, 0.0, 0.0 ); // DXTnm\r\n", normalDecl, new TypeOp(GFXSCT_Float4), sampleNormalOp)); + meta->addStatement(new GenOp(" @.z = sqrt( 1.0 - dot( @.xy, @.xy ) ); // DXTnm\r\n", normalVar, normalVar, normalVar)); + } + else if (hasBc5) + { + // BC5 + meta->addStatement(new GenOp(" @ = @( @.gr * 2.0 - 1.0, 0.0, 0.0 ); // bc5nm\r\n", normalDecl, new TypeOp(GFXSCT_Float4), sampleNormalOp)); + meta->addStatement(new GenOp(" @.z = sqrt( 1.0 - dot( @.xy, @.xy ) ); // bc5nm\r\n", normalVar, normalVar, normalVar)); + } + } + else + { + meta->addStatement(new GenOp(" @ = @;\r\n", normalDecl, sampleNormalOp)); + meta->addStatement(new GenOp(" @.xyz = @.xyz * 2.0 - 1.0;\r\n", normalVar, normalVar)); + } + + return meta; +} + ShaderFeature::Resources ShaderFeature::getResources( const MaterialFeatureData &fd ) { Resources temp; diff --git a/Engine/source/shaderGen/shaderOp.cpp b/Engine/source/shaderGen/shaderOp.cpp index 80e467080..8a514f21c 100644 --- a/Engine/source/shaderGen/shaderOp.cpp +++ b/Engine/source/shaderGen/shaderOp.cpp @@ -26,7 +26,7 @@ #include "shaderOp.h" -bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo) +bool ShaderOp::resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo) { outVar = nullptr; outInfo = nullptr; @@ -35,18 +35,18 @@ bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& o if (Var* v = dynamic_cast(elem)) { outVar = v; - outInfo = LangElement::getTypeInfo(LangElement::stringToConstType((const char*)v->type)); + outInfo = getTypeInfo(stringToConstType((const char*)v->type)); return outInfo != nullptr; } // INDEX OP: arrVar[index] if (IndexOp* idx = dynamic_cast(elem)) { - Var* arr = idx->arrVar; + Var* arr = dynamic_cast(idx->mInput[0]); if (!arr) return false; - const ShaderTypeInfo* arrInfo = LangElement::getTypeInfo(LangElement::stringToConstType((const char*)arr->type)); + const ShaderTypeInfo* arrInfo = getTypeInfo(stringToConstType((const char*)arr->type)); if (!arrInfo) return false; @@ -56,10 +56,23 @@ bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& o return true; } + if (CastOp* cast = dynamic_cast(elem)) + { + Var* castVar = dynamic_cast(cast->mInput[0]); + if (!castVar) + return false; + + const ShaderTypeInfo* castInfo = getTypeInfo(cast->mTargetType);// get the casts target type. + if (!castInfo) + return false; + + outVar = castVar; + outInfo = castInfo; + } + return false; } - //************************************************************************** // Shader Operations //************************************************************************** @@ -117,9 +130,8 @@ void EchoOp::print( Stream &stream ) //************************************************************************** // Index operation //************************************************************************** -IndexOp::IndexOp( Var* var, U32 index ) : Parent( NULL, NULL ) +IndexOp::IndexOp( Var* var, U32 index ) : Parent(var, NULL ) { - arrVar = var; // need to keep hold of it for casts. mInput[0] = var; mIndex = index; } @@ -266,13 +278,13 @@ void CastOp::print(Stream& stream) const ShaderTypeInfo* dstInfo = getTypeInfo(mTargetType); // no info? types match? nothing to do. - if (!srcInfo || !dstInfo || (srcInfo->type == dstInfo->type)) + if (!srcInfo || !dstInfo) { srcElem->print(stream); // print something.... return; } - bool glsl = (GFX->getAdapterType() == OpenGL); + const bool glsl = (GFX->getAdapterType() == OpenGL); const char* dstName = glsl ? dstInfo->glslName : dstInfo->hlslName; U32 srcSize = srcInfo->cols; @@ -322,7 +334,7 @@ void CastOp::print(Stream& stream) } // vector -> vector widening - if (srcSize < dstSize) + if (srcSize <= dstSize) { WRITESTR(dstName); WRITESTR("("); @@ -452,7 +464,18 @@ void MatrixInitializeOp::print(Stream& stream) // If not enough elements → pad with zeros while (count < matSize) { - WRITESTR("0"); + U32 row = count / cols; + U32 col = count % cols; + + if (row == col) + { + WRITESTR("1"); + } + else + { + WRITESTR("0"); + } + count++; if (count < matSize) @@ -472,3 +495,71 @@ void MatrixInitializeOp::print(Stream& stream) } } +MatrixMultiplyOp::MatrixMultiplyOp(LangElement* left, LangElement* right) : Parent(left, right) +{ + mInput[0] = left; + mInput[1] = right; +} + +void MatrixMultiplyOp::print(Stream& stream) +{ + LangElement* leftElem = mInput[0]; + Var* leftVar = nullptr; + const ShaderTypeInfo* leftInfo = nullptr; + + if (!resolveSourceType(leftElem, leftVar, leftInfo)) + { + return; + } + + LangElement* rightElem = mInput[0]; + Var* rightVar = nullptr; + const ShaderTypeInfo* rightInfo = nullptr; + + if (!resolveSourceType(rightElem, rightVar, rightInfo)) + { + return; + } + + if (leftInfo->isMatrix() && rightInfo->isVector()) + { + if (rightInfo->cols != leftInfo->cols) + { + rightElem = new CastOp(rightVar, (GFXShaderConstType)(GFXSCT_Float + (leftInfo->cols - 1))); + } + } + + if (leftInfo->isVector() && rightInfo->isMatrix()) + { + if (rightInfo->cols != leftInfo->cols) + { + leftElem = new CastOp(leftVar, (GFXShaderConstType)(GFXSCT_Float + (rightInfo->cols - 1))); + } + } + + else if (leftInfo->isMatrix() && rightInfo->isMatrix()) + { + if (leftInfo->cols != rightInfo->rows) + Con::warnf("MatrixMultiplyOp: incompatible matrices: (%dx%d) × (%dx%d)", + leftInfo->rows, leftInfo->cols, + rightInfo->rows, rightInfo->cols); + } + + const bool glsl = (GFX->getAdapterType() == OpenGL); + + if (!glsl) + { + WRITESTR("mul("); + leftElem->print(stream); + WRITESTR(", "); + rightElem->print(stream); + WRITESTR(")"); + } + else + { + leftElem->print(stream); + WRITESTR(" * "); + rightElem->print(stream); + } + +} diff --git a/Engine/source/shaderGen/shaderOp.h b/Engine/source/shaderGen/shaderOp.h index 8a42168dd..7fb3da152 100644 --- a/Engine/source/shaderGen/shaderOp.h +++ b/Engine/source/shaderGen/shaderOp.h @@ -45,7 +45,7 @@ */ //************************************************************************** -bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo); + ///************************************************************************** /// Shader operation base class @@ -56,6 +56,7 @@ protected: LangElement * mInput[2]; public: + bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo); ShaderOp( LangElement *in1, LangElement *in2 ); }; @@ -118,7 +119,6 @@ class IndexOp : public ShaderOp typedef ShaderOp Parent; U32 mIndex; public: - Var* arrVar; IndexOp( Var* var, U32 index ); void print( Stream &stream ) override; }; @@ -185,10 +185,10 @@ public: class CastOp : public ShaderOp { typedef ShaderOp Parent; - GFXShaderConstType mTargetType; Vector mSwizzle; // "x", "y", "z", "w" Vector mFillValues; // "0", "0", "0", "1" public: + GFXShaderConstType mTargetType; CastOp( LangElement* srcVar, GFXShaderConstType type, const char* swizzleStr = "x;y;z;w", @@ -243,4 +243,13 @@ public: */ //---------------------------------------------------------------------------- +class MatrixMultiplyOp : public ShaderOp +{ + typedef ShaderOp Parent; +public: + MatrixMultiplyOp(LangElement* left, LangElement* right); + + void print(Stream& stream) override; +}; + #endif // _SHADEROP_H_ From 966b109bd683f1eae7c586c568f8ba32a3ee070c Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Sun, 30 Nov 2025 14:46:42 +0000 Subject: [PATCH 05/17] use resolveSourceType Use resolve source type for other shaderops This allows us to be more dynamic in the shaderop logic moved function out to langelement, this allows it to be used inside shaderFeatures --- .../source/shaderGen/NODE/shaderGenNodes.cpp | 2 +- Engine/source/shaderGen/langElement.cpp | 51 +++++++++ Engine/source/shaderGen/langElement.h | 4 +- Engine/source/shaderGen/shaderOp.cpp | 106 +++++------------- Engine/source/shaderGen/shaderOp.h | 5 +- 5 files changed, 85 insertions(+), 83 deletions(-) diff --git a/Engine/source/shaderGen/NODE/shaderGenNodes.cpp b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp index 2ef78ab47..5ae6de566 100644 --- a/Engine/source/shaderGen/NODE/shaderGenNodes.cpp +++ b/Engine/source/shaderGen/NODE/shaderGenNodes.cpp @@ -509,7 +509,7 @@ void NormalMapFeature::processPix(Vector& componentList, const MultiLine* meta = new MultiLine; - // TEMP float3 for base decoded normal + // TEMP float4 for base decoded normal Var* tempNorm = new Var; tempNorm->setName(params->inputName + "_normTemp"); tempNorm->setType(GFXSCT_Float4); diff --git a/Engine/source/shaderGen/langElement.cpp b/Engine/source/shaderGen/langElement.cpp index e0e197c9b..7148e85f3 100644 --- a/Engine/source/shaderGen/langElement.cpp +++ b/Engine/source/shaderGen/langElement.cpp @@ -23,6 +23,7 @@ #include "core/strings/stringFunctions.h" #include "core/util/str.h" #include "gfx/gfxDevice.h" +#include "shaderGen/shaderOp.h" #include "langElement.h" //************************************************************************** @@ -132,6 +133,56 @@ GFXShaderConstType LangElement::stringToConstType(const char* name) return GFXSCT_Uknown; } +bool LangElement::resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo) +{ + outVar = nullptr; + outInfo = nullptr; + + // DIRECT VAR + if (Var* v = dynamic_cast(elem)) + { + outVar = v; + outInfo = getTypeInfo(stringToConstType((const char*)v->type)); + return outInfo != nullptr; + } + + // INDEX OP: arrVar[index] + if (IndexOp* idx = dynamic_cast(elem)) + { + Var* arr = dynamic_cast(idx->mInput[0]); + if (!arr) + return false; + + const ShaderTypeInfo* arrInfo = getTypeInfo(stringToConstType((const char*)arr->type)); + if (!arrInfo) + return false; + + // array element type = same as var type but no array dimension + outVar = arr; + outInfo = arrInfo; + return true; + } + + // CAST OP: cast var + if (CastOp* cast = dynamic_cast(elem)) + { + Var* castVar = dynamic_cast(cast->mInput[0]); + if (!castVar) + return false; + + const ShaderTypeInfo* castInfo = getTypeInfo(cast->mTargetType);// get the casts target type. + if (!castInfo) + return false; + + outVar = castVar; // we should probably return null as we should just write the castop langelement, not a var. + outInfo = castInfo; + return true; + } + + return false; +} + + //-------------------------------------------------------------------------- // Constructor //-------------------------------------------------------------------------- diff --git a/Engine/source/shaderGen/langElement.h b/Engine/source/shaderGen/langElement.h index 48c6afcb2..3105e8ce5 100644 --- a/Engine/source/shaderGen/langElement.h +++ b/Engine/source/shaderGen/langElement.h @@ -92,6 +92,7 @@ struct ShaderTypeInfo //************************************************************************** // Language element //************************************************************************** +struct Var; // forward declaration struct LangElement { static void buildTypeMaps(); @@ -99,7 +100,8 @@ struct LangElement static LangElement * find( const char *name ); static void deleteElements(); static const ShaderTypeInfo* getTypeInfo(GFXShaderConstType type); - + static bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo); + U8 name[32]; static const char* constTypeToString(GFXShaderConstType constType, bool sampler = false, bool matrix = false); diff --git a/Engine/source/shaderGen/shaderOp.cpp b/Engine/source/shaderGen/shaderOp.cpp index 8a514f21c..d8979b011 100644 --- a/Engine/source/shaderGen/shaderOp.cpp +++ b/Engine/source/shaderGen/shaderOp.cpp @@ -26,53 +26,6 @@ #include "shaderOp.h" -bool ShaderOp::resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo) -{ - outVar = nullptr; - outInfo = nullptr; - - // DIRECT VAR - if (Var* v = dynamic_cast(elem)) - { - outVar = v; - outInfo = getTypeInfo(stringToConstType((const char*)v->type)); - return outInfo != nullptr; - } - - // INDEX OP: arrVar[index] - if (IndexOp* idx = dynamic_cast(elem)) - { - Var* arr = dynamic_cast(idx->mInput[0]); - if (!arr) - return false; - - const ShaderTypeInfo* arrInfo = getTypeInfo(stringToConstType((const char*)arr->type)); - if (!arrInfo) - return false; - - // array element type = same as var type but no array dimension - outVar = arr; - outInfo = arrInfo; - return true; - } - - if (CastOp* cast = dynamic_cast(elem)) - { - Var* castVar = dynamic_cast(cast->mInput[0]); - if (!castVar) - return false; - - const ShaderTypeInfo* castInfo = getTypeInfo(cast->mTargetType);// get the casts target type. - if (!castInfo) - return false; - - outVar = castVar; - outInfo = castInfo; - } - - return false; -} - //************************************************************************** // Shader Operations //************************************************************************** @@ -396,32 +349,40 @@ void MatrixInitializeOp::print(Stream& stream) U32 count = 0; for (U32 elem = 0; elem < mInitialVals.size(); elem++) { - LangElement* writeOut = NULL; - Var* curVar = dynamic_cast(mInitialVals[elem]); - if (curVar) // is a var + LangElement* initElem = mInitialVals[elem]; + Var* initVar = nullptr; + const ShaderTypeInfo* initInfo = nullptr; + + if (!resolveSourceType(initElem, initVar, initInfo)) { - const ShaderTypeInfo* curInfo = getTypeInfo(stringToConstType((const char*)curVar->type)); - if (!curInfo) // no info, cant do it cleanly. - return; + return; + } - const U32 curSize = curInfo->cols; + if (dynamic_cast(initElem) || dynamic_cast(initElem)) // if we are a cast or index, write out. + { + count += initInfo->cols; + initElem->print(stream); + } + else if(dynamic_cast(initElem)) // we are a var (hopefully) + { + const U32 varSize = initInfo->cols; + const bool cast = cols != varSize; - // if we are an array - if (curVar->arraySize > 1) + if (initVar->arraySize > 1) { - for (U32 arr = 0; arr < curVar->arraySize; arr++) + for (U32 arr = 0; arr < initVar->arraySize; arr++) { - writeOut = new IndexOp(curVar, arr); - if (curSize != cols) + initElem = new IndexOp(initVar, arr); + if (cast) { - CastOp* cast = new CastOp(writeOut, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); + CastOp* cast = new CastOp(initElem, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); cast->print(stream); count += cols; } else { - writeOut->print(stream); - count += curSize; + initElem->print(stream); + count += varSize; } if (count < matSize) @@ -432,16 +393,16 @@ void MatrixInitializeOp::print(Stream& stream) } else { - if (curSize != cols) + if (cast) { - CastOp* cast = new CastOp(curVar, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); + CastOp* cast = new CastOp(initElem, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); cast->print(stream); count += cols; } else { - curVar->print(stream); - count += curSize; + initElem->print(stream); + count += varSize; } if (count < matSize) @@ -450,18 +411,9 @@ void MatrixInitializeOp::print(Stream& stream) } } } - else - { - // Non-var LangElement, assume it produces correct vector - mInitialVals[elem]->print(stream); - count += cols; - - if (count < matSize) - WRITESTR(",\r\n"); - } } - // If not enough elements → pad with zeros + // If not enough elements → pad with identity while (count < matSize) { U32 row = count / cols; @@ -512,7 +464,7 @@ void MatrixMultiplyOp::print(Stream& stream) return; } - LangElement* rightElem = mInput[0]; + LangElement* rightElem = mInput[1]; Var* rightVar = nullptr; const ShaderTypeInfo* rightInfo = nullptr; diff --git a/Engine/source/shaderGen/shaderOp.h b/Engine/source/shaderGen/shaderOp.h index 7fb3da152..d7035d496 100644 --- a/Engine/source/shaderGen/shaderOp.h +++ b/Engine/source/shaderGen/shaderOp.h @@ -52,11 +52,8 @@ ///************************************************************************** class ShaderOp : public LangElement { -protected: - LangElement * mInput[2]; - public: - bool resolveSourceType(LangElement* elem, Var*& outVar, const ShaderTypeInfo*& outInfo); + LangElement* mInput[2]; ShaderOp( LangElement *in1, LangElement *in2 ); }; From 6a6ab76f36cc81b5c37f32de6b378f281a892683 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Sun, 30 Nov 2025 14:47:27 +0000 Subject: [PATCH 06/17] Update shaderOp.cpp --- Engine/source/shaderGen/shaderOp.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Engine/source/shaderGen/shaderOp.cpp b/Engine/source/shaderGen/shaderOp.cpp index d8979b011..3becd8fed 100644 --- a/Engine/source/shaderGen/shaderOp.cpp +++ b/Engine/source/shaderGen/shaderOp.cpp @@ -375,8 +375,8 @@ void MatrixInitializeOp::print(Stream& stream) initElem = new IndexOp(initVar, arr); if (cast) { - CastOp* cast = new CastOp(initElem, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); - cast->print(stream); + CastOp* castOp = new CastOp(initElem, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); + castOp->print(stream); count += cols; } else @@ -395,8 +395,8 @@ void MatrixInitializeOp::print(Stream& stream) { if (cast) { - CastOp* cast = new CastOp(initElem, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); - cast->print(stream); + CastOp* castOp = new CastOp(initElem, (GFXShaderConstType)(GFXSCT_Float + (cols - 1))); + castOp->print(stream); count += cols; } else From 7ea1cb28435f1f0297a451c14e3d0e7e6822c558 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Mon, 8 Dec 2025 07:16:00 +0000 Subject: [PATCH 07/17] ShaderGen stage dynamics ShaderGen now generates a ShaderData class to simplify macro switchups ShaderData caches mInstancingFormat ShaderGen now creates a cache of the files that already exist, and if it exists it will return and use that file instead of regenning a new one. Vertex files can be used for multiple pixel files and vice versa Requires partial shadernode setup due to changes on how shader feature parameters are handled --- .../source/forest/hlsl/windDeformationHLSL.h | 2 + Engine/source/gfx/gfxAPI.h | 1 + Engine/source/gfx/gfxShader.cpp | 8 +- Engine/source/gfx/gfxShader.h | 4 +- .../hlsl/advancedLightingFeaturesHLSL.h | 2 + .../hlsl/deferredShadingFeaturesHLSL.h | 6 + Engine/source/materials/shaderData.cpp | 53 +++- Engine/source/materials/shaderData.h | 10 +- .../source/shaderGen/HLSL/accuFeatureHLSL.h | 12 +- .../shaderGen/HLSL/debugVizFeatureHLSL.h | 2 + .../source/shaderGen/HLSL/shaderFeatureHLSL.h | 13 + .../source/shaderGen/conditionerFeature.cpp | 2 +- Engine/source/shaderGen/conditionerFeature.h | 6 +- Engine/source/shaderGen/langElement.h | 2 + Engine/source/shaderGen/shaderDependency.cpp | 10 +- Engine/source/shaderGen/shaderDependency.h | 11 +- Engine/source/shaderGen/shaderFeature.h | 2 + Engine/source/shaderGen/shaderGen.cpp | 296 ++++++++++++------ Engine/source/shaderGen/shaderGen.h | 29 +- Engine/source/terrain/hlsl/terrFeatureHLSL.h | 3 + 20 files changed, 333 insertions(+), 141 deletions(-) diff --git a/Engine/source/forest/hlsl/windDeformationHLSL.h b/Engine/source/forest/hlsl/windDeformationHLSL.h index 99aa49fe0..08e2a19a6 100644 --- a/Engine/source/forest/hlsl/windDeformationHLSL.h +++ b/Engine/source/forest/hlsl/windDeformationHLSL.h @@ -46,6 +46,8 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; } + String getName() override { return "Wind Effect"; diff --git a/Engine/source/gfx/gfxAPI.h b/Engine/source/gfx/gfxAPI.h index a3610c3c4..00f219166 100644 --- a/Engine/source/gfx/gfxAPI.h +++ b/Engine/source/gfx/gfxAPI.h @@ -44,6 +44,7 @@ DefineEnumType( GFXTextureFilterType ); DefineEnumType( GFXCullMode ); DefineEnumType( GFXStencilOp ); DefineEnumType( GFXBlendOp ); +DefineEnumType(GFXShaderConstType); DefineEnumType( GFXAdapterType ); DECLARE_STRUCT( GFXVideoMode ); diff --git a/Engine/source/gfx/gfxShader.cpp b/Engine/source/gfx/gfxShader.cpp index 62f6aff96..af4930d63 100644 --- a/Engine/source/gfx/gfxShader.cpp +++ b/Engine/source/gfx/gfxShader.cpp @@ -42,8 +42,12 @@ GFXShader::GFXShader() GFXShader::~GFXShader() { - Torque::FS::RemoveChangeNotification( mVertexFile, this, &GFXShader::_onFileChanged ); - Torque::FS::RemoveChangeNotification( mPixelFile, this, &GFXShader::_onFileChanged ); + if (!mVertexFile.isEmpty()) + Torque::FS::RemoveChangeNotification( mVertexFile, this, &GFXShader::_onFileChanged ); + if (!mPixelFile.isEmpty()) + Torque::FS::RemoveChangeNotification( mPixelFile, this, &GFXShader::_onFileChanged ); + if (!mGeometryFile.isEmpty()) + Torque::FS::RemoveChangeNotification(mGeometryFile, this, &GFXShader::_onFileChanged); SAFE_DELETE(mInstancingFormat); } diff --git a/Engine/source/gfx/gfxShader.h b/Engine/source/gfx/gfxShader.h index edb20a127..720d76290 100644 --- a/Engine/source/gfx/gfxShader.h +++ b/Engine/source/gfx/gfxShader.h @@ -77,7 +77,9 @@ enum GFXShaderStage GEOMETRY_SHADER = BIT(2), DOMAIN_SHADER = BIT(3), HULL_SHADER = BIT(4), - COMPUTE_SHADER = BIT(5) + COMPUTE_SHADER = BIT(5), + ALL_STAGES = VERTEX_SHADER | PIXEL_SHADER | GEOMETRY_SHADER | + DOMAIN_SHADER | HULL_SHADER | COMPUTE_SHADER }; /// Instances of this struct are returned GFXShaderConstBuffer diff --git a/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h b/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h index 17dd9b86d..fd1560f16 100644 --- a/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h +++ b/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h @@ -138,6 +138,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Sub-Surface Approximation [Deferred]"; diff --git a/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h b/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h index 472f22115..b7fd23f9e 100644 --- a/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h +++ b/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h @@ -57,6 +57,8 @@ public: const MaterialFeatureData &fd ) override; U32 getOutputTargets(const MaterialFeatureData& fd) const override; + + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } }; class ORMConfigVarsHLSL : public ShaderFeatureHLSL @@ -68,6 +70,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } }; class GlowMapHLSL : public ShaderFeatureHLSL @@ -78,6 +82,8 @@ public: void processPix(Vector &componentList, const MaterialFeatureData &fd) override; + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + U32 getOutputTargets(const MaterialFeatureData& fd) const override; Resources getResources(const MaterialFeatureData& fd) override; diff --git a/Engine/source/materials/shaderData.cpp b/Engine/source/materials/shaderData.cpp index 85b6d1446..253838573 100644 --- a/Engine/source/materials/shaderData.cpp +++ b/Engine/source/materials/shaderData.cpp @@ -78,6 +78,8 @@ ShaderData::ShaderData() mOGLVertexShaderName = StringTable->EmptyString(); mOGLPixelShaderName = StringTable->EmptyString(); mOGLGeometryShaderName = StringTable->EmptyString(); + + mInstancingFormat = NULL; } void ShaderData::initPersistFields() @@ -204,7 +206,7 @@ const Vector& ShaderData::_getMacros() return mShaderMacros; } -GFXShader* ShaderData::getShader( const Vector ¯os ) +GFXShader* ShaderData::getShader( const Vector ¯os) { PROFILE_SCOPE( ShaderData_GetShader ); @@ -224,7 +226,7 @@ GFXShader* ShaderData::getShader( const Vector ¯os ) // Create the shader instance... if it fails then // bail out and return nothing to the caller. - GFXShader *shader = _createShader( finalMacros ); + GFXShader *shader = _createShader( finalMacros); if ( !shader ) return NULL; @@ -235,7 +237,7 @@ GFXShader* ShaderData::getShader( const Vector ¯os ) return shader; } -GFXShader* ShaderData::_createShader( const Vector ¯os ) +GFXShader* ShaderData::_createShader( const Vector ¯os) { F32 pixver = mPixVersion; if ( mUseDevicePixVersion ) @@ -257,30 +259,32 @@ GFXShader* ShaderData::_createShader( const Vector ¯os ) { case Direct3D11: { - if (mDXVertexShaderName != String::EmptyString) + if (mDXVertexShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mDXVertexShaderName); - if (mDXPixelShaderName != String::EmptyString) + if (mDXPixelShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mDXPixelShaderName); - if (mDXGeometryShaderName != String::EmptyString) + if (mDXGeometryShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mDXGeometryShaderName); success = shader->init( pixver, macros, - samplers); + samplers, + mInstancingFormat); break; } case OpenGL: { - if(mOGLVertexShaderName != String::EmptyString) + if(mOGLVertexShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mOGLVertexShaderName); - if (mOGLPixelShaderName != String::EmptyString) + if (mOGLPixelShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mOGLPixelShaderName); - if (mOGLGeometryShaderName != String::EmptyString) + if (mOGLGeometryShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mOGLGeometryShaderName); success = shader->init( pixver, macros, - samplers); + samplers, + mInstancingFormat); break; } @@ -348,6 +352,33 @@ void ShaderData::_onLMActivate( const char *lm, bool activate ) reloadAllShaders(); } +void ShaderData::setShaderStageFile(GFXShaderStage stage, String fileName) +{ + const bool isGL = GFX->getAdapterType() == GFXAdapterType::OpenGL; + switch (stage) + { + case VERTEX_SHADER: + isGL ? mOGLVertexShaderName = StringTable->insert(fileName) : mDXVertexShaderName = StringTable->insert(fileName); + break; + case PIXEL_SHADER: + isGL ? mOGLPixelShaderName = StringTable->insert(fileName) : mDXPixelShaderName = StringTable->insert(fileName); + break; + case GEOMETRY_SHADER: + isGL ? mOGLGeometryShaderName = StringTable->insert(fileName) : mDXGeometryShaderName = StringTable->insert(fileName); + break; + case DOMAIN_SHADER: + break; + case HULL_SHADER: + break; + case COMPUTE_SHADER: + break; + case ALL_STAGES: + break; + default: + break; + } +} + bool ShaderData::hasSamplerDef(const String &_samplerName, int &pos) const { String samplerName = _samplerName.startsWith("$") ? _samplerName : "$"+_samplerName; diff --git a/Engine/source/materials/shaderData.h b/Engine/source/materials/shaderData.h index 2fa63271f..9b9af2a15 100644 --- a/Engine/source/materials/shaderData.h +++ b/Engine/source/materials/shaderData.h @@ -80,10 +80,8 @@ protected: /// them if the content has changed. const Vector& _getMacros(); - /// Helper for converting an array of macros - /// into a formatted string. - void _stringizeMacros(const Vector& macros, - String* outString); + // the instancing format. + GFXVertexFormat* mInstancingFormat; /// Creates a new shader returning NULL on error. GFXShader* _createShader(const Vector& macros); @@ -106,8 +104,11 @@ public: void setSamplerName(const String& name, int idx) { mSamplerNames[idx] = name; } String getSamplerName(int idx) const { return mSamplerNames[idx]; } + void setShaderStageFile(GFXShaderStage stage, String fileName); + bool hasSamplerDef(const String& samplerName, int& pos) const; bool hasRTParamsDef(const int pos) const { return mRTParams[pos]; } + void setInstancingFormat(GFXVertexFormat* instancingFormat) { mInstancingFormat = instancingFormat; } ShaderData(); @@ -122,6 +123,7 @@ public: /// all loaded ShaderData objects in the system. static void reloadAllShaders(); + void setPixVersion(F32 pixVersion) { mPixVersion = pixVersion; } /// Returns the required pixel shader version for this shader. F32 getPixVersion() const { return mPixVersion; } diff --git a/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h b/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h index 5f4682178..beef20dbc 100644 --- a/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h @@ -98,6 +98,8 @@ public: } } + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Accu Scale"; } }; @@ -118,6 +120,8 @@ public: } } + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Accu Direction"; } }; @@ -138,6 +142,8 @@ public: } } + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Accu Strength"; } }; @@ -158,6 +164,8 @@ public: } } + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Accu Coverage"; } }; @@ -179,7 +187,9 @@ public: } } + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Accu Specular"; } }; -#endif \ No newline at end of file +#endif diff --git a/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h b/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h index 53c0151e9..b799d0eb2 100644 --- a/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h @@ -37,5 +37,7 @@ public: void processPix(Vector& componentList, const MaterialFeatureData& fd) override; + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Debug Viz"; } }; diff --git a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h index 627f1019a..57873b03d 100644 --- a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h @@ -180,6 +180,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } U32 getOutputTargets( const MaterialFeatureData &fd ) const override { return mOutputTargetMask; } }; @@ -307,6 +309,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + Material::BlendOp getBlendOp() override{ return Material::None; } U32 getOutputTargets(const MaterialFeatureData &fd) const override; @@ -558,6 +562,9 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + String getName() override { return "Glow Mask"; @@ -582,6 +589,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + Material::BlendOp getBlendOp() override { return Material::None; } String getName() override { return "HDR Output"; } @@ -626,6 +635,8 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; } + String getName() override { return "Particle Normal Generation Feature"; @@ -672,6 +683,8 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; } + String getName() override { return "Hardware Skinning"; } }; diff --git a/Engine/source/shaderGen/conditionerFeature.cpp b/Engine/source/shaderGen/conditionerFeature.cpp index 1de0b7681..76e2b8230 100644 --- a/Engine/source/shaderGen/conditionerFeature.cpp +++ b/Engine/source/shaderGen/conditionerFeature.cpp @@ -227,7 +227,7 @@ void ConditionerFeature::printFooterComment( MethodType methodType, const String meta->addStatement( new GenOp( "\r\n\r\n" ) ); } -void ConditionerMethodDependency::print( Stream &s ) const +void ConditionerMethodDependency::print( Stream &s, GFXShaderStage stage) const { mConditioner->_printMethod(mMethodType, mConditioner->getShaderMethodName(mMethodType), s); } diff --git a/Engine/source/shaderGen/conditionerFeature.h b/Engine/source/shaderGen/conditionerFeature.h index b834418d7..5de418bb6 100644 --- a/Engine/source/shaderGen/conditionerFeature.h +++ b/Engine/source/shaderGen/conditionerFeature.h @@ -126,9 +126,11 @@ protected: public: ConditionerMethodDependency( ConditionerFeature *conditioner, const ConditionerFeature::MethodType methodType ) : - mConditioner(conditioner), mMethodType(methodType) {} + mConditioner(conditioner), mMethodType(methodType) { + stages = (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER); + } - void print( Stream &s ) const override; + void print( Stream &s, GFXShaderStage stage ) const override; // Auto insert information into a macro virtual void createMethodMacro( const String &methodName, Vector ¯os ); diff --git a/Engine/source/shaderGen/langElement.h b/Engine/source/shaderGen/langElement.h index 3105e8ce5..f5d4f9206 100644 --- a/Engine/source/shaderGen/langElement.h +++ b/Engine/source/shaderGen/langElement.h @@ -124,6 +124,8 @@ enum ConstantSortPosition cspPotentialPrimitive, /// Updated one per pass cspPass, + /// Set once per scene + cspScene, /// Count var, do not use csp_Count }; diff --git a/Engine/source/shaderGen/shaderDependency.cpp b/Engine/source/shaderGen/shaderDependency.cpp index 22d30572f..e14948904 100644 --- a/Engine/source/shaderGen/shaderDependency.cpp +++ b/Engine/source/shaderGen/shaderDependency.cpp @@ -30,6 +30,7 @@ ShaderIncludeDependency::ShaderIncludeDependency( const Torque::Path &pathToInclude ) : mIncludePath( pathToInclude ) { + stages = (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER); } bool ShaderIncludeDependency::operator==( const ShaderDependency &cmpTo ) const @@ -39,9 +40,12 @@ bool ShaderIncludeDependency::operator==( const ShaderDependency &cmpTo ) const static_cast( &cmpTo )->mIncludePath == mIncludePath ); } -void ShaderIncludeDependency::print( Stream &s ) const +void ShaderIncludeDependency::print( Stream &s , GFXShaderStage stage) const { // Print the include... all shaders support #includes. - String include = String::ToString( "#include \"%s\"\r\n", mIncludePath.getFullPath().c_str() ); - s.write( include.length(), include.c_str() ); + if (stages & stage) + { + String include = String::ToString("#include \"%s\"\r\n", mIncludePath.getFullPath().c_str()); + s.write(include.length(), include.c_str()); + } } diff --git a/Engine/source/shaderGen/shaderDependency.h b/Engine/source/shaderGen/shaderDependency.h index 63a6de48b..967d5cada 100644 --- a/Engine/source/shaderGen/shaderDependency.h +++ b/Engine/source/shaderGen/shaderDependency.h @@ -26,7 +26,9 @@ #ifndef _PATH_H_ #include "core/util/path.h" #endif - +#ifndef _GFXSHADER_H_ +#include "gfx/gfxShader.h" +#endif class Stream; @@ -35,6 +37,7 @@ class Stream; class ShaderDependency { public: + U32 stages; virtual ~ShaderDependency() {} /// Compare this dependency to another one. @@ -44,7 +47,7 @@ public: } /// Print the dependency into the header of a shader. - virtual void print( Stream &s ) const = 0; + virtual void print( Stream &s, GFXShaderStage stage) const = 0; }; @@ -60,7 +63,7 @@ public: ShaderIncludeDependency( const Torque::Path &pathToInclude ); bool operator==( const ShaderDependency &cmpTo ) const override; - void print( Stream &s ) const override; + void print( Stream &s, GFXShaderStage stage) const override; }; -#endif // _SHADER_DEPENDENCY_H_ \ No newline at end of file +#endif // _SHADER_DEPENDENCY_H_ diff --git a/Engine/source/shaderGen/shaderFeature.h b/Engine/source/shaderGen/shaderFeature.h index 9bed4e7ac..c19bab55b 100644 --- a/Engine/source/shaderGen/shaderFeature.h +++ b/Engine/source/shaderGen/shaderFeature.h @@ -237,6 +237,8 @@ public: /// Allows the feature to add macros to vertex shader compiles. virtual void processVertMacros( Vector ¯os, const MaterialFeatureData &fd ) {}; + virtual U32 getShaderStages() { return (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER); } + /// Identifies what type of blending a feature uses. This is used to /// group features with the same blend operation together in a multipass /// situation. diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index 16db02946..b212d5b33 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -38,6 +38,48 @@ #include "shaderGen/GLSL/customFeatureGLSL.h" #endif +static const U32 gStageOrder[] = +{ + GFXShaderStage::VERTEX_SHADER, + GFXShaderStage::HULL_SHADER, + GFXShaderStage::DOMAIN_SHADER, + GFXShaderStage::GEOMETRY_SHADER, + GFXShaderStage::PIXEL_SHADER, + GFXShaderStage::COMPUTE_SHADER +}; + +static const char* _getStagePostfix(GFXShaderStage stage) +{ + switch (stage) + { + case GFXShaderStage::VERTEX_SHADER: return "_V"; + case GFXShaderStage::HULL_SHADER: return "_H"; + case GFXShaderStage::DOMAIN_SHADER: return "_D"; + case GFXShaderStage::GEOMETRY_SHADER: return "_G"; + case GFXShaderStage::PIXEL_SHADER: return "_P"; + case GFXShaderStage::COMPUTE_SHADER: return "_C"; + } + + return "_U"; // Unknown +} + +// Generate a single 64bit hash from the input string. +// +// Don't get paranoid! This has 1 in 18446744073709551616 +// chance for collision... it won't happen in this lifetime. +// +String getHashForName(const String& in) +{ + String cacheKey = in; + cacheKey.replace("\n", " "); + U64 hash = Torque::hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0); + hash = convertHostToLEndian(hash); + U32 high = (U32)(hash >> 32); + U32 low = (U32)(hash & 0x00000000FFFFFFFF); + cacheKey = String::ToString("%x%x", high, low); + return cacheKey; +} + MODULE_BEGIN( ShaderGen ) MODULE_INIT_BEFORE( GFX ) @@ -99,6 +141,8 @@ void ShaderGen::initShaderGen() return; const GFXAdapterType adapterType = GFX->getAdapterType(); + const bool isGl = adapterType == GFXAdapterType::OpenGL; + if (!mInitDelegates[adapterType]) return; @@ -134,14 +178,22 @@ void ShaderGen::initShaderGen() // Delete the auto-generated conditioner include file. Torque::FS::Remove( "shadergen:/" + ConditionerFeature::ConditionerIncludeFileName ); + Vector fileList; + String pattern = "*."; + pattern += isGl ? "glsl" : "hlsl"; + S32 numShaderFiles = Torque::FS::FindByPattern("shadergen:/", pattern, false, fileList); + for (U32 i = 0; i < numShaderFiles; i++) + { + Torque::Path filePath = fileList[i]; + mFileCache[filePath.getFileName()] = true; + } + // build our type maps. LangElement::buildTypeMaps(); } -void ShaderGen::generateShader( const MaterialFeatureData &featureData, - char *vertFile, - char *pixFile, - F32 *pixVersion, +void ShaderGen::generateShader( const MaterialFeatureData& featureData, + ShaderData* shaderData, const GFXVertexFormat *vertexFormat, const char* cacheName, Vector ¯os) @@ -150,69 +202,111 @@ void ShaderGen::generateShader( const MaterialFeatureData &featureData, mFeatureData = featureData; mVertexFormat = vertexFormat; + mInstancingFormat.clear(); _uninit(); _init(); - char vertShaderName[256]; - char pixShaderName[256]; + const bool skipRegen = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); + const FeatureSet& features = mFeatureData.features; + U32 stages = 0; - // Note: We use a postfix of _V/_P here so that it sorts the matching - // vert and pixel shaders together when listed alphabetically. - dSprintf( vertShaderName, sizeof(vertShaderName), "shadergen:/%s_V.%s", cacheName, mFileEnding.c_str() ); - dSprintf( pixShaderName, sizeof(pixShaderName), "shadergen:/%s_P.%s", cacheName, mFileEnding.c_str() ); - - dStrcpy( vertFile, vertShaderName, 256 ); - dStrcpy( pixFile, pixShaderName, 256 ); - - // this needs to change - need to optimize down to ps v.1.1 - *pixVersion = GFX->getPixelShaderVersion(); - - if ( !Con::getBoolVariable( "ShaderGen::GenNewShaders", true ) ) + // loop through and see which stages this featureset is expecting to make. + for (U32 i = 0; i < features.getCount(); i++) { - // If we are not regenerating the shader we will return here. - // But we must fill in the shader macros first! - _processVertFeatures( macros, true ); - _processPixFeatures( macros, true ); + const FeatureType& type = features.getAt(i); + ShaderFeature* feat = FEATUREMGR->getByType(type); + stages |= feat->getShaderStages(); - return; } - // create vertex shader - //------------------------ - FileStream* s = new FileStream(); - if(!s->open(vertShaderName, Torque::FS::File::Write )) + for (U32 s = 0; s < (sizeof(gStageOrder) / sizeof(U32)); s++) { - AssertFatal(false, "Failed to open Shader Stream" ); - return; + U32 stage = gStageOrder[s]; + + // skip unused stages + if (!(stages & stage)) + continue; + + GFXShaderStage curStage = (GFXShaderStage)stage; + + char fileName[256]; + const char* postfix = _getStagePostfix(curStage); + String stageName; + + if (curStage & GFXShaderStage::VERTEX_SHADER) + stageName += vertexFormat->getDescription(); + + // build our filename. + for (U32 i = 0; i < features.getCount(); i++) + { + const FeatureType& type = features.getAt(i); + if (stage & FEATUREMGR->getByType(type)->getShaderStages()) + { + stageName += type.getName().c_str(); + } + } + + stageName = getHashForName(stageName); + stageName += postfix; + + FileCacheSet::iterator file = mFileCache.find(stageName); + if (file != mFileCache.end()) + { + // set the shaderdata file for this stage, shaderdata ptr needs to be passed in here. + dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str()); + shaderData->setShaderStageFile(curStage, fileName); + continue; + } + + mFileCache[stageName] = true; + + dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str()); + + shaderData->setShaderStageFile(curStage, fileName); + + mOutput = new MultiLine; + FileStream* stream = new FileStream(); + if (!stream->open(fileName, Torque::FS::File::Write)) + { + AssertFatal(false, "Failed to open Shader Stream"); + return; + } + + switch (curStage) + { + case VERTEX_SHADER: + _processVertFeatures(macros, skipRegen); + if (skipRegen) + continue; + _printVertShader(*stream); + ((ShaderConnector*)mComponents[C_CONNECTOR])->reset(); + break; + case PIXEL_SHADER: + _processPixFeatures(macros, skipRegen); + if (skipRegen) + continue; + _printPixShader(*stream); + break; + case GEOMETRY_SHADER: + break; + case DOMAIN_SHADER: + break; + case HULL_SHADER: + break; + case COMPUTE_SHADER: + break; + case ALL_STAGES: + break; + default: + break; + } + + delete stream; + LangElement::deleteElements(); + } - - mOutput = new MultiLine; - mInstancingFormat.clear(); - _processVertFeatures(macros); - _printVertShader( *s ); - delete s; - - ((ShaderConnector*)mComponents[C_CONNECTOR])->reset(); - LangElement::deleteElements(); - - // create pixel shader - //------------------------ - s = new FileStream(); - if(!s->open(pixShaderName, Torque::FS::File::Write )) - { - AssertFatal(false, "Failed to open Shader Stream" ); - delete s; - return; - } - - mOutput = new MultiLine; - _processPixFeatures(macros); - _printPixShader( *s ); - - delete s; - LangElement::deleteElements(); } void ShaderGen::_init() @@ -267,7 +361,7 @@ void ShaderGen::_processVertFeatures( Vector ¯os, bool macro else feature = FEATUREMGR->getByType( type ); - if ( feature ) + if ( feature && (feature->getShaderStages() & GFXShaderStage::VERTEX_SHADER)) { feature->setProcessIndex( index ); @@ -315,7 +409,7 @@ void ShaderGen::_processPixFeatures( Vector ¯os, bool macros feature = FEATUREMGR->createFeature(type, args); else feature = FEATUREMGR->getByType(type); - if ( feature ) + if ( feature && (feature->getShaderStages() & GFXShaderStage::PIXEL_SHADER)) { feature->setProcessIndex( index ); @@ -377,7 +471,7 @@ void ShaderGen::_printFeatureList(Stream &stream) mPrinter->printLine(stream, ""); } -void ShaderGen::_printDependencies(Stream &stream) +void ShaderGen::_printDependencies(Stream &stream, GFXShaderStage stage) { Vector dependencies; @@ -415,7 +509,7 @@ void ShaderGen::_printDependencies(Stream &stream) mPrinter->printLine(stream, "// Dependencies:"); for( S32 i = 0; i < dependencies.size(); i++ ) - dependencies[i]->print( stream ); + dependencies[i]->print( stream, stage); mPrinter->printLine(stream, ""); } @@ -430,7 +524,7 @@ void ShaderGen::_printVertShader( Stream &stream ) { mPrinter->printShaderHeader(stream); - _printDependencies(stream); // TODO: Split into vert and pix dependencies? + _printDependencies(stream, GFXShaderStage::VERTEX_SHADER); // TODO: Split into vert and pix dependencies? _printFeatureList(stream); // print out structures @@ -452,7 +546,7 @@ void ShaderGen::_printPixShader( Stream &stream ) { mPrinter->printShaderHeader(stream); - _printDependencies(stream); // TODO: Split into vert and pix dependencies? + _printDependencies(stream, GFXShaderStage::PIXEL_SHADER); // TODO: Split into vert and pix dependencies? _printFeatureList(stream); mComponents[C_CONNECTOR]->print( stream, false ); @@ -469,60 +563,60 @@ void ShaderGen::_printPixShader( Stream &stream ) mPrinter->printPixelShaderCloser(stream); } -GFXShader* ShaderGen::getShader( const MaterialFeatureData &featureData, const GFXVertexFormat *vertexFormat, const Vector *macros, const Vector &samplers ) +GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GFXVertexFormat* vertexFormat, const Vector* macros, const Vector& samplers) { - PROFILE_SCOPE( ShaderGen_GetShader ); + PROFILE_SCOPE(ShaderGen_GetShader); - const FeatureSet &features = featureData.codify(); + const FeatureSet& features = featureData.codify(); // Build a description string from the features // and vertex format combination ( and macros ). String shaderDescription = vertexFormat->getDescription() + features.getDescription(); - // Generate a single 64bit hash from the description string. - // - // Don't get paranoid! This has 1 in 18446744073709551616 - // chance for collision... it won't happen in this lifetime. - // - shaderDescription.replace("\n", " "); - U64 hash = Torque::hash64( (const U8*)shaderDescription.c_str(), shaderDescription.length(), 0 ); - hash = convertHostToLEndian(hash); - U32 high = (U32)( hash >> 32 ); - U32 low = (U32)( hash & 0x00000000FFFFFFFF ); - String cacheKey = String::ToString( "%x%x", high, low ); - // return shader if exists - GFXShader *match = mProcShaders[cacheKey]; - if ( match ) - return match; - // if not, then create it - char vertFile[256]; - char pixFile[256]; - F32 pixVersion; + String cacheKey = getHashForName(shaderDescription); - Vector shaderMacros; - shaderMacros.push_back( GFXShaderMacro( "TORQUE_SHADERGEN" ) ); - if ( macros ) - shaderMacros.merge( *macros ); - generateShader( featureData, vertFile, pixFile, &pixVersion, vertexFormat, cacheKey, shaderMacros ); - - GFXShader *shader = GFX->createShader(); - shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, vertFile); - shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, pixFile); - - if (!shader->init(pixVersion, shaderMacros, samplers, &mInstancingFormat)) + ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey); + if (dat != mProcShaderData.end()) { - delete shader; - return NULL; + Vector shaderMacros; + shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN")); + if (macros) + shaderMacros.merge(*macros); + + // should we loop vertex shader features to build mInstancingFormat before sending it down to see old hob? + return dat->value->getShader(shaderMacros); } - mProcShaders[cacheKey] = shader; + ShaderData* shaderData = new ShaderData; - return shader; + shaderData->setPixVersion(GFX->getPixelShaderVersion()); + + for (U32 samp = 0; samp < samplers.size(); samp++) + { + shaderData->setSamplerName(samplers[samp], samp); + } + + Vector shaderMacros; + shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN")); + if (macros) + shaderMacros.merge(*macros); + + generateShader(featureData, shaderData, vertexFormat, cacheKey, shaderMacros); + + shaderData->setInstancingFormat(&mInstancingFormat); + shaderData->registerObject(); + + mProcShaderData[cacheKey] = shaderData; + return shaderData->getShader(shaderMacros); } void ShaderGen::flushProceduralShaders() { - // The shaders are reference counted, so we - // just need to clear the map. - mProcShaders.clear(); + for (auto data : mProcShaderData) + { + data.value->deleteObject(); + } + + mProcShaderData.clear(); + mFileCache.clear(); } diff --git a/Engine/source/shaderGen/shaderGen.h b/Engine/source/shaderGen/shaderGen.h index 8ff8ef0f6..f19306772 100644 --- a/Engine/source/shaderGen/shaderGen.h +++ b/Engine/source/shaderGen/shaderGen.h @@ -46,6 +46,10 @@ #ifndef _MATERIALFEATUREDATA_H_ #include "materials/materialFeatureData.h" #endif +#ifndef _SHADERDATA_H_ +#include "materials/shaderData.h" +#endif // !_SHADERDATA_H_ + /// Base class used by shaderGen to be API agnostic. Subclasses implement the various methods /// in an API specific way. @@ -145,13 +149,11 @@ public: /// the vertex and pixel shader files. pixVersion is also filled in by /// this function. /// @param assignNum used to assign a specific number as the filename - void generateShader( const MaterialFeatureData &featureData, - char *vertFile, - char *pixFile, - F32 *pixVersion, - const GFXVertexFormat *vertexFormat, + void generateShader( const MaterialFeatureData& featureData, + ShaderData* shaderData, + const GFXVertexFormat* vertexFormat, const char* cacheName, - Vector ¯os); + Vector& macros); // Returns a shader that implements the features listed by dat. GFXShader* getShader( const MaterialFeatureData &dat, const GFXVertexFormat *vertexFormat, const Vector *macros, const Vector &samplers ); @@ -191,10 +193,15 @@ protected: FeatureInitSignal mFeatureInitSignal; bool mRegisteredWithGFX; Torque::FS::FileSystemRef mMemFS; - - /// Map of cache string -> shaders - typedef Map ShaderMap; - ShaderMap mProcShaders; + + /// + /// Map of shaderdata, string should be built up of stage files + /// + typedef HashMap> ShaderDataMap; + ShaderDataMap mProcShaderData; + + typedef HashMap FileCacheSet; // we use a hashmap because it is quicker for finding. + FileCacheSet mFileCache; ShaderGen(); @@ -215,7 +222,7 @@ protected: /// print out the processed features to the file stream void _printFeatures( Stream &stream ); - void _printDependencies( Stream &stream ); + void _printDependencies( Stream &stream, GFXShaderStage stage); void _processPixFeatures( Vector ¯os, bool macrosOnly = false ); void _printPixShader( Stream &stream ); diff --git a/Engine/source/terrain/hlsl/terrFeatureHLSL.h b/Engine/source/terrain/hlsl/terrFeatureHLSL.h index b65b1335b..9901faf34 100644 --- a/Engine/source/terrain/hlsl/terrFeatureHLSL.h +++ b/Engine/source/terrain/hlsl/terrFeatureHLSL.h @@ -153,6 +153,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } Resources getResources( const MaterialFeatureData &fd ) override; @@ -182,6 +184,7 @@ public: void processPix(Vector &componentList, const MaterialFeatureData &fd) override; String getName() override { return "Blank Matinfo map"; } + U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } }; class TerrainHeightMapBlendHLSL : public TerrainFeatHLSL From f5414c9e6459ea2b4a43f69f8cd399800d9b8b31 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Mon, 8 Dec 2025 07:46:33 +0000 Subject: [PATCH 08/17] Update shaderFeature.cpp --- Engine/source/shaderGen/shaderFeature.cpp | 137 ++++++---------------- 1 file changed, 35 insertions(+), 102 deletions(-) diff --git a/Engine/source/shaderGen/shaderFeature.cpp b/Engine/source/shaderGen/shaderFeature.cpp index 6dfd7b553..eb9fb472a 100644 --- a/Engine/source/shaderGen/shaderFeature.cpp +++ b/Engine/source/shaderGen/shaderFeature.cpp @@ -27,141 +27,74 @@ #include "shaderGen/shaderOp.h" -void ShaderFeature::addDependency( const ShaderDependency *dependsOn ) +void ShaderFeature::addDependency(const ShaderDependency* dependsOn) { - for ( U32 i = 0; i < mDependencies.size(); i++ ) + for (U32 i = 0; i < mDependencies.size(); i++) { - if ( *mDependencies[i] == *dependsOn ) + if (*mDependencies[i] == *dependsOn) return; } - mDependencies.push_back( dependsOn ); + mDependencies.push_back(dependsOn); } -Var* ShaderFeature::getVertTexCoord(const String& name) +ShaderFeature::Resources ShaderFeature::getResources(const MaterialFeatureData& fd) { - Var* inTex = NULL; - - for (U32 i = 0; i < LangElement::elementList.size(); i++) - { - if (!String::compare((char*)LangElement::elementList[i]->name, name.c_str())) - { - inTex = dynamic_cast(LangElement::elementList[i]); - if (inTex) - { - // NOTE: This used to do this check... - // - // String::compare( (char*)inTex->structName, "IN" ) - // - // ... to ensure that the var was from the input - // vertex structure, but this kept some features - // ( ie. imposter vert ) from decoding their own - // coords for other features to use. - // - // If we run into issues with collisions between - // IN vars and local vars we may need to revise. - - break; - } - } - } - - return inTex; + Resources temp; + return temp; } -LangElement* ShaderFeature::expandNormalMap(LangElement* sampleNormalOp, LangElement* normalDecl, LangElement* normalVar, const MaterialFeatureData& fd) -{ - MultiLine* meta = new MultiLine; - const bool hasBc3 = fd.features.hasFeature(MFT_IsBC3nm, getProcessIndex()); - const bool hasBc5 = fd.features.hasFeature(MFT_IsBC5nm, getProcessIndex()); - - if (hasBc3 || hasBc5) - { - if (fd.features[MFT_ImposterVert]) - { - // The imposter system uses object space normals and - // encodes them with the z axis in the alpha component. - meta->addStatement(new GenOp(" @ = @( normalize( @.xyw * 2.0 - 1.0 ), 0.0 ); // Obj DXTnm\r\n", normalDecl, new TypeOp(GFXSCT_Float4), sampleNormalOp)); - } - else if (hasBc3) - { - // BC3 Swizzle trick - meta->addStatement(new GenOp(" @ = @( @.ag * 2.0 - 1.0, 0.0, 0.0 ); // DXTnm\r\n", normalDecl, new TypeOp(GFXSCT_Float4), sampleNormalOp)); - meta->addStatement(new GenOp(" @.z = sqrt( 1.0 - dot( @.xy, @.xy ) ); // DXTnm\r\n", normalVar, normalVar, normalVar)); - } - else if (hasBc5) - { - // BC5 - meta->addStatement(new GenOp(" @ = @( @.gr * 2.0 - 1.0, 0.0, 0.0 ); // bc5nm\r\n", normalDecl, new TypeOp(GFXSCT_Float4), sampleNormalOp)); - meta->addStatement(new GenOp(" @.z = sqrt( 1.0 - dot( @.xy, @.xy ) ); // bc5nm\r\n", normalVar, normalVar, normalVar)); - } - } - else - { - meta->addStatement(new GenOp(" @ = @;\r\n", normalDecl, sampleNormalOp)); - meta->addStatement(new GenOp(" @.xyz = @.xyz * 2.0 - 1.0;\r\n", normalVar, normalVar)); - } - - return meta; -} - -ShaderFeature::Resources ShaderFeature::getResources( const MaterialFeatureData &fd ) -{ - Resources temp; - return temp; -} - -const char* ShaderFeature::getOutputTargetVarName( OutputTarget target ) const +const char* ShaderFeature::getOutputTargetVarName(OutputTarget target) const { const char* targName = "col"; - switch(target) + switch (target) { - case DefaultTarget: - targName = "col"; - break; + case DefaultTarget: + targName = "col"; + break; - case RenderTarget1: + case RenderTarget1: targName = "col1"; - break; + break; - case RenderTarget2: - targName = "col2"; - break; + case RenderTarget2: + targName = "col2"; + break; - case RenderTarget3: - targName = "col3"; - break; + case RenderTarget3: + targName = "col3"; + break; - case RenderTarget4: - targName = "col4"; - break; + case RenderTarget4: + targName = "col4"; + break; - case RenderTarget5: - targName = "col5"; - break; + case RenderTarget5: + targName = "col5"; + break; } return targName; } -Var* ShaderFeature::findOrCreateLocal( const char *name, - const char *type, - MultiLine *multi ) +Var* ShaderFeature::findOrCreateLocal(const char* name, + const char* type, + MultiLine* multi) { - Var *outVar = (Var*)LangElement::find( name ); - if ( !outVar ) + Var* outVar = (Var*)LangElement::find(name); + if (!outVar) { outVar = new Var; - outVar->setType( type ); - outVar->setName( name ); - multi->addStatement( new GenOp( " @;\r\n", new DecOp( outVar ) ) ); + outVar->setType(type); + outVar->setName(name); + multi->addStatement(new GenOp(" @;\r\n", new DecOp(outVar))); } return outVar; } -void ShaderFeature::setInstancingFormat(GFXVertexFormat *format) +void ShaderFeature::setInstancingFormat(GFXVertexFormat* format) { mInstancingFormat = format; } From d22e3ebfc98d0a408a63d50f6265c99a384334e9 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Mon, 8 Dec 2025 11:23:09 +0000 Subject: [PATCH 09/17] Other cache changes Added index to the cache name Moved the hash out to the torque namespace DX now outputs the compiled shader --- Engine/source/core/util/hashFunction.cpp | 19 +++++++ Engine/source/core/util/hashFunction.h | 2 + Engine/source/gfx/D3D11/gfxD3D11Shader.cpp | 66 +++++++++++++++++++++- Engine/source/materials/shaderData.cpp | 2 + Engine/source/shaderGen/shaderGen.cpp | 23 +------- 5 files changed, 91 insertions(+), 21 deletions(-) diff --git a/Engine/source/core/util/hashFunction.cpp b/Engine/source/core/util/hashFunction.cpp index 69206dc30..dbc30e287 100644 --- a/Engine/source/core/util/hashFunction.cpp +++ b/Engine/source/core/util/hashFunction.cpp @@ -31,6 +31,8 @@ #include "core/util/hashFunction.h" +#include "core/util/endian.h" + namespace Torque { @@ -268,4 +270,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval ) return c; } +// Generate a single 64bit hash from the input string. +// +// Don't get paranoid! This has 1 in 18446744073709551616 +// chance for collision... it won't happen in this lifetime. +// +String getStringHash64(const String& in) +{ + String cacheKey = in; + cacheKey.replace("\n", " "); + U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0); + hash = convertHostToLEndian(hash); + U32 high = (U32)(hash >> 32); + U32 low = (U32)(hash & 0x00000000FFFFFFFF); + cacheKey = String::ToString("%x%x", high, low); + return cacheKey; +} + } // namespace diff --git a/Engine/source/core/util/hashFunction.h b/Engine/source/core/util/hashFunction.h index 02ecd019b..f3a57cc9a 100644 --- a/Engine/source/core/util/hashFunction.h +++ b/Engine/source/core/util/hashFunction.h @@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval); extern U64 hash64(const U8 *k, U32 length, U64 initval); +extern String getStringHash64(const String& in); + } #endif // _HASHFUNCTION_H_ diff --git a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp index 6259be330..4a919cffd 100644 --- a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp +++ b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp @@ -681,6 +681,24 @@ bool GFXD3D11Shader::_init() return true; } +static String buildMacroHash(const D3D_SHADER_MACRO* defines) +{ + String combined; + + if (!defines) + return ""; + + for (const D3D_SHADER_MACRO* m = defines; m->Name != nullptr; ++m) + { + combined += m->Name; + combined += "="; + combined += m->Definition ? m->Definition : ""; + combined += ";"; + } + + return Torque::getStringHash64(combined); +} + bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, GFXShaderStage shaderStage, const D3D_SHADER_MACRO *defines) @@ -706,8 +724,40 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, Con::printf( "Compiling Shader: '%s'", filePath.getFullPath().c_str() ); #endif + String macroHash = buildMacroHash(defines); + + Torque::Path cachePath = filePath; + cachePath.setExtension("tso"); + cachePath.setFileName(cachePath.getFileName() + "_" + macroHash); + + if (Torque::FS::IsFile(cachePath)) + { + Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(filePath); + Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath); + + if (rawFile != NULL && cachedFile != NULL) + { + if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime()) + { + + FileStream fs; + if (fs.open(cachePath, Torque::FS::File::Read)) + { + U32 size = fs.getStreamSize(); + + D3DCreateBlob(size, &code); + fs.read(size, code->GetBufferPointer()); + + res = 1; + } + } + } + } + + bool loadedFromCache = (code != NULL); + // Is it an HLSL shader? - if(filePath.getExtension().equal("hlsl", String::NoCase)) + if(filePath.getExtension().equal("hlsl", String::NoCase) && !loadedFromCache) { // Set this so that the D3DInclude::Open will have this // information for relative paths. @@ -788,6 +838,20 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, AssertISV(SUCCEEDED(res), "Unable to compile shader!"); + // succeeded write out a cache + if (!loadedFromCache) + { + if (SUCCEEDED(res) && code) + { + // Save cache + FileStream out; + if (out.open(cachePath, Torque::FS::File::Write)) + { + out.write(code->GetBufferSize(), code->GetBufferPointer()); + } + } + } + if(code != NULL) { switch (shaderStage) diff --git a/Engine/source/materials/shaderData.cpp b/Engine/source/materials/shaderData.cpp index 253838573..8e987e4dc 100644 --- a/Engine/source/materials/shaderData.cpp +++ b/Engine/source/materials/shaderData.cpp @@ -219,6 +219,8 @@ GFXShader* ShaderData::getShader( const Vector ¯os) String cacheKey; GFXShaderMacro::stringize( macros, &cacheKey ); + cacheKey = Torque::getStringHash64(cacheKey); + // Lookup the shader for this instance. ShaderCache::Iterator iter = mShaders.find( cacheKey ); if ( iter != mShaders.end() ) diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index b212d5b33..d9f69e523 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -63,23 +63,6 @@ static const char* _getStagePostfix(GFXShaderStage stage) return "_U"; // Unknown } -// Generate a single 64bit hash from the input string. -// -// Don't get paranoid! This has 1 in 18446744073709551616 -// chance for collision... it won't happen in this lifetime. -// -String getHashForName(const String& in) -{ - String cacheKey = in; - cacheKey.replace("\n", " "); - U64 hash = Torque::hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0); - hash = convertHostToLEndian(hash); - U32 high = (U32)(hash >> 32); - U32 low = (U32)(hash & 0x00000000FFFFFFFF); - cacheKey = String::ToString("%x%x", high, low); - return cacheKey; -} - MODULE_BEGIN( ShaderGen ) MODULE_INIT_BEFORE( GFX ) @@ -244,11 +227,11 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, const FeatureType& type = features.getAt(i); if (stage & FEATUREMGR->getByType(type)->getShaderStages()) { - stageName += type.getName().c_str(); + stageName += type.getName() + "," + String::ToString(i); } } - stageName = getHashForName(stageName); + stageName = Torque::getStringHash64(stageName); stageName += postfix; FileCacheSet::iterator file = mFileCache.find(stageName); @@ -573,7 +556,7 @@ GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GF // and vertex format combination ( and macros ). String shaderDescription = vertexFormat->getDescription() + features.getDescription(); - String cacheKey = getHashForName(shaderDescription); + String cacheKey = Torque::getStringHash64(shaderDescription); ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey); if (dat != mProcShaderData.end()) From ae785f0e5e711b08ba8e543972ec2a252307a36c Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Mon, 8 Dec 2025 13:39:55 +0000 Subject: [PATCH 10/17] shader fixes some shader features require these to be checked by both vert and pixel stages (similar to alphatest) --- .../lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h | 2 +- Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h | 4 ++-- Engine/source/shaderGen/shaderGen.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h b/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h index b7fd23f9e..21abe52df 100644 --- a/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h +++ b/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h @@ -82,7 +82,7 @@ public: void processPix(Vector &componentList, const MaterialFeatureData &fd) override; - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + //U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } U32 getOutputTargets(const MaterialFeatureData& fd) const override; diff --git a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h index 57873b03d..58a3eb4bf 100644 --- a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h @@ -309,7 +309,7 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + //U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } Material::BlendOp getBlendOp() override{ return Material::None; } @@ -563,7 +563,7 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } + //U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } String getName() override { diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index d9f69e523..ab29205cf 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -227,7 +227,7 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, const FeatureType& type = features.getAt(i); if (stage & FEATUREMGR->getByType(type)->getShaderStages()) { - stageName += type.getName() + "," + String::ToString(i); + stageName += type.getName(); } } From 21534473c922da89a7e62cf958582a157ebc56db Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Mon, 8 Dec 2025 14:22:29 +0000 Subject: [PATCH 11/17] Update shaderGen.cpp vertex shaders need to regen on run again to get the vertex format setup --- Engine/source/shaderGen/shaderGen.cpp | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index ab29205cf..2cd88eb29 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -93,6 +93,8 @@ ShaderGen::~ShaderGen() { GFXDevice::getDeviceEventSignal().remove(this, &ShaderGen::_handleGFXEvent); _uninit(); + + mFileCache.clear(); } void ShaderGen::registerInitDelegate(GFXAdapterType adapterType, ShaderGenInitDelegate& initDelegate) @@ -190,7 +192,7 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, _uninit(); _init(); - const bool skipRegen = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); + bool skipRegen = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); const FeatureSet& features = mFeatureData.features; U32 stages = 0; @@ -240,7 +242,12 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, // set the shaderdata file for this stage, shaderdata ptr needs to be passed in here. dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str()); shaderData->setShaderStageFile(curStage, fileName); - continue; + if (!(curStage & GFXShaderStage::VERTEX_SHADER)) + { + continue; + } + + skipRegen = true; } mFileCache[stageName] = true; @@ -251,10 +258,13 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, mOutput = new MultiLine; FileStream* stream = new FileStream(); - if (!stream->open(fileName, Torque::FS::File::Write)) + if (!skipRegen) { - AssertFatal(false, "Failed to open Shader Stream"); - return; + if (!stream->open(fileName, Torque::FS::File::Write)) + { + AssertFatal(false, "Failed to open Shader Stream"); + return; + } } switch (curStage) @@ -350,15 +360,15 @@ void ShaderGen::_processVertFeatures( Vector ¯os, bool macro feature->processVertMacros( macros, mFeatureData ); - if ( macrosOnly ) - continue; - feature->setInstancingFormat( &mInstancingFormat ); feature->mVertexFormat = mVertexFormat; feature->processVert( mComponents, mFeatureData ); + if (macrosOnly) + continue; + String line; if ( index > -1 ) line = String::ToString( " // %s %d\r\n", feature->getName().c_str(), index ); @@ -601,5 +611,4 @@ void ShaderGen::flushProceduralShaders() } mProcShaderData.clear(); - mFileCache.clear(); } From 4ceeb07ea084b74464f486a934e529340d3a9712 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Wed, 10 Dec 2025 07:22:07 +0000 Subject: [PATCH 12/17] Update shaderGen.cpp move skipregen initialization --- Engine/source/shaderGen/shaderGen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index 2cd88eb29..f78033c34 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -192,7 +192,6 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, _uninit(); _init(); - bool skipRegen = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); const FeatureSet& features = mFeatureData.features; U32 stages = 0; @@ -214,6 +213,7 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, if (!(stages & stage)) continue; + bool skipRegen = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); GFXShaderStage curStage = (GFXShaderStage)stage; char fileName[256]; From dc2a9ed2e569c5436420e4ddf1ad64a8bea7d7cc Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Wed, 10 Dec 2025 13:24:25 +0000 Subject: [PATCH 13/17] few more features seem to cause conflicts move clearing of the map out of flushproceduralshaders, this function is now empty, not sure if this is going to cause issues elsewhere but what was happening was postfx manager was setting the hdr format after the level had been loaded, the materials for a few objects were being setup but then the shadergen process would execute on them again and it would clear the cache and cause double ups --- .../source/shaderGen/HLSL/shaderFeatureHLSL.h | 2 - Engine/source/shaderGen/shaderGen.cpp | 38 ++++++++++--------- Engine/source/terrain/hlsl/terrFeatureHLSL.h | 3 -- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h index 58a3eb4bf..aa2f59261 100644 --- a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h @@ -180,8 +180,6 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } U32 getOutputTargets( const MaterialFeatureData &fd ) const override { return mOutputTargetMask; } }; diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index f78033c34..23fa7a398 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -30,6 +30,7 @@ #include "gfx/gfxDevice.h" #include "core/memVolume.h" #include "core/module.h" +#include "console/persistenceManager.h" #ifdef TORQUE_D3D11 #include "shaderGen/HLSL/customFeatureHLSL.h" @@ -95,6 +96,14 @@ ShaderGen::~ShaderGen() _uninit(); mFileCache.clear(); + + for (ShaderDataMap::Pair data : mProcShaderData) + { + if (data.value->isProperlyAdded() && !data.value->isDeleted()) + data.value->unregisterObject(); + } + + mProcShaderData.clear(); } void ShaderGen::registerInitDelegate(GFXAdapterType adapterType, ShaderGenInitDelegate& initDelegate) @@ -272,14 +281,20 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, case VERTEX_SHADER: _processVertFeatures(macros, skipRegen); if (skipRegen) + { + LangElement::deleteElements(); continue; + } _printVertShader(*stream); ((ShaderConnector*)mComponents[C_CONNECTOR])->reset(); break; case PIXEL_SHADER: _processPixFeatures(macros, skipRegen); if (skipRegen) + { + LangElement::deleteElements(); continue; + } _printPixShader(*stream); break; case GEOMETRY_SHADER: @@ -568,14 +583,14 @@ GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GF String cacheKey = Torque::getStringHash64(shaderDescription); + Vector shaderMacros; + shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN")); + if (macros) + shaderMacros.merge(*macros); + ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey); if (dat != mProcShaderData.end()) { - Vector shaderMacros; - shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN")); - if (macros) - shaderMacros.merge(*macros); - // should we loop vertex shader features to build mInstancingFormat before sending it down to see old hob? return dat->value->getShader(shaderMacros); } @@ -589,26 +604,15 @@ GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GF shaderData->setSamplerName(samplers[samp], samp); } - Vector shaderMacros; - shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN")); - if (macros) - shaderMacros.merge(*macros); - generateShader(featureData, shaderData, vertexFormat, cacheKey, shaderMacros); shaderData->setInstancingFormat(&mInstancingFormat); - shaderData->registerObject(); - mProcShaderData[cacheKey] = shaderData; + mProcShaderData.insert(cacheKey, shaderData); return shaderData->getShader(shaderMacros); } void ShaderGen::flushProceduralShaders() { - for (auto data : mProcShaderData) - { - data.value->deleteObject(); - } - mProcShaderData.clear(); } diff --git a/Engine/source/terrain/hlsl/terrFeatureHLSL.h b/Engine/source/terrain/hlsl/terrFeatureHLSL.h index 9901faf34..b65b1335b 100644 --- a/Engine/source/terrain/hlsl/terrFeatureHLSL.h +++ b/Engine/source/terrain/hlsl/terrFeatureHLSL.h @@ -153,8 +153,6 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } Resources getResources( const MaterialFeatureData &fd ) override; @@ -184,7 +182,6 @@ public: void processPix(Vector &componentList, const MaterialFeatureData &fd) override; String getName() override { return "Blank Matinfo map"; } - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } }; class TerrainHeightMapBlendHLSL : public TerrainFeatHLSL From 2ca11b4a949d93463435e47df74e265f5d5a1b0b Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Wed, 10 Dec 2025 14:17:00 +0000 Subject: [PATCH 14/17] remove all stage specifics and dynamic naming always running into issues with this as at times features are coming in with a different index and that affects how another stage is interacting with that shader. This produces alot more shaders, previous run with these was around 500 but without it is producing over 800 (this includes tso compiled output) This may be something that will only work for the shadernodes as users will be designing those. --- .../source/forest/hlsl/windDeformationHLSL.h | 2 -- .../hlsl/advancedLightingFeaturesHLSL.h | 2 -- .../hlsl/deferredShadingFeaturesHLSL.h | 6 ----- .../source/shaderGen/HLSL/accuFeatureHLSL.h | 10 -------- .../shaderGen/HLSL/debugVizFeatureHLSL.h | 2 -- .../source/shaderGen/HLSL/shaderFeatureHLSL.h | 10 -------- Engine/source/shaderGen/shaderGen.cpp | 25 ++++++++++--------- 7 files changed, 13 insertions(+), 44 deletions(-) diff --git a/Engine/source/forest/hlsl/windDeformationHLSL.h b/Engine/source/forest/hlsl/windDeformationHLSL.h index 08e2a19a6..99aa49fe0 100644 --- a/Engine/source/forest/hlsl/windDeformationHLSL.h +++ b/Engine/source/forest/hlsl/windDeformationHLSL.h @@ -46,8 +46,6 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; - U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; } - String getName() override { return "Wind Effect"; diff --git a/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h b/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h index fd1560f16..17dd9b86d 100644 --- a/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h +++ b/Engine/source/lighting/advanced/hlsl/advancedLightingFeaturesHLSL.h @@ -138,8 +138,6 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Sub-Surface Approximation [Deferred]"; diff --git a/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h b/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h index 21abe52df..472f22115 100644 --- a/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h +++ b/Engine/source/lighting/advanced/hlsl/deferredShadingFeaturesHLSL.h @@ -57,8 +57,6 @@ public: const MaterialFeatureData &fd ) override; U32 getOutputTargets(const MaterialFeatureData& fd) const override; - - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } }; class ORMConfigVarsHLSL : public ShaderFeatureHLSL @@ -70,8 +68,6 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } }; class GlowMapHLSL : public ShaderFeatureHLSL @@ -82,8 +78,6 @@ public: void processPix(Vector &componentList, const MaterialFeatureData &fd) override; - //U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - U32 getOutputTargets(const MaterialFeatureData& fd) const override; Resources getResources(const MaterialFeatureData& fd) override; diff --git a/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h b/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h index beef20dbc..b97045681 100644 --- a/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h @@ -98,8 +98,6 @@ public: } } - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Accu Scale"; } }; @@ -120,8 +118,6 @@ public: } } - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Accu Direction"; } }; @@ -142,8 +138,6 @@ public: } } - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Accu Strength"; } }; @@ -164,8 +158,6 @@ public: } } - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Accu Coverage"; } }; @@ -187,8 +179,6 @@ public: } } - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Accu Specular"; } }; diff --git a/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h b/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h index b799d0eb2..53c0151e9 100644 --- a/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h @@ -37,7 +37,5 @@ public: void processPix(Vector& componentList, const MaterialFeatureData& fd) override; - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Debug Viz"; } }; diff --git a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h index aa2f59261..8acfb3991 100644 --- a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h @@ -307,8 +307,6 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - //U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - Material::BlendOp getBlendOp() override{ return Material::None; } U32 getOutputTargets(const MaterialFeatureData &fd) const override; @@ -561,8 +559,6 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - //U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - String getName() override { return "Glow Mask"; @@ -587,8 +583,6 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; - U32 getShaderStages() override { return GFXShaderStage::PIXEL_SHADER; } - Material::BlendOp getBlendOp() override { return Material::None; } String getName() override { return "HDR Output"; } @@ -633,8 +627,6 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; - U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; } - String getName() override { return "Particle Normal Generation Feature"; @@ -681,8 +673,6 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; - U32 getShaderStages() override { return GFXShaderStage::VERTEX_SHADER; } - String getName() override { return "Hardware Skinning"; } }; diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index 23fa7a398..38272b3e7 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -229,20 +229,21 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, const char* postfix = _getStagePostfix(curStage); String stageName; - if (curStage & GFXShaderStage::VERTEX_SHADER) - stageName += vertexFormat->getDescription(); + //if (curStage & GFXShaderStage::VERTEX_SHADER) + // stageName += vertexFormat->getDescription(); - // build our filename. - for (U32 i = 0; i < features.getCount(); i++) - { - const FeatureType& type = features.getAt(i); - if (stage & FEATUREMGR->getByType(type)->getShaderStages()) - { - stageName += type.getName(); - } - } + //// build our filename. + //for (U32 i = 0; i < features.getCount(); i++) + //{ + // const FeatureType& type = features.getAt(i); + // if (stage & FEATUREMGR->getByType(type)->getShaderStages()) + // { + // stageName += type.getName(); + // } + //} - stageName = Torque::getStringHash64(stageName); + //stageName = Torque::getStringHash64(stageName); + stageName = cacheName; stageName += postfix; FileCacheSet::iterator file = mFileCache.find(stageName); From 7d312d642915e66a84d39c16f74ca1d39133c8ab Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Fri, 12 Dec 2025 13:59:27 +0000 Subject: [PATCH 15/17] Update shaderGen.cpp 3 clean run throughs with switches between levels and did not see the same issues as before. so we should be able to reintroduce the stage seperation checks --- Engine/source/shaderGen/shaderGen.cpp | 69 ++++++++++++--------------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index 38272b3e7..2907fbb21 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -196,7 +196,6 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, mFeatureData = featureData; mVertexFormat = vertexFormat; - mInstancingFormat.clear(); _uninit(); _init(); @@ -222,28 +221,28 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, if (!(stages & stage)) continue; - bool skipRegen = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); + bool macrosOnly = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); + bool skipPrint = false; GFXShaderStage curStage = (GFXShaderStage)stage; char fileName[256]; const char* postfix = _getStagePostfix(curStage); String stageName; - //if (curStage & GFXShaderStage::VERTEX_SHADER) - // stageName += vertexFormat->getDescription(); + if (curStage & GFXShaderStage::VERTEX_SHADER) + stageName += vertexFormat->getDescription(); - //// build our filename. - //for (U32 i = 0; i < features.getCount(); i++) - //{ - // const FeatureType& type = features.getAt(i); - // if (stage & FEATUREMGR->getByType(type)->getShaderStages()) - // { - // stageName += type.getName(); - // } - //} + // build our filename. + for (U32 i = 0; i < features.getCount(); i++) + { + const FeatureType& type = features.getAt(i); + if (stage & FEATUREMGR->getByType(type)->getShaderStages()) + { + stageName += type.getName(); + } + } - //stageName = Torque::getStringHash64(stageName); - stageName = cacheName; + stageName = Torque::getStringHash64(stageName); stageName += postfix; FileCacheSet::iterator file = mFileCache.find(stageName); @@ -257,7 +256,7 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, continue; } - skipRegen = true; + skipPrint = true; } mFileCache[stageName] = true; @@ -266,9 +265,9 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, shaderData->setShaderStageFile(curStage, fileName); - mOutput = new MultiLine; + FileStream* stream = new FileStream(); - if (!skipRegen) + if (!skipPrint) { if (!stream->open(fileName, Torque::FS::File::Write)) { @@ -280,23 +279,20 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, switch (curStage) { case VERTEX_SHADER: - _processVertFeatures(macros, skipRegen); - if (skipRegen) - { - LangElement::deleteElements(); - continue; - } - _printVertShader(*stream); + mOutput = new MultiLine; + mInstancingFormat.clear(); + _processVertFeatures(macros, macrosOnly); + if(!skipPrint || macrosOnly) _printVertShader(*stream); + delete stream; ((ShaderConnector*)mComponents[C_CONNECTOR])->reset(); + LangElement::deleteElements(); break; case PIXEL_SHADER: - _processPixFeatures(macros, skipRegen); - if (skipRegen) - { - LangElement::deleteElements(); - continue; - } - _printPixShader(*stream); + mOutput = new MultiLine; + _processPixFeatures(macros, macrosOnly); + if (!skipPrint || macrosOnly)_printPixShader(*stream); + delete stream; + LangElement::deleteElements(); break; case GEOMETRY_SHADER: break; @@ -312,9 +308,6 @@ void ShaderGen::generateShader( const MaterialFeatureData& featureData, break; } - delete stream; - LangElement::deleteElements(); - } } @@ -376,15 +369,15 @@ void ShaderGen::_processVertFeatures( Vector ¯os, bool macro feature->processVertMacros( macros, mFeatureData ); + if (macrosOnly) + continue; + feature->setInstancingFormat( &mInstancingFormat ); feature->mVertexFormat = mVertexFormat; feature->processVert( mComponents, mFeatureData ); - if (macrosOnly) - continue; - String line; if ( index > -1 ) line = String::ToString( " // %s %d\r\n", feature->getName().c_str(), index ); From 7cd94f5de5024d43ba25ac90affefa7108e759e4 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Fri, 12 Dec 2025 14:39:30 +0000 Subject: [PATCH 16/17] tested and working hopefully it stays that way --- Engine/source/shaderGen/HLSL/accuFeatureHLSL.h | 10 ++++++++++ Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h | 2 ++ Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h | 13 +++++++++++++ 3 files changed, 25 insertions(+) diff --git a/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h b/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h index b97045681..1912ecbb8 100644 --- a/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/accuFeatureHLSL.h @@ -98,6 +98,8 @@ public: } } + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + String getName() override { return "Accu Scale"; } }; @@ -118,6 +120,8 @@ public: } } + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + String getName() override { return "Accu Direction"; } }; @@ -138,6 +142,8 @@ public: } } + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + String getName() override { return "Accu Strength"; } }; @@ -158,6 +164,8 @@ public: } } + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + String getName() override { return "Accu Coverage"; } }; @@ -179,6 +187,8 @@ public: } } + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + String getName() override { return "Accu Specular"; } }; diff --git a/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h b/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h index 53c0151e9..05ffc69a7 100644 --- a/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/debugVizFeatureHLSL.h @@ -37,5 +37,7 @@ public: void processPix(Vector& componentList, const MaterialFeatureData& fd) override; + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + String getName() override { return "Debug Viz"; } }; diff --git a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h index 8acfb3991..f5988c022 100644 --- a/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h +++ b/Engine/source/shaderGen/HLSL/shaderFeatureHLSL.h @@ -180,6 +180,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } U32 getOutputTargets( const MaterialFeatureData &fd ) const override { return mOutputTargetMask; } }; @@ -306,6 +308,7 @@ class DiffuseFeatureHLSL : public ShaderFeatureHLSL public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } Material::BlendOp getBlendOp() override{ return Material::None; } @@ -540,6 +543,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + Material::BlendOp getBlendOp() override { return Material::None; } String getName() override @@ -559,6 +564,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return ( GFXShaderStage::PIXEL_SHADER); } + String getName() override { return "Glow Mask"; @@ -583,6 +590,8 @@ public: void processPix( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return (GFXShaderStage::PIXEL_SHADER); } + Material::BlendOp getBlendOp() override { return Material::None; } String getName() override { return "HDR Output"; } @@ -627,6 +636,8 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return (GFXShaderStage::VERTEX_SHADER); } + String getName() override { return "Particle Normal Generation Feature"; @@ -673,6 +684,8 @@ public: void processVert( Vector &componentList, const MaterialFeatureData &fd ) override; + U32 getShaderStages() override { return (GFXShaderStage::VERTEX_SHADER); } + String getName() override { return "Hardware Skinning"; } }; From 1fd1b8e8571e7d63913baeb6abec754b34cf308f Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Thu, 19 Feb 2026 14:30:29 +0000 Subject: [PATCH 17/17] file stream close async This is just firing the mfile->close off to another thread for files that are caches of data such as dts and tso for shaders. What happens here is the closing of the filestream causes a hang as the os frees the file and writes metadata, we dont care about this as we are only ever caching data that we already have. next time torque loads it uses these cached versions not during runtime. --- Engine/source/core/stream/fileStream.cpp | 70 +++++++++++++++++-- Engine/source/core/stream/fileStream.h | 14 +++- Engine/source/gfx/D3D11/gfxD3D11Shader.cpp | 4 +- Engine/source/ts/assimp/assimpShapeLoader.cpp | 2 +- .../source/ts/collada/colladaShapeLoader.cpp | 2 +- 5 files changed, 78 insertions(+), 14 deletions(-) diff --git a/Engine/source/core/stream/fileStream.cpp b/Engine/source/core/stream/fileStream.cpp index 29ce3a933..a5c7e8c13 100644 --- a/Engine/source/core/stream/fileStream.cpp +++ b/Engine/source/core/stream/fileStream.cpp @@ -22,15 +22,68 @@ #include "platform/platform.h" #include "core/stream/fileStream.h" - +#include "platform/threads/threadPool.h" //----------------------------------------------------------------------------- // FileStream methods... //----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -FileStream::FileStream() +static void writeBufferToFile( + Torque::FS::FileRef& file, + U32 buffHead, + U32 buffTail, + const U8* buffer) { + if (!file || buffHead == FileStream::BUFFER_INVALID) + return; + + // match FileStream::flush logic + if (buffHead != file->getPosition()) + file->setPosition(buffHead, Torque::FS::File::Begin); + + U32 blockHead; + FileStream::calcBlockHead(buffHead, &blockHead); + + file->write((char*)buffer + (buffHead - blockHead), buffTail - buffHead + 1); +} + +struct FileCloseWorkItem : public ThreadPool::WorkItem +{ + Torque::FS::FileRef mFile; + +public: + FileCloseWorkItem(Torque::FS::FileRef file) + : mFile(file) + { + } + +protected: + void execute() override + { + if (!mFile) + return; + + // When platforms free a file they + // remove the handle which causes the stall + // as they write metadata. + mFile->close(); + } +}; + +void FileStream::dispatchAsyncClose() +{ + if (!mFile) + return; + + FileCloseWorkItem* job = new FileCloseWorkItem(mFile); + + ThreadPool::GLOBAL().queueWorkItem(job); +} + +//----------------------------------------------------------------------------- +FileStream::FileStream(AsyncMode flushMode) +{ + mAsyncMode = flushMode; dMemset(mBuffer, 0, sizeof(mBuffer)); // initialize the file stream init(); @@ -190,10 +243,13 @@ void FileStream::close() if (mDirty) flush(); - // and close the file - mFile->close(); - - AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed"); + if (mAsyncMode == Background) + dispatchAsyncClose(); + else + { + mFile->close(); + AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed"); + } mFile = NULL; } diff --git a/Engine/source/core/stream/fileStream.h b/Engine/source/core/stream/fileStream.h index 9ae20798a..c4c5ecd17 100644 --- a/Engine/source/core/stream/fileStream.h +++ b/Engine/source/core/stream/fileStream.h @@ -39,9 +39,17 @@ public: BUFFER_INVALID = 0xffffffff // file offsets must all be less than this }; + enum AsyncMode + { + Blocking, // current behavior + Background // write-behind + }; + typedef char Ch; //!< Character type. Only support char. public: - FileStream(); // default constructor + AsyncMode mAsyncMode; + void dispatchAsyncClose(); + FileStream(AsyncMode flushMode = Blocking); // default constructor virtual ~FileStream(); // destructor // This function will allocate a new FileStream and open it. @@ -64,7 +72,7 @@ public: //rjson compatibility bool Flush() { return flush(); } FileStream* clone() const override; - + static void calcBlockHead(const U32 i_position, U32* o_blockHead); protected: // more mandatory methods from Stream base class... bool _read(const U32 i_numBytes, void *o_pBuffer) override; @@ -73,7 +81,7 @@ protected: void init(); bool fillBuffer(const U32 i_startPosition); void clearBuffer(); - static void calcBlockHead(const U32 i_position, U32 *o_blockHead); + static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail); void setStatus(); diff --git a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp index 4a919cffd..b84f64907 100644 --- a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp +++ b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp @@ -842,9 +842,9 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, if (!loadedFromCache) { if (SUCCEEDED(res) && code) - { + { // Save cache - FileStream out; + FileStream out(FileStream::AsyncMode::Background); if (out.open(cachePath, Torque::FS::File::Write)) { out.write(code->GetBufferSize(), code->GetBufferPointer()); diff --git a/Engine/source/ts/assimp/assimpShapeLoader.cpp b/Engine/source/ts/assimp/assimpShapeLoader.cpp index 4bcc1df91..a3b78326d 100644 --- a/Engine/source/ts/assimp/assimpShapeLoader.cpp +++ b/Engine/source/ts/assimp/assimpShapeLoader.cpp @@ -1092,7 +1092,7 @@ TSShape* assimpLoadShape(const Torque::Path &path) // Cache the model to a DTS file for faster loading next time. cachedPath.setExtension("cached.dts"); // Cache the model to a DTS file for faster loading next time. - FileStream dtsStream; + FileStream dtsStream(FileStream::AsyncMode::Background); if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write)) { Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str()); diff --git a/Engine/source/ts/collada/colladaShapeLoader.cpp b/Engine/source/ts/collada/colladaShapeLoader.cpp index 66a859934..17a3557a2 100644 --- a/Engine/source/ts/collada/colladaShapeLoader.cpp +++ b/Engine/source/ts/collada/colladaShapeLoader.cpp @@ -803,7 +803,7 @@ TSShape* loadColladaShape(const Torque::Path &path) { // Cache the Collada model to a DTS file for faster loading next time. cachedPath.setExtension("cached.dts"); - FileStream dtsStream; + FileStream dtsStream(FileStream::AsyncMode::Background); if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write)) { Torque::FS::FileSystemRef ref = Torque::FS::GetFileSystem(daePath);