TorqueGameEngines/Torque3D
1
0
Fork 0
mirror of https://github.com/TorqueGameEngines/Torque3D.git synced 2026-07-09 05:34:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #1760 from marauder2k9-torque/ShaderGen-produce-shaderdata

Browse source 
Shader Gen to produce ShaderData
This commit is contained in:
Brian Roberts 2026-06-07 18:23:21 -05:00 • committed by GitHub
commit bac9ed99b3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 477 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

52
Engine/source/core/stream/fileStream.cpp
View file

@ -22,23 +22,57 @@
#include "platform/platform.h"
#include "core/stream/fileStream.h"
#include "platform/threads/threadPool.h"
//-----------------------------------------------------------------------------
// FileStream methods...
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
FileStream::FileStream()
struct FileCloseWorkItem : public ThreadPool::WorkItem
{
Torque::FS::FileRef mFile;
public:
FileCloseWorkItem(Torque::FS::FileRef file)
: mFile(file)
{
}
protected:
void execute() override
{
if (!mFile)
return;
// When platforms free a file they
// remove the handle which causes the stall
// as they write metadata.
mFile->close();
}
};
void FileStream::dispatchAsyncClose()
{
if (!mFile)
return;
FileCloseWorkItem* job = new FileCloseWorkItem(mFile);
ThreadPool::GLOBAL().queueWorkItem(job);
}
//-----------------------------------------------------------------------------
FileStream::FileStream(AsyncMode flushMode)
{
mAsyncMode = flushMode;
dMemset(mBuffer, 0, sizeof(mBuffer));
// initialize the file stream
init();
}
FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode)
FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode)
{
FileStream *newStream = new FileStream;
FileStream *newStream = new FileStream(flushMode);
bool success = newStream->open( inFileName, inMode );
@ -193,7 +227,13 @@ void FileStream::close()
// and close the file
mFile->close();
AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
if (mAsyncMode == Background)
dispatchAsyncClose();
else
{
mFile->close();
AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
}
mFile = NULL;
}

14
Engine/source/core/stream/fileStream.h
View file

@ -39,15 +39,23 @@ public:
BUFFER_INVALID = 0xffffffff // file offsets must all be less than this
};
enum AsyncMode
{
Blocking, // current behavior
Background // write-behind
};
typedef char Ch; //!< Character type. Only support char.
public:
FileStream(); // default constructor
AsyncMode mAsyncMode;
void dispatchAsyncClose();
FileStream(AsyncMode flushMode = Blocking);// default constructor
virtual ~FileStream(); // destructor
// This function will allocate a new FileStream and open it.
// If it fails to allocate or fails to open, it will return NULL.
// The caller is responsible for deleting the instance.
static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode);
static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode = Blocking);
// mandatory methods from Stream base class...
bool hasCapability(const Capability i_cap) const override;
@ -64,6 +72,7 @@ public:
//rjson compatibility
bool Flush() { return flush(); }
FileStream* clone() const override;
static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
protected:
// more mandatory methods from Stream base class...
@ -73,7 +82,6 @@ protected:
void init();
bool fillBuffer(const U32 i_startPosition);
void clearBuffer();
static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail);
void setStatus();

18
Engine/source/core/util/hashFunction.cpp
View file

@ -30,6 +30,7 @@
#include "platform/platform.h"
#include "core/util/hashFunction.h"
#include "core/util/endian.h"
namespace Torque
{
@ -268,4 +269,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval )
return c;
}
// Generate a single 64bit hash from the input string.
//
// Don't get paranoid! This has 1 in 18446744073709551616
// chance for collision... it won't happen in this lifetime.
//
String getStringHash64(const String& in)
{
String cacheKey = in;
cacheKey.replace("\n", " ");
U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0);
hash = convertHostToLEndian(hash);
U32 high = (U32)(hash >> 32);
U32 low = (U32)(hash & 0x00000000FFFFFFFF);
cacheKey = String::ToString("%x%x", high, low);
return cacheKey;
}
} // namespace

2
Engine/source/core/util/hashFunction.h
View file

@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval);
extern U64 hash64(const U8 *k, U32 length, U64 initval);
extern String getStringHash64(const String& in);
}
#endif // _HASHFUNCTION_H_

66
Engine/source/gfx/D3D11/gfxD3D11Shader.cpp
View file

@ -681,6 +681,24 @@ bool GFXD3D11Shader::_init()
return true;
}
static String buildMacroHash(const D3D_SHADER_MACRO* defines)
{
String combined;
if (!defines)
return "";
for (const D3D_SHADER_MACRO* m = defines; m->Name != nullptr; ++m)
{
combined += m->Name;
combined += "=";
combined += m->Definition ? m->Definition : "";
combined += ";";
}
return Torque::getStringHash64(combined);
}
bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
GFXShaderStage shaderStage,
const D3D_SHADER_MACRO *defines)
@ -706,8 +724,40 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
Con::printf( "Compiling Shader: '%s'", filePath.getFullPath().c_str() );
#endif
String macroHash = buildMacroHash(defines);
Torque::Path cachePath = filePath;
cachePath.setExtension("tso");
cachePath.setFileName(cachePath.getFileName() + "_" + macroHash);
if (Torque::FS::IsFile(cachePath))
{
Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(filePath);
Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath);
if (rawFile != NULL && cachedFile != NULL)
{
if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime())
{
FileStream fs;
if (fs.open(cachePath, Torque::FS::File::Read))
{
U32 size = fs.getStreamSize();
D3DCreateBlob(size, &code);
fs.read(size, code->GetBufferPointer());
res = 1;
}
}
}
}
bool loadedFromCache = (code != NULL);
// Is it an HLSL shader?
if(filePath.getExtension().equal("hlsl", String::NoCase))
if(filePath.getExtension().equal("hlsl", String::NoCase) && !loadedFromCache)
{
// Set this so that the D3DInclude::Open will have this
// information for relative paths.
@ -788,6 +838,20 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
AssertISV(SUCCEEDED(res), "Unable to compile shader!");
// succeeded write out a cache
if (!loadedFromCache)
{
if (SUCCEEDED(res) && code)
{
// Save cache
FileStream out(FileStream::AsyncMode::Background);
if (out.open(cachePath, Torque::FS::File::Write))
{
out.write(code->GetBufferSize(), code->GetBufferPointer());
}
}
}
if(code != NULL)
{
switch (shaderStage)

92
Engine/source/gfx/gl/gfxGLShader.cpp
View file

@ -542,6 +542,77 @@ bool GFXGLShader::_init()
macros.last().name = "TORQUE_VERTEX_SHADER";
macros.last().value = "";
Torque::Path cachePath = mVertexFile;
cachePath.setExtension("tso");
String macroHash;
GFXShaderMacro::stringize(macros, &macroHash);
macroHash = Torque::getStringHash64(macroHash);
String fileName = cachePath.getFileName();
if (!mVertexFile.isEmpty())
fileName += "_" + mVertexFile.getFileName();
if (!mPixelFile.isEmpty())
fileName += "_" + mPixelFile.getFileName();
if (!mGeometryFile.isEmpty())
fileName += "_" + mGeometryFile.getFileName();
fileName += "_" + macroHash;
cachePath.setFileName(fileName);
if (Torque::FS::IsFile(cachePath))
{
Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(mVertexFile);
Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath);
if (rawFile != NULL && cachedFile != NULL)
{
if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime())
{
FileStream fs;
if (fs.open(cachePath, Torque::FS::File::Read))
{
U32 size;
FrameAllocatorMarker bin;
GLenum gl_format;
fs.read(&gl_format);
fs.read(&size);
char* bin_data = (char*)bin.alloc(size);
fs.read(size, bin_data);
glProgramBinary(
mProgram,
gl_format,
bin_data,
size);
GLint linked;
glGetProgramiv(mProgram, GL_LINK_STATUS, &linked);
if (linked == GL_TRUE)
{
initConstantDescs();
initHandles();
// Notify Buffers we might have changed in size.
// If this was our first init then we won't have any activeBuffers
// to worry about unnecessarily calling.
Vector<GFXShaderConstBuffer*>::iterator biter = mActiveBuffers.begin();
for (; biter != mActiveBuffers.end(); biter++)
((GFXGLShaderConstBuffer*)(*biter))->onShaderReload(this);
return true;
}
}
}
}
}
// Default to true so we're "successful" if a vertex/pixel shader wasn't specified.
bool compiledVertexShader = true;
bool compiledPixelShader = true;
@ -659,6 +730,27 @@ bool GFXGLShader::_init()
if (linkStatus == GL_FALSE)
return false;
GLint binaryLength;
glGetProgramiv(mProgram, GL_PROGRAM_BINARY_LENGTH, &binaryLength);
if (binaryLength)
{
FrameAllocatorMarker bin;
char* bin_data = (char*)bin.alloc(binaryLength);
GLenum binaryFormat;
GLint length;
glGetProgramBinary(mProgram, binaryLength, &length, &binaryFormat, bin_data);
AssertWarn(length == binaryLength, "GFXGLShader: Binary length does not match");
FileStream out(FileStream::AsyncMode::Background);
if (out.open(cachePath, Torque::FS::File::Write))
{
out.write(binaryFormat);
out.write(length);
out.write(length, bin_data);
}
}
initConstantDescs();
initHandles();

49
Engine/source/materials/shaderData.cpp
View file

@ -71,6 +71,8 @@ ShaderData::ShaderData()
for( int i = 0; i < NumTextures; ++i)
mRTParams[i] = false;
mInstancingFormat = NULL;
mDXVertexShaderName = StringTable->EmptyString();
mDXPixelShaderName = StringTable->EmptyString();
mDXGeometryShaderName = StringTable->EmptyString();
@ -217,6 +219,8 @@ GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
String cacheKey;
GFXShaderMacro::stringize( macros, &cacheKey );
cacheKey = Torque::getStringHash64(cacheKey);
// Lookup the shader for this instance.
ShaderCache::Iterator iter = mShaders.find( cacheKey );
if ( iter != mShaders.end() )
@ -257,30 +261,32 @@ GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
{
case Direct3D11:
{
if (mDXVertexShaderName != String::EmptyString)
if (mDXVertexShaderName != StringTable->EmptyString())
shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mDXVertexShaderName);
if (mDXPixelShaderName != String::EmptyString)
if (mDXPixelShaderName != StringTable->EmptyString())
shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mDXPixelShaderName);
if (mDXGeometryShaderName != String::EmptyString)
if (mDXGeometryShaderName != StringTable->EmptyString())
shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mDXGeometryShaderName);
success = shader->init( pixver,
macros,
samplers);
samplers,
mInstancingFormat);
break;
}
case OpenGL:
{
if(mOGLVertexShaderName != String::EmptyString)
if(mOGLVertexShaderName != StringTable->EmptyString())
shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mOGLVertexShaderName);
if (mOGLPixelShaderName != String::EmptyString)
if (mOGLPixelShaderName != StringTable->EmptyString())
shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mOGLPixelShaderName);
if (mOGLGeometryShaderName != String::EmptyString)
if (mOGLGeometryShaderName != StringTable->EmptyString())
shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mOGLGeometryShaderName);
success = shader->init( pixver,
macros,
samplers);
samplers,
mInstancingFormat);
break;
}
@ -321,6 +327,33 @@ GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
return shader;
}
void ShaderData::setShaderStageFile(GFXShaderStage stage, String fileName)
{
const bool isGL = GFX->getAdapterType() == GFXAdapterType::OpenGL;
switch (stage)
{
case VERTEX_SHADER:
isGL ? mOGLVertexShaderName = StringTable->insert(fileName) : mDXVertexShaderName = StringTable->insert(fileName);
break;
case PIXEL_SHADER:
isGL ? mOGLPixelShaderName = StringTable->insert(fileName) : mDXPixelShaderName = StringTable->insert(fileName);
break;
case GEOMETRY_SHADER:
isGL ? mOGLGeometryShaderName = StringTable->insert(fileName) : mDXGeometryShaderName = StringTable->insert(fileName);
break;
case DOMAIN_SHADER:
break;
case HULL_SHADER:
break;
case COMPUTE_SHADER:
break;
case ALL_STAGES:
break;
default:
break;
}
}
void ShaderData::reloadShaders()
{
ShaderCache::Iterator iter = mShaders.begin();

10
Engine/source/materials/shaderData.h
View file

@ -80,11 +80,6 @@ protected:
/// them if the content has changed.
const Vector<GFXShaderMacro>& _getMacros();
/// Helper for converting an array of macros
/// into a formatted string.
void _stringizeMacros(const Vector<GFXShaderMacro>& macros,
String* outString);
/// Creates a new shader returning NULL on error.
GFXShader* _createShader(const Vector<GFXShaderMacro>& macros);
@ -98,6 +93,8 @@ protected:
String mSamplerNames[NumTextures];
bool mRTParams[NumTextures];
// the instancing format.
GFXVertexFormat* mInstancingFormat;
bool _checkDefinition(GFXShader* shader);
@ -105,9 +102,11 @@ public:
void setSamplerName(const String& name, int idx) { mSamplerNames[idx] = name; }
String getSamplerName(int idx) const { return mSamplerNames[idx]; }
void setShaderStageFile(GFXShaderStage stage, String fileName);
bool hasSamplerDef(const String& samplerName, int& pos) const;
bool hasRTParamsDef(const int pos) const { return mRTParams[pos]; }
void setInstancingFormat(GFXVertexFormat* instancingFormat) { mInstancingFormat = instancingFormat; }
ShaderData();
@ -122,6 +121,7 @@ public:
/// all loaded ShaderData objects in the system.
static void reloadAllShaders();
void setPixVersion(F32 pixVersion) { mPixVersion = pixVersion; }
/// Returns the required pixel shader version for this shader.
F32 getPixVersion() const { return mPixVersion; }

2
Engine/source/shaderGen/shaderFeature.h
View file

@ -237,6 +237,8 @@ public:
/// Allows the feature to add macros to vertex shader compiles.
virtual void processVertMacros( Vector<GFXShaderMacro> &macros, const MaterialFeatureData &fd ) {};
virtual U32 getShaderStages() { return (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER); }
/// Identifies what type of blending a feature uses. This is used to
/// group features with the same blend operation together in a multipass
/// situation.

278
Engine/source/shaderGen/shaderGen.cpp
View file

@ -30,6 +30,7 @@
#include "gfx/gfxDevice.h"
#include "core/memVolume.h"
#include "core/module.h"
#include "console/persistenceManager.h"
#ifdef TORQUE_D3D11
#include "shaderGen/HLSL/customFeatureHLSL.h"
@ -55,6 +56,32 @@ MODULE_BEGIN( ShaderGen )
MODULE_END;
static const U32 gStageOrder[] =
{
GFXShaderStage::VERTEX_SHADER,
GFXShaderStage::HULL_SHADER,
GFXShaderStage::DOMAIN_SHADER,
GFXShaderStage::GEOMETRY_SHADER,
GFXShaderStage::PIXEL_SHADER,
GFXShaderStage::COMPUTE_SHADER
};
static const char* _getStagePostfix(GFXShaderStage stage)
{
switch (stage)
{
case GFXShaderStage::VERTEX_SHADER: return "_V";
case GFXShaderStage::HULL_SHADER: return "_H";
case GFXShaderStage::DOMAIN_SHADER: return "_D";
case GFXShaderStage::GEOMETRY_SHADER: return "_G";
case GFXShaderStage::PIXEL_SHADER: return "_P";
case GFXShaderStage::COMPUTE_SHADER: return "_C";
}
return "_U"; // Unknown
}
String ShaderGen::smCommonShaderPath("shaders/common");
ShaderGen::ShaderGen()
@ -68,6 +95,14 @@ ShaderGen::~ShaderGen()
{
GFXDevice::getDeviceEventSignal().remove(this, &ShaderGen::_handleGFXEvent);
_uninit();
for (ShaderDataMap::Pair data : mProcShaderData)
{
if (data.value->isProperlyAdded() && !data.value->isDeleted())
data.value->unregisterObject();
}
mProcShaderData.clear();
}
void ShaderGen::registerInitDelegate(GFXAdapterType adapterType, ShaderGenInitDelegate& initDelegate)
@ -99,6 +134,7 @@ void ShaderGen::initShaderGen()
return;
const GFXAdapterType adapterType = GFX->getAdapterType();
const bool isGl = adapterType == GFXAdapterType::OpenGL;
if (!mInitDelegates[adapterType])
return;
@ -133,17 +169,25 @@ void ShaderGen::initShaderGen()
// Delete the auto-generated conditioner include file.
Torque::FS::Remove( "shadergen:/" + ConditionerFeature::ConditionerIncludeFileName );
Vector<String> fileList;
String pattern = "*.";
pattern += isGl ? "glsl" : "hlsl";
S32 numShaderFiles = Torque::FS::FindByPattern("shadergen:/", pattern, false, fileList);
for (U32 i = 0; i < numShaderFiles; i++)
{
Torque::Path filePath = fileList[i];
mFileCache[filePath.getFileName()] = true;
}
}
void ShaderGen::generateShader( const MaterialFeatureData &featureData,
char *vertFile,
char *pixFile,
F32 *pixVersion,
const GFXVertexFormat *vertexFormat,
const char* cacheName,
Vector<GFXShaderMacro> &macros)
void ShaderGen::generateShader(const MaterialFeatureData& featureData,
ShaderData* shaderData,
const GFXVertexFormat* vertexFormat,
const char* cacheName,
Vector<GFXShaderMacro>& macros)
{
PROFILE_SCOPE( ShaderGen_GenerateShader );
PROFILE_SCOPE(ShaderGen_GenerateShader);
mFeatureData = featureData;
mVertexFormat = vertexFormat;
@ -151,65 +195,115 @@ void ShaderGen::generateShader( const MaterialFeatureData &featureData,
_uninit();
_init();
char vertShaderName[256];
char pixShaderName[256];
const FeatureSet& features = mFeatureData.features;
U32 stages = 0;
// Note: We use a postfix of _V/_P here so that it sorts the matching
// vert and pixel shaders together when listed alphabetically.
dSprintf( vertShaderName, sizeof(vertShaderName), "shadergen:/%s_V.%s", cacheName, mFileEnding.c_str() );
dSprintf( pixShaderName, sizeof(pixShaderName), "shadergen:/%s_P.%s", cacheName, mFileEnding.c_str() );
dStrcpy( vertFile, vertShaderName, 256 );
dStrcpy( pixFile, pixShaderName, 256 );
// this needs to change - need to optimize down to ps v.1.1
*pixVersion = GFX->getPixelShaderVersion();
if ( !Con::getBoolVariable( "ShaderGen::GenNewShaders", true ) )
// loop through and see which stages this featureset is expecting to make.
for (U32 i = 0; i < features.getCount(); i++)
{
// If we are not regenerating the shader we will return here.
// But we must fill in the shader macros first!
_processVertFeatures( macros, true );
_processPixFeatures( macros, true );
const FeatureType& type = features.getAt(i);
ShaderFeature* feat = FEATUREMGR->getByType(type);
stages |= feat->getShaderStages();
return;
}
// create vertex shader
//------------------------
FileStream* s = new FileStream();
if(!s->open(vertShaderName, Torque::FS::File::Write ))
for (U32 s = 0; s < (sizeof(gStageOrder) / sizeof(U32)); s++)
{
AssertFatal(false, "Failed to open Shader Stream" );
return;
U32 stage = gStageOrder[s];
// skip unused stages
if (!(stages & stage))
continue;
bool macrosOnly = !Con::getBoolVariable("ShaderGen::GenNewShaders", true);
bool skipPrint = false;
GFXShaderStage curStage = (GFXShaderStage)stage;
char fileName[256];
const char* postfix = _getStagePostfix(curStage);
String stageName;
if (curStage & GFXShaderStage::VERTEX_SHADER)
stageName += vertexFormat->getDescription();
// build our filename.
for (U32 i = 0; i < features.getCount(); i++)
{
const FeatureType& type = features.getAt(i);
if (stage & FEATUREMGR->getByType(type)->getShaderStages())
{
stageName += type.getName();
}
}
stageName = Torque::getStringHash64(stageName);
stageName += postfix;
FileCacheSet::iterator file = mFileCache.find(stageName);
if (file != mFileCache.end())
{
// set the shaderdata file for this stage, shaderdata ptr needs to be passed in here.
dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str());
shaderData->setShaderStageFile(curStage, fileName);
if (!(curStage & GFXShaderStage::VERTEX_SHADER))
{
continue;
}
skipPrint = true;
}
mFileCache[stageName] = true;
dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str());
shaderData->setShaderStageFile(curStage, fileName);
FileStream* stream = new FileStream(FileStream::AsyncMode::Background);
if (!skipPrint)
{
if (!stream->open(fileName, Torque::FS::File::Write))
{
AssertFatal(false, "Failed to open Shader Stream");
return;
}
}
switch (curStage)
{
case VERTEX_SHADER:
mOutput = new MultiLine;
mInstancingFormat.clear();
_processVertFeatures(macros, macrosOnly);
if (!skipPrint || macrosOnly) _printVertShader(*stream);
delete stream;
((ShaderConnector*)mComponents[C_CONNECTOR])->reset();
LangElement::deleteElements();
break;
case PIXEL_SHADER:
mOutput = new MultiLine;
_processPixFeatures(macros, macrosOnly);
if (!skipPrint || macrosOnly)_printPixShader(*stream);
delete stream;
LangElement::deleteElements();
break;
case GEOMETRY_SHADER:
break;
case DOMAIN_SHADER:
break;
case HULL_SHADER:
break;
case COMPUTE_SHADER:
break;
case ALL_STAGES:
break;
default:
break;
}
}
mOutput = new MultiLine;
mInstancingFormat.clear();
_processVertFeatures(macros);
_printVertShader( *s );
delete s;
((ShaderConnector*)mComponents[C_CONNECTOR])->reset();
LangElement::deleteElements();
// create pixel shader
//------------------------
s = new FileStream();
if(!s->open(pixShaderName, Torque::FS::File::Write ))
{
AssertFatal(false, "Failed to open Shader Stream" );
delete s;
return;
}
mOutput = new MultiLine;
_processPixFeatures(macros);
_printPixShader( *s );
delete s;
LangElement::deleteElements();
}
void ShaderGen::_init()
@ -264,7 +358,7 @@ void ShaderGen::_processVertFeatures( Vector<GFXShaderMacro> &macros, bool macro
else
feature = FEATUREMGR->getByType( type );
if ( feature )
if ( feature && (feature->getShaderStages() & GFXShaderStage::VERTEX_SHADER) )
{
feature->setProcessIndex( index );
@ -312,7 +406,7 @@ void ShaderGen::_processPixFeatures( Vector<GFXShaderMacro> &macros, bool macros
feature = FEATUREMGR->createFeature(type, args);
else
feature = FEATUREMGR->getByType(type);
if ( feature )
if ( feature && (feature->getShaderStages() & GFXShaderStage::PIXEL_SHADER) )
{
feature->setProcessIndex( index );
@ -466,60 +560,50 @@ void ShaderGen::_printPixShader( Stream &stream )
mPrinter->printPixelShaderCloser(stream);
}
GFXShader* ShaderGen::getShader( const MaterialFeatureData &featureData, const GFXVertexFormat *vertexFormat, const Vector<GFXShaderMacro> *macros, const Vector<String> &samplers )
GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GFXVertexFormat* vertexFormat, const Vector<GFXShaderMacro>* macros, const Vector<String>& samplers)
{
PROFILE_SCOPE( ShaderGen_GetShader );
PROFILE_SCOPE(ShaderGen_GetShader);
const FeatureSet &features = featureData.codify();
const FeatureSet& features = featureData.codify();
// Build a description string from the features
// and vertex format combination ( and macros ).
String shaderDescription = vertexFormat->getDescription() + features.getDescription();
// Generate a single 64bit hash from the description string.
//
// Don't get paranoid! This has 1 in 18446744073709551616
// chance for collision... it won't happen in this lifetime.
//
shaderDescription.replace("\n", " ");
U64 hash = Torque::hash64( (const U8*)shaderDescription.c_str(), shaderDescription.length(), 0 );
hash = convertHostToLEndian(hash);
U32 high = (U32)( hash >> 32 );
U32 low = (U32)( hash & 0x00000000FFFFFFFF );
String cacheKey = String::ToString( "%x%x", high, low );
// return shader if exists
GFXShader *match = mProcShaders[cacheKey];
if ( match )
return match;
// if not, then create it
char vertFile[256];
char pixFile[256];
F32 pixVersion;
String cacheKey = Torque::getStringHash64(shaderDescription);
Vector<GFXShaderMacro> shaderMacros;
shaderMacros.push_back( GFXShaderMacro( "TORQUE_SHADERGEN" ) );
if ( macros )
shaderMacros.merge( *macros );
generateShader( featureData, vertFile, pixFile, &pixVersion, vertexFormat, cacheKey, shaderMacros );
shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN"));
if (macros)
shaderMacros.merge(*macros);
GFXShader *shader = GFX->createShader();
shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, vertFile);
shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, pixFile);
if (!shader->init(pixVersion, shaderMacros, samplers, &mInstancingFormat))
ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey);
if (dat != mProcShaderData.end())
{
delete shader;
return NULL;
// should we loop vertex shader features to build mInstancingFormat before sending it down to see old hob?
return dat->value->getShader(shaderMacros);
}
mProcShaders[cacheKey] = shader;
ShaderData* shaderData = new ShaderData;
return shader;
shaderData->setPixVersion(GFX->getPixelShaderVersion());
for (U32 samp = 0; samp < samplers.size(); samp++)
{
shaderData->setSamplerName(samplers[samp], samp);
}
generateShader(featureData, shaderData, vertexFormat, cacheKey, shaderMacros);
shaderData->setInstancingFormat(&mInstancingFormat);
mProcShaderData.insert(cacheKey, shaderData);
return shaderData->getShader(shaderMacros);
}
void ShaderGen::flushProceduralShaders()
{
// The shaders are reference counted, so we
// just need to clear the map.
mProcShaders.clear();
mProcShaderData.clear();
}

18
Engine/source/shaderGen/shaderGen.h
View file

@ -46,6 +46,9 @@
#ifndef _MATERIALFEATUREDATA_H_
#include "materials/materialFeatureData.h"
#endif
#ifndef _SHADERDATA_H_
#include "materials/shaderData.h"
#endif // !_SHADERDATA_H_
/// Base class used by shaderGen to be API agnostic. Subclasses implement the various methods
/// in an API specific way.
@ -146,9 +149,7 @@ public:
/// this function.
/// @param assignNum used to assign a specific number as the filename
void generateShader( const MaterialFeatureData &featureData,
char *vertFile,
char *pixFile,
F32 *pixVersion,
ShaderData* shaderData,
const GFXVertexFormat *vertexFormat,
const char* cacheName,
Vector<GFXShaderMacro> &macros);
@ -192,9 +193,14 @@ protected:
bool mRegisteredWithGFX;
Torque::FS::FileSystemRef mMemFS;
/// Map of cache string -> shaders
typedef Map<String, GFXShaderRef> ShaderMap;
ShaderMap mProcShaders;
/// <summary>
/// Map of shaderdata, string should be built up of stage files
/// </summary>
typedef HashMap<String, SimObjectPtr<ShaderData>> ShaderDataMap;
ShaderDataMap mProcShaderData;
typedef HashMap<String, bool> FileCacheSet; // we use a hashmap because it is quicker for finding.
FileCacheSet mFileCache;
ShaderGen();

2
Engine/source/ts/assimp/assimpShapeLoader.cpp
View file

@ -771,7 +771,7 @@ void AssimpShapeLoader::extractTexture(U32 index, aiTexture* pTex)
{ // Compressed format, write the data directly to disc
texPath.setExtension(pTex->achFormatHint);
FileStream *outputStream;
if ((outputStream = FileStream::createAndOpen(texPath.getFullPath(), Torque::FS::File::Write)) != NULL)
if ((outputStream = FileStream::createAndOpen(texPath.getFullPath(), Torque::FS::File::Write, FileStream::AsyncMode::Background)) != NULL)
{
outputStream->setPosition(0);
outputStream->write(pTex->mWidth, pTex->pcData);

2
Engine/source/ts/collada/colladaShapeLoader.cpp
View file

@ -720,7 +720,7 @@ static bool sReadCollada(const Torque::Path& path, TSShape*& res_shape)
// Cache the model to a DTS file for faster loading next time.
cachedPath.setExtension("cached.dts");
// Cache the model to a DTS file for faster loading next time.
FileStream dtsStream;
FileStream dtsStream(FileStream::AsyncMode::Background);
if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write))
{
Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str());