diff --git a/Engine/source/core/stream/fileStream.cpp b/Engine/source/core/stream/fileStream.cpp index 29ce3a933..df3dade77 100644 --- a/Engine/source/core/stream/fileStream.cpp +++ b/Engine/source/core/stream/fileStream.cpp @@ -22,23 +22,57 @@ #include "platform/platform.h" #include "core/stream/fileStream.h" - +#include "platform/threads/threadPool.h" //----------------------------------------------------------------------------- // FileStream methods... //----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -FileStream::FileStream() +struct FileCloseWorkItem : public ThreadPool::WorkItem { + Torque::FS::FileRef mFile; + +public: + FileCloseWorkItem(Torque::FS::FileRef file) + : mFile(file) + { + } + +protected: + void execute() override + { + if (!mFile) + return; + + // When platforms free a file they + // remove the handle which causes the stall + // as they write metadata. + mFile->close(); + } +}; + +void FileStream::dispatchAsyncClose() +{ + if (!mFile) + return; + + FileCloseWorkItem* job = new FileCloseWorkItem(mFile); + + ThreadPool::GLOBAL().queueWorkItem(job); +} + +//----------------------------------------------------------------------------- +FileStream::FileStream(AsyncMode flushMode) +{ + mAsyncMode = flushMode; dMemset(mBuffer, 0, sizeof(mBuffer)); // initialize the file stream init(); } -FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode) +FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode) { - FileStream *newStream = new FileStream; + FileStream *newStream = new FileStream(flushMode); bool success = newStream->open( inFileName, inMode ); @@ -193,7 +227,13 @@ void FileStream::close() // and close the file mFile->close(); - AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed"); + if (mAsyncMode == Background) + dispatchAsyncClose(); + else + { + mFile->close(); + AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed"); + } mFile = NULL; } diff --git a/Engine/source/core/stream/fileStream.h b/Engine/source/core/stream/fileStream.h index 9ae20798a..cb7364037 100644 --- a/Engine/source/core/stream/fileStream.h +++ b/Engine/source/core/stream/fileStream.h @@ -39,15 +39,23 @@ public: BUFFER_INVALID = 0xffffffff // file offsets must all be less than this }; + enum AsyncMode + { + Blocking, // current behavior + Background // write-behind + }; + typedef char Ch; //!< Character type. Only support char. public: - FileStream(); // default constructor + AsyncMode mAsyncMode; + void dispatchAsyncClose(); + FileStream(AsyncMode flushMode = Blocking);// default constructor virtual ~FileStream(); // destructor // This function will allocate a new FileStream and open it. // If it fails to allocate or fails to open, it will return NULL. // The caller is responsible for deleting the instance. - static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode); + static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode = Blocking); // mandatory methods from Stream base class... bool hasCapability(const Capability i_cap) const override; @@ -64,6 +72,7 @@ public: //rjson compatibility bool Flush() { return flush(); } FileStream* clone() const override; + static void calcBlockHead(const U32 i_position, U32 *o_blockHead); protected: // more mandatory methods from Stream base class... @@ -73,7 +82,6 @@ protected: void init(); bool fillBuffer(const U32 i_startPosition); void clearBuffer(); - static void calcBlockHead(const U32 i_position, U32 *o_blockHead); static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail); void setStatus(); diff --git a/Engine/source/core/util/hashFunction.cpp b/Engine/source/core/util/hashFunction.cpp index 69206dc30..4e29e924e 100644 --- a/Engine/source/core/util/hashFunction.cpp +++ b/Engine/source/core/util/hashFunction.cpp @@ -30,6 +30,7 @@ #include "platform/platform.h" #include "core/util/hashFunction.h" +#include "core/util/endian.h" namespace Torque { @@ -268,4 +269,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval ) return c; } +// Generate a single 64bit hash from the input string. +// +// Don't get paranoid! This has 1 in 18446744073709551616 +// chance for collision... it won't happen in this lifetime. +// +String getStringHash64(const String& in) +{ + String cacheKey = in; + cacheKey.replace("\n", " "); + U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0); + hash = convertHostToLEndian(hash); + U32 high = (U32)(hash >> 32); + U32 low = (U32)(hash & 0x00000000FFFFFFFF); + cacheKey = String::ToString("%x%x", high, low); + return cacheKey; +} + } // namespace diff --git a/Engine/source/core/util/hashFunction.h b/Engine/source/core/util/hashFunction.h index 02ecd019b..f3a57cc9a 100644 --- a/Engine/source/core/util/hashFunction.h +++ b/Engine/source/core/util/hashFunction.h @@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval); extern U64 hash64(const U8 *k, U32 length, U64 initval); +extern String getStringHash64(const String& in); + } #endif // _HASHFUNCTION_H_ diff --git a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp index 6259be330..7ab351a1f 100644 --- a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp +++ b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp @@ -681,6 +681,24 @@ bool GFXD3D11Shader::_init() return true; } +static String buildMacroHash(const D3D_SHADER_MACRO* defines) +{ + String combined; + + if (!defines) + return ""; + + for (const D3D_SHADER_MACRO* m = defines; m->Name != nullptr; ++m) + { + combined += m->Name; + combined += "="; + combined += m->Definition ? m->Definition : ""; + combined += ";"; + } + + return Torque::getStringHash64(combined); +} + bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, GFXShaderStage shaderStage, const D3D_SHADER_MACRO *defines) @@ -706,8 +724,40 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, Con::printf( "Compiling Shader: '%s'", filePath.getFullPath().c_str() ); #endif + String macroHash = buildMacroHash(defines); + + Torque::Path cachePath = filePath; + cachePath.setExtension("tso"); + cachePath.setFileName(cachePath.getFileName() + "_" + macroHash); + + if (Torque::FS::IsFile(cachePath)) + { + Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(filePath); + Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath); + + if (rawFile != NULL && cachedFile != NULL) + { + if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime()) + { + + FileStream fs; + if (fs.open(cachePath, Torque::FS::File::Read)) + { + U32 size = fs.getStreamSize(); + + D3DCreateBlob(size, &code); + fs.read(size, code->GetBufferPointer()); + + res = 1; + } + } + } + } + + bool loadedFromCache = (code != NULL); + // Is it an HLSL shader? - if(filePath.getExtension().equal("hlsl", String::NoCase)) + if(filePath.getExtension().equal("hlsl", String::NoCase) && !loadedFromCache) { // Set this so that the D3DInclude::Open will have this // information for relative paths. @@ -788,6 +838,20 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, AssertISV(SUCCEEDED(res), "Unable to compile shader!"); + // succeeded write out a cache + if (!loadedFromCache) + { + if (SUCCEEDED(res) && code) + { + // Save cache + FileStream out(FileStream::AsyncMode::Background); + if (out.open(cachePath, Torque::FS::File::Write)) + { + out.write(code->GetBufferSize(), code->GetBufferPointer()); + } + } + } + if(code != NULL) { switch (shaderStage) diff --git a/Engine/source/gfx/gl/gfxGLShader.cpp b/Engine/source/gfx/gl/gfxGLShader.cpp index a304b9bc2..65dcbec26 100644 --- a/Engine/source/gfx/gl/gfxGLShader.cpp +++ b/Engine/source/gfx/gl/gfxGLShader.cpp @@ -542,6 +542,77 @@ bool GFXGLShader::_init() macros.last().name = "TORQUE_VERTEX_SHADER"; macros.last().value = ""; + Torque::Path cachePath = mVertexFile; + cachePath.setExtension("tso"); + String macroHash; + GFXShaderMacro::stringize(macros, ¯oHash); + macroHash = Torque::getStringHash64(macroHash); + + String fileName = cachePath.getFileName(); + + if (!mVertexFile.isEmpty()) + fileName += "_" + mVertexFile.getFileName(); + + if (!mPixelFile.isEmpty()) + fileName += "_" + mPixelFile.getFileName(); + + if (!mGeometryFile.isEmpty()) + fileName += "_" + mGeometryFile.getFileName(); + + fileName += "_" + macroHash; + + cachePath.setFileName(fileName); + + if (Torque::FS::IsFile(cachePath)) + { + Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(mVertexFile); + Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath); + + if (rawFile != NULL && cachedFile != NULL) + { + if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime()) + { + + FileStream fs; + if (fs.open(cachePath, Torque::FS::File::Read)) + { + U32 size; + FrameAllocatorMarker bin; + GLenum gl_format; + fs.read(&gl_format); + fs.read(&size); + + char* bin_data = (char*)bin.alloc(size); + fs.read(size, bin_data); + + glProgramBinary( + mProgram, + gl_format, + bin_data, + size); + + + GLint linked; + glGetProgramiv(mProgram, GL_LINK_STATUS, &linked); + if (linked == GL_TRUE) + { + initConstantDescs(); + initHandles(); + + // Notify Buffers we might have changed in size. + // If this was our first init then we won't have any activeBuffers + // to worry about unnecessarily calling. + Vector::iterator biter = mActiveBuffers.begin(); + for (; biter != mActiveBuffers.end(); biter++) + ((GFXGLShaderConstBuffer*)(*biter))->onShaderReload(this); + + return true; + } + } + } + } + } + // Default to true so we're "successful" if a vertex/pixel shader wasn't specified. bool compiledVertexShader = true; bool compiledPixelShader = true; @@ -659,6 +730,27 @@ bool GFXGLShader::_init() if (linkStatus == GL_FALSE) return false; + GLint binaryLength; + glGetProgramiv(mProgram, GL_PROGRAM_BINARY_LENGTH, &binaryLength); + + if (binaryLength) + { + FrameAllocatorMarker bin; + char* bin_data = (char*)bin.alloc(binaryLength); + GLenum binaryFormat; + GLint length; + glGetProgramBinary(mProgram, binaryLength, &length, &binaryFormat, bin_data); + AssertWarn(length == binaryLength, "GFXGLShader: Binary length does not match"); + + FileStream out(FileStream::AsyncMode::Background); + if (out.open(cachePath, Torque::FS::File::Write)) + { + out.write(binaryFormat); + out.write(length); + out.write(length, bin_data); + } + } + initConstantDescs(); initHandles(); diff --git a/Engine/source/materials/shaderData.cpp b/Engine/source/materials/shaderData.cpp index 85b6d1446..be65fc4f7 100644 --- a/Engine/source/materials/shaderData.cpp +++ b/Engine/source/materials/shaderData.cpp @@ -71,6 +71,8 @@ ShaderData::ShaderData() for( int i = 0; i < NumTextures; ++i) mRTParams[i] = false; + mInstancingFormat = NULL; + mDXVertexShaderName = StringTable->EmptyString(); mDXPixelShaderName = StringTable->EmptyString(); mDXGeometryShaderName = StringTable->EmptyString(); @@ -217,6 +219,8 @@ GFXShader* ShaderData::getShader( const Vector ¯os ) String cacheKey; GFXShaderMacro::stringize( macros, &cacheKey ); + cacheKey = Torque::getStringHash64(cacheKey); + // Lookup the shader for this instance. ShaderCache::Iterator iter = mShaders.find( cacheKey ); if ( iter != mShaders.end() ) @@ -257,30 +261,32 @@ GFXShader* ShaderData::_createShader( const Vector ¯os ) { case Direct3D11: { - if (mDXVertexShaderName != String::EmptyString) + if (mDXVertexShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mDXVertexShaderName); - if (mDXPixelShaderName != String::EmptyString) + if (mDXPixelShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mDXPixelShaderName); - if (mDXGeometryShaderName != String::EmptyString) + if (mDXGeometryShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mDXGeometryShaderName); success = shader->init( pixver, macros, - samplers); + samplers, + mInstancingFormat); break; } case OpenGL: { - if(mOGLVertexShaderName != String::EmptyString) + if(mOGLVertexShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mOGLVertexShaderName); - if (mOGLPixelShaderName != String::EmptyString) + if (mOGLPixelShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mOGLPixelShaderName); - if (mOGLGeometryShaderName != String::EmptyString) + if (mOGLGeometryShaderName != StringTable->EmptyString()) shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mOGLGeometryShaderName); success = shader->init( pixver, macros, - samplers); + samplers, + mInstancingFormat); break; } @@ -321,6 +327,33 @@ GFXShader* ShaderData::_createShader( const Vector ¯os ) return shader; } +void ShaderData::setShaderStageFile(GFXShaderStage stage, String fileName) +{ + const bool isGL = GFX->getAdapterType() == GFXAdapterType::OpenGL; + switch (stage) + { + case VERTEX_SHADER: + isGL ? mOGLVertexShaderName = StringTable->insert(fileName) : mDXVertexShaderName = StringTable->insert(fileName); + break; + case PIXEL_SHADER: + isGL ? mOGLPixelShaderName = StringTable->insert(fileName) : mDXPixelShaderName = StringTable->insert(fileName); + break; + case GEOMETRY_SHADER: + isGL ? mOGLGeometryShaderName = StringTable->insert(fileName) : mDXGeometryShaderName = StringTable->insert(fileName); + break; + case DOMAIN_SHADER: + break; + case HULL_SHADER: + break; + case COMPUTE_SHADER: + break; + case ALL_STAGES: + break; + default: + break; + } +} + void ShaderData::reloadShaders() { ShaderCache::Iterator iter = mShaders.begin(); diff --git a/Engine/source/materials/shaderData.h b/Engine/source/materials/shaderData.h index 2fa63271f..77459fb9a 100644 --- a/Engine/source/materials/shaderData.h +++ b/Engine/source/materials/shaderData.h @@ -80,11 +80,6 @@ protected: /// them if the content has changed. const Vector& _getMacros(); - /// Helper for converting an array of macros - /// into a formatted string. - void _stringizeMacros(const Vector& macros, - String* outString); - /// Creates a new shader returning NULL on error. GFXShader* _createShader(const Vector& macros); @@ -98,6 +93,8 @@ protected: String mSamplerNames[NumTextures]; bool mRTParams[NumTextures]; + // the instancing format. + GFXVertexFormat* mInstancingFormat; bool _checkDefinition(GFXShader* shader); @@ -105,9 +102,11 @@ public: void setSamplerName(const String& name, int idx) { mSamplerNames[idx] = name; } String getSamplerName(int idx) const { return mSamplerNames[idx]; } + void setShaderStageFile(GFXShaderStage stage, String fileName); bool hasSamplerDef(const String& samplerName, int& pos) const; bool hasRTParamsDef(const int pos) const { return mRTParams[pos]; } + void setInstancingFormat(GFXVertexFormat* instancingFormat) { mInstancingFormat = instancingFormat; } ShaderData(); @@ -122,6 +121,7 @@ public: /// all loaded ShaderData objects in the system. static void reloadAllShaders(); + void setPixVersion(F32 pixVersion) { mPixVersion = pixVersion; } /// Returns the required pixel shader version for this shader. F32 getPixVersion() const { return mPixVersion; } diff --git a/Engine/source/shaderGen/shaderFeature.h b/Engine/source/shaderGen/shaderFeature.h index 9bed4e7ac..c19bab55b 100644 --- a/Engine/source/shaderGen/shaderFeature.h +++ b/Engine/source/shaderGen/shaderFeature.h @@ -237,6 +237,8 @@ public: /// Allows the feature to add macros to vertex shader compiles. virtual void processVertMacros( Vector ¯os, const MaterialFeatureData &fd ) {}; + virtual U32 getShaderStages() { return (GFXShaderStage::VERTEX_SHADER | GFXShaderStage::PIXEL_SHADER); } + /// Identifies what type of blending a feature uses. This is used to /// group features with the same blend operation together in a multipass /// situation. diff --git a/Engine/source/shaderGen/shaderGen.cpp b/Engine/source/shaderGen/shaderGen.cpp index be242c952..ee8ee0be5 100644 --- a/Engine/source/shaderGen/shaderGen.cpp +++ b/Engine/source/shaderGen/shaderGen.cpp @@ -30,6 +30,7 @@ #include "gfx/gfxDevice.h" #include "core/memVolume.h" #include "core/module.h" +#include "console/persistenceManager.h" #ifdef TORQUE_D3D11 #include "shaderGen/HLSL/customFeatureHLSL.h" @@ -55,6 +56,32 @@ MODULE_BEGIN( ShaderGen ) MODULE_END; + +static const U32 gStageOrder[] = +{ + GFXShaderStage::VERTEX_SHADER, + GFXShaderStage::HULL_SHADER, + GFXShaderStage::DOMAIN_SHADER, + GFXShaderStage::GEOMETRY_SHADER, + GFXShaderStage::PIXEL_SHADER, + GFXShaderStage::COMPUTE_SHADER +}; + +static const char* _getStagePostfix(GFXShaderStage stage) +{ + switch (stage) + { + case GFXShaderStage::VERTEX_SHADER: return "_V"; + case GFXShaderStage::HULL_SHADER: return "_H"; + case GFXShaderStage::DOMAIN_SHADER: return "_D"; + case GFXShaderStage::GEOMETRY_SHADER: return "_G"; + case GFXShaderStage::PIXEL_SHADER: return "_P"; + case GFXShaderStage::COMPUTE_SHADER: return "_C"; + } + + return "_U"; // Unknown +} + String ShaderGen::smCommonShaderPath("shaders/common"); ShaderGen::ShaderGen() @@ -68,6 +95,14 @@ ShaderGen::~ShaderGen() { GFXDevice::getDeviceEventSignal().remove(this, &ShaderGen::_handleGFXEvent); _uninit(); + + for (ShaderDataMap::Pair data : mProcShaderData) + { + if (data.value->isProperlyAdded() && !data.value->isDeleted()) + data.value->unregisterObject(); + } + + mProcShaderData.clear(); } void ShaderGen::registerInitDelegate(GFXAdapterType adapterType, ShaderGenInitDelegate& initDelegate) @@ -99,6 +134,7 @@ void ShaderGen::initShaderGen() return; const GFXAdapterType adapterType = GFX->getAdapterType(); + const bool isGl = adapterType == GFXAdapterType::OpenGL; if (!mInitDelegates[adapterType]) return; @@ -133,17 +169,25 @@ void ShaderGen::initShaderGen() // Delete the auto-generated conditioner include file. Torque::FS::Remove( "shadergen:/" + ConditionerFeature::ConditionerIncludeFileName ); + + Vector fileList; + String pattern = "*."; + pattern += isGl ? "glsl" : "hlsl"; + S32 numShaderFiles = Torque::FS::FindByPattern("shadergen:/", pattern, false, fileList); + for (U32 i = 0; i < numShaderFiles; i++) + { + Torque::Path filePath = fileList[i]; + mFileCache[filePath.getFileName()] = true; + } } -void ShaderGen::generateShader( const MaterialFeatureData &featureData, - char *vertFile, - char *pixFile, - F32 *pixVersion, - const GFXVertexFormat *vertexFormat, - const char* cacheName, - Vector ¯os) +void ShaderGen::generateShader(const MaterialFeatureData& featureData, + ShaderData* shaderData, + const GFXVertexFormat* vertexFormat, + const char* cacheName, + Vector& macros) { - PROFILE_SCOPE( ShaderGen_GenerateShader ); + PROFILE_SCOPE(ShaderGen_GenerateShader); mFeatureData = featureData; mVertexFormat = vertexFormat; @@ -151,65 +195,115 @@ void ShaderGen::generateShader( const MaterialFeatureData &featureData, _uninit(); _init(); - char vertShaderName[256]; - char pixShaderName[256]; + const FeatureSet& features = mFeatureData.features; + U32 stages = 0; - // Note: We use a postfix of _V/_P here so that it sorts the matching - // vert and pixel shaders together when listed alphabetically. - dSprintf( vertShaderName, sizeof(vertShaderName), "shadergen:/%s_V.%s", cacheName, mFileEnding.c_str() ); - dSprintf( pixShaderName, sizeof(pixShaderName), "shadergen:/%s_P.%s", cacheName, mFileEnding.c_str() ); - - dStrcpy( vertFile, vertShaderName, 256 ); - dStrcpy( pixFile, pixShaderName, 256 ); - - // this needs to change - need to optimize down to ps v.1.1 - *pixVersion = GFX->getPixelShaderVersion(); - - if ( !Con::getBoolVariable( "ShaderGen::GenNewShaders", true ) ) + // loop through and see which stages this featureset is expecting to make. + for (U32 i = 0; i < features.getCount(); i++) { - // If we are not regenerating the shader we will return here. - // But we must fill in the shader macros first! - _processVertFeatures( macros, true ); - _processPixFeatures( macros, true ); + const FeatureType& type = features.getAt(i); + ShaderFeature* feat = FEATUREMGR->getByType(type); + stages |= feat->getShaderStages(); - return; } - // create vertex shader - //------------------------ - FileStream* s = new FileStream(); - if(!s->open(vertShaderName, Torque::FS::File::Write )) + for (U32 s = 0; s < (sizeof(gStageOrder) / sizeof(U32)); s++) { - AssertFatal(false, "Failed to open Shader Stream" ); - return; + U32 stage = gStageOrder[s]; + + // skip unused stages + if (!(stages & stage)) + continue; + + bool macrosOnly = !Con::getBoolVariable("ShaderGen::GenNewShaders", true); + bool skipPrint = false; + GFXShaderStage curStage = (GFXShaderStage)stage; + + char fileName[256]; + const char* postfix = _getStagePostfix(curStage); + String stageName; + + if (curStage & GFXShaderStage::VERTEX_SHADER) + stageName += vertexFormat->getDescription(); + + // build our filename. + for (U32 i = 0; i < features.getCount(); i++) + { + const FeatureType& type = features.getAt(i); + if (stage & FEATUREMGR->getByType(type)->getShaderStages()) + { + stageName += type.getName(); + } + } + + stageName = Torque::getStringHash64(stageName); + stageName += postfix; + + FileCacheSet::iterator file = mFileCache.find(stageName); + if (file != mFileCache.end()) + { + // set the shaderdata file for this stage, shaderdata ptr needs to be passed in here. + dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str()); + shaderData->setShaderStageFile(curStage, fileName); + if (!(curStage & GFXShaderStage::VERTEX_SHADER)) + { + continue; + } + + skipPrint = true; + } + + mFileCache[stageName] = true; + + dSprintf(fileName, sizeof(fileName), "shadergen:/%s.%s", stageName.c_str(), mFileEnding.c_str()); + + shaderData->setShaderStageFile(curStage, fileName); + + + FileStream* stream = new FileStream(FileStream::AsyncMode::Background); + if (!skipPrint) + { + if (!stream->open(fileName, Torque::FS::File::Write)) + { + AssertFatal(false, "Failed to open Shader Stream"); + return; + } + } + + switch (curStage) + { + case VERTEX_SHADER: + mOutput = new MultiLine; + mInstancingFormat.clear(); + _processVertFeatures(macros, macrosOnly); + if (!skipPrint || macrosOnly) _printVertShader(*stream); + delete stream; + ((ShaderConnector*)mComponents[C_CONNECTOR])->reset(); + LangElement::deleteElements(); + break; + case PIXEL_SHADER: + mOutput = new MultiLine; + _processPixFeatures(macros, macrosOnly); + if (!skipPrint || macrosOnly)_printPixShader(*stream); + delete stream; + LangElement::deleteElements(); + break; + case GEOMETRY_SHADER: + break; + case DOMAIN_SHADER: + break; + case HULL_SHADER: + break; + case COMPUTE_SHADER: + break; + case ALL_STAGES: + break; + default: + break; + } + } - - mOutput = new MultiLine; - mInstancingFormat.clear(); - _processVertFeatures(macros); - _printVertShader( *s ); - delete s; - - ((ShaderConnector*)mComponents[C_CONNECTOR])->reset(); - LangElement::deleteElements(); - - // create pixel shader - //------------------------ - s = new FileStream(); - if(!s->open(pixShaderName, Torque::FS::File::Write )) - { - AssertFatal(false, "Failed to open Shader Stream" ); - delete s; - return; - } - - mOutput = new MultiLine; - _processPixFeatures(macros); - _printPixShader( *s ); - - delete s; - LangElement::deleteElements(); } void ShaderGen::_init() @@ -264,7 +358,7 @@ void ShaderGen::_processVertFeatures( Vector ¯os, bool macro else feature = FEATUREMGR->getByType( type ); - if ( feature ) + if ( feature && (feature->getShaderStages() & GFXShaderStage::VERTEX_SHADER) ) { feature->setProcessIndex( index ); @@ -312,7 +406,7 @@ void ShaderGen::_processPixFeatures( Vector ¯os, bool macros feature = FEATUREMGR->createFeature(type, args); else feature = FEATUREMGR->getByType(type); - if ( feature ) + if ( feature && (feature->getShaderStages() & GFXShaderStage::PIXEL_SHADER) ) { feature->setProcessIndex( index ); @@ -466,60 +560,50 @@ void ShaderGen::_printPixShader( Stream &stream ) mPrinter->printPixelShaderCloser(stream); } -GFXShader* ShaderGen::getShader( const MaterialFeatureData &featureData, const GFXVertexFormat *vertexFormat, const Vector *macros, const Vector &samplers ) +GFXShader* ShaderGen::getShader(const MaterialFeatureData& featureData, const GFXVertexFormat* vertexFormat, const Vector* macros, const Vector& samplers) { - PROFILE_SCOPE( ShaderGen_GetShader ); + PROFILE_SCOPE(ShaderGen_GetShader); - const FeatureSet &features = featureData.codify(); + const FeatureSet& features = featureData.codify(); // Build a description string from the features // and vertex format combination ( and macros ). String shaderDescription = vertexFormat->getDescription() + features.getDescription(); - // Generate a single 64bit hash from the description string. - // - // Don't get paranoid! This has 1 in 18446744073709551616 - // chance for collision... it won't happen in this lifetime. - // - shaderDescription.replace("\n", " "); - U64 hash = Torque::hash64( (const U8*)shaderDescription.c_str(), shaderDescription.length(), 0 ); - hash = convertHostToLEndian(hash); - U32 high = (U32)( hash >> 32 ); - U32 low = (U32)( hash & 0x00000000FFFFFFFF ); - String cacheKey = String::ToString( "%x%x", high, low ); - // return shader if exists - GFXShader *match = mProcShaders[cacheKey]; - if ( match ) - return match; - // if not, then create it - char vertFile[256]; - char pixFile[256]; - F32 pixVersion; + String cacheKey = Torque::getStringHash64(shaderDescription); Vector shaderMacros; - shaderMacros.push_back( GFXShaderMacro( "TORQUE_SHADERGEN" ) ); - if ( macros ) - shaderMacros.merge( *macros ); - generateShader( featureData, vertFile, pixFile, &pixVersion, vertexFormat, cacheKey, shaderMacros ); + shaderMacros.push_back(GFXShaderMacro("TORQUE_SHADERGEN")); + if (macros) + shaderMacros.merge(*macros); - GFXShader *shader = GFX->createShader(); - shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, vertFile); - shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, pixFile); - - if (!shader->init(pixVersion, shaderMacros, samplers, &mInstancingFormat)) + ShaderDataMap::iterator dat = mProcShaderData.find(cacheKey); + if (dat != mProcShaderData.end()) { - delete shader; - return NULL; + // should we loop vertex shader features to build mInstancingFormat before sending it down to see old hob? + return dat->value->getShader(shaderMacros); } - mProcShaders[cacheKey] = shader; + ShaderData* shaderData = new ShaderData; - return shader; + shaderData->setPixVersion(GFX->getPixelShaderVersion()); + + for (U32 samp = 0; samp < samplers.size(); samp++) + { + shaderData->setSamplerName(samplers[samp], samp); + } + + generateShader(featureData, shaderData, vertexFormat, cacheKey, shaderMacros); + + shaderData->setInstancingFormat(&mInstancingFormat); + + mProcShaderData.insert(cacheKey, shaderData); + return shaderData->getShader(shaderMacros); } void ShaderGen::flushProceduralShaders() { // The shaders are reference counted, so we // just need to clear the map. - mProcShaders.clear(); + mProcShaderData.clear(); } diff --git a/Engine/source/shaderGen/shaderGen.h b/Engine/source/shaderGen/shaderGen.h index 8ff8ef0f6..f39e624d4 100644 --- a/Engine/source/shaderGen/shaderGen.h +++ b/Engine/source/shaderGen/shaderGen.h @@ -46,6 +46,9 @@ #ifndef _MATERIALFEATUREDATA_H_ #include "materials/materialFeatureData.h" #endif +#ifndef _SHADERDATA_H_ +#include "materials/shaderData.h" +#endif // !_SHADERDATA_H_ /// Base class used by shaderGen to be API agnostic. Subclasses implement the various methods /// in an API specific way. @@ -146,9 +149,7 @@ public: /// this function. /// @param assignNum used to assign a specific number as the filename void generateShader( const MaterialFeatureData &featureData, - char *vertFile, - char *pixFile, - F32 *pixVersion, + ShaderData* shaderData, const GFXVertexFormat *vertexFormat, const char* cacheName, Vector ¯os); @@ -192,9 +193,14 @@ protected: bool mRegisteredWithGFX; Torque::FS::FileSystemRef mMemFS; - /// Map of cache string -> shaders - typedef Map ShaderMap; - ShaderMap mProcShaders; + /// + /// Map of shaderdata, string should be built up of stage files + /// + typedef HashMap> ShaderDataMap; + ShaderDataMap mProcShaderData; + + typedef HashMap FileCacheSet; // we use a hashmap because it is quicker for finding. + FileCacheSet mFileCache; ShaderGen(); diff --git a/Engine/source/ts/assimp/assimpShapeLoader.cpp b/Engine/source/ts/assimp/assimpShapeLoader.cpp index d0560345a..794eb6aa2 100644 --- a/Engine/source/ts/assimp/assimpShapeLoader.cpp +++ b/Engine/source/ts/assimp/assimpShapeLoader.cpp @@ -771,7 +771,7 @@ void AssimpShapeLoader::extractTexture(U32 index, aiTexture* pTex) { // Compressed format, write the data directly to disc texPath.setExtension(pTex->achFormatHint); FileStream *outputStream; - if ((outputStream = FileStream::createAndOpen(texPath.getFullPath(), Torque::FS::File::Write)) != NULL) + if ((outputStream = FileStream::createAndOpen(texPath.getFullPath(), Torque::FS::File::Write, FileStream::AsyncMode::Background)) != NULL) { outputStream->setPosition(0); outputStream->write(pTex->mWidth, pTex->pcData); diff --git a/Engine/source/ts/collada/colladaShapeLoader.cpp b/Engine/source/ts/collada/colladaShapeLoader.cpp index b082678bd..55747661c 100644 --- a/Engine/source/ts/collada/colladaShapeLoader.cpp +++ b/Engine/source/ts/collada/colladaShapeLoader.cpp @@ -720,7 +720,7 @@ static bool sReadCollada(const Torque::Path& path, TSShape*& res_shape) // Cache the model to a DTS file for faster loading next time. cachedPath.setExtension("cached.dts"); // Cache the model to a DTS file for faster loading next time. - FileStream dtsStream; + FileStream dtsStream(FileStream::AsyncMode::Background); if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write)) { Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str());