From 1fd1b8e8571e7d63913baeb6abec754b34cf308f Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Thu, 19 Feb 2026 14:30:29 +0000 Subject: [PATCH] file stream close async This is just firing the mfile->close off to another thread for files that are caches of data such as dts and tso for shaders. What happens here is the closing of the filestream causes a hang as the os frees the file and writes metadata, we dont care about this as we are only ever caching data that we already have. next time torque loads it uses these cached versions not during runtime. --- Engine/source/core/stream/fileStream.cpp | 70 +++++++++++++++++-- Engine/source/core/stream/fileStream.h | 14 +++- Engine/source/gfx/D3D11/gfxD3D11Shader.cpp | 4 +- Engine/source/ts/assimp/assimpShapeLoader.cpp | 2 +- .../source/ts/collada/colladaShapeLoader.cpp | 2 +- 5 files changed, 78 insertions(+), 14 deletions(-) diff --git a/Engine/source/core/stream/fileStream.cpp b/Engine/source/core/stream/fileStream.cpp index 29ce3a933..a5c7e8c13 100644 --- a/Engine/source/core/stream/fileStream.cpp +++ b/Engine/source/core/stream/fileStream.cpp @@ -22,15 +22,68 @@ #include "platform/platform.h" #include "core/stream/fileStream.h" - +#include "platform/threads/threadPool.h" //----------------------------------------------------------------------------- // FileStream methods... //----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -FileStream::FileStream() +static void writeBufferToFile( + Torque::FS::FileRef& file, + U32 buffHead, + U32 buffTail, + const U8* buffer) { + if (!file || buffHead == FileStream::BUFFER_INVALID) + return; + + // match FileStream::flush logic + if (buffHead != file->getPosition()) + file->setPosition(buffHead, Torque::FS::File::Begin); + + U32 blockHead; + FileStream::calcBlockHead(buffHead, &blockHead); + + file->write((char*)buffer + (buffHead - blockHead), buffTail - buffHead + 1); +} + +struct FileCloseWorkItem : public ThreadPool::WorkItem +{ + Torque::FS::FileRef mFile; + +public: + FileCloseWorkItem(Torque::FS::FileRef file) + : mFile(file) + { + } + +protected: + void execute() override + { + if (!mFile) + return; + + // When platforms free a file they + // remove the handle which causes the stall + // as they write metadata. + mFile->close(); + } +}; + +void FileStream::dispatchAsyncClose() +{ + if (!mFile) + return; + + FileCloseWorkItem* job = new FileCloseWorkItem(mFile); + + ThreadPool::GLOBAL().queueWorkItem(job); +} + +//----------------------------------------------------------------------------- +FileStream::FileStream(AsyncMode flushMode) +{ + mAsyncMode = flushMode; dMemset(mBuffer, 0, sizeof(mBuffer)); // initialize the file stream init(); @@ -190,10 +243,13 @@ void FileStream::close() if (mDirty) flush(); - // and close the file - mFile->close(); - - AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed"); + if (mAsyncMode == Background) + dispatchAsyncClose(); + else + { + mFile->close(); + AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed"); + } mFile = NULL; } diff --git a/Engine/source/core/stream/fileStream.h b/Engine/source/core/stream/fileStream.h index 9ae20798a..c4c5ecd17 100644 --- a/Engine/source/core/stream/fileStream.h +++ b/Engine/source/core/stream/fileStream.h @@ -39,9 +39,17 @@ public: BUFFER_INVALID = 0xffffffff // file offsets must all be less than this }; + enum AsyncMode + { + Blocking, // current behavior + Background // write-behind + }; + typedef char Ch; //!< Character type. Only support char. public: - FileStream(); // default constructor + AsyncMode mAsyncMode; + void dispatchAsyncClose(); + FileStream(AsyncMode flushMode = Blocking); // default constructor virtual ~FileStream(); // destructor // This function will allocate a new FileStream and open it. @@ -64,7 +72,7 @@ public: //rjson compatibility bool Flush() { return flush(); } FileStream* clone() const override; - + static void calcBlockHead(const U32 i_position, U32* o_blockHead); protected: // more mandatory methods from Stream base class... bool _read(const U32 i_numBytes, void *o_pBuffer) override; @@ -73,7 +81,7 @@ protected: void init(); bool fillBuffer(const U32 i_startPosition); void clearBuffer(); - static void calcBlockHead(const U32 i_position, U32 *o_blockHead); + static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail); void setStatus(); diff --git a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp index 4a919cffd..b84f64907 100644 --- a/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp +++ b/Engine/source/gfx/D3D11/gfxD3D11Shader.cpp @@ -842,9 +842,9 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath, if (!loadedFromCache) { if (SUCCEEDED(res) && code) - { + { // Save cache - FileStream out; + FileStream out(FileStream::AsyncMode::Background); if (out.open(cachePath, Torque::FS::File::Write)) { out.write(code->GetBufferSize(), code->GetBufferPointer()); diff --git a/Engine/source/ts/assimp/assimpShapeLoader.cpp b/Engine/source/ts/assimp/assimpShapeLoader.cpp index 4bcc1df91..a3b78326d 100644 --- a/Engine/source/ts/assimp/assimpShapeLoader.cpp +++ b/Engine/source/ts/assimp/assimpShapeLoader.cpp @@ -1092,7 +1092,7 @@ TSShape* assimpLoadShape(const Torque::Path &path) // Cache the model to a DTS file for faster loading next time. cachedPath.setExtension("cached.dts"); // Cache the model to a DTS file for faster loading next time. - FileStream dtsStream; + FileStream dtsStream(FileStream::AsyncMode::Background); if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write)) { Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str()); diff --git a/Engine/source/ts/collada/colladaShapeLoader.cpp b/Engine/source/ts/collada/colladaShapeLoader.cpp index 66a859934..17a3557a2 100644 --- a/Engine/source/ts/collada/colladaShapeLoader.cpp +++ b/Engine/source/ts/collada/colladaShapeLoader.cpp @@ -803,7 +803,7 @@ TSShape* loadColladaShape(const Torque::Path &path) { // Cache the Collada model to a DTS file for faster loading next time. cachedPath.setExtension("cached.dts"); - FileStream dtsStream; + FileStream dtsStream(FileStream::AsyncMode::Background); if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write)) { Torque::FS::FileSystemRef ref = Torque::FS::GetFileSystem(daePath);