file stream close async

This is just firing the mfile->close off to another thread for files that are caches of data such as dts and tso for shaders.

What happens here is the closing of the filestream causes a hang as the os frees the file and writes metadata, we dont care about this as we are only ever caching data that we already have. next time torque loads it uses these cached versions not during runtime.

Engine/source/core/stream/fileStream.cpp

@@ -22,15 +22,68 @@
 
 #include "platform/platform.h"
 #include "core/stream/fileStream.h"
-
+#include "platform/threads/threadPool.h"
 
 //-----------------------------------------------------------------------------
 // FileStream methods...
 //-----------------------------------------------------------------------------
 
-//-----------------------------------------------------------------------------
+static void writeBufferToFile(
-FileStream::FileStream()
+   Torque::FS::FileRef& file,
+   U32 buffHead,
+   U32 buffTail,
+   const U8* buffer)
 {
+   if (!file || buffHead == FileStream::BUFFER_INVALID)
+      return;
+
+   // match FileStream::flush logic
+   if (buffHead != file->getPosition())
+      file->setPosition(buffHead, Torque::FS::File::Begin);
+
+   U32 blockHead;
+   FileStream::calcBlockHead(buffHead, &blockHead);
+
+   file->write((char*)buffer + (buffHead - blockHead), buffTail - buffHead + 1);
+}
+
+struct FileCloseWorkItem : public ThreadPool::WorkItem
+{
+   Torque::FS::FileRef mFile;
+
+public:
+   FileCloseWorkItem(Torque::FS::FileRef file)
+      : mFile(file)
+   {
+   }
+
+protected:
+   void execute() override
+   {
+      if (!mFile)
+         return;
+
+      // When platforms free a file they
+      // remove the handle which causes the stall
+      // as they write metadata.
+      mFile->close();
+   }
+};
+
+void FileStream::dispatchAsyncClose()
+{
+   if (!mFile)
+      return;
+
+   FileCloseWorkItem* job = new FileCloseWorkItem(mFile);
+
+   ThreadPool::GLOBAL().queueWorkItem(job);
+}
+
+//-----------------------------------------------------------------------------
+FileStream::FileStream(AsyncMode flushMode)
+{
+   mAsyncMode = flushMode;
    dMemset(mBuffer, 0, sizeof(mBuffer));
    // initialize the file stream
    init();
@@ -190,10 +243,13 @@ void FileStream::close()
       if (mDirty)
          flush();
 
-      // and close the file
+      if (mAsyncMode == Background)
-      mFile->close();
+         dispatchAsyncClose();
-
+      else
-      AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
+      {
+         mFile->close();
+         AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
+      }
 
       mFile = NULL;
    }

Engine/source/core/stream/fileStream.h

@@ -39,9 +39,17 @@ public:
       BUFFER_INVALID = 0xffffffff   // file offsets must all be less than this
    };
 
+   enum AsyncMode
+   {
+      Blocking,      // current behavior
+      Background     // write-behind
+   };
+
    typedef char Ch;    //!< Character type. Only support char.
 public:
-   FileStream();                       // default constructor
+   AsyncMode mAsyncMode;
+   void dispatchAsyncClose();
+   FileStream(AsyncMode flushMode = Blocking);                       // default constructor
    virtual ~FileStream();              // destructor
 
    // This function will allocate a new FileStream and open it.
@@ -64,7 +72,7 @@ public:
    //rjson compatibility
    bool Flush() { return flush(); }
    FileStream* clone() const override;
-
+   static void calcBlockHead(const U32 i_position, U32* o_blockHead);
 protected:
    // more mandatory methods from Stream base class...
    bool _read(const U32 i_numBytes, void *o_pBuffer) override;
@@ -73,7 +81,7 @@ protected:
    void init();
    bool fillBuffer(const U32 i_startPosition);
    void clearBuffer();
-   static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
+   
    static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail);
    void setStatus();
 

Engine/source/gfx/D3D11/gfxD3D11Shader.cpp

@@ -844,7 +844,7 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
       if (SUCCEEDED(res) && code)
       { 
          // Save cache
-         FileStream out;
+         FileStream out(FileStream::AsyncMode::Background);
          if (out.open(cachePath, Torque::FS::File::Write))
          {
             out.write(code->GetBufferSize(), code->GetBufferPointer());

Engine/source/ts/assimp/assimpShapeLoader.cpp

@@ -1092,7 +1092,7 @@ TSShape* assimpLoadShape(const Torque::Path &path)
          // Cache the model to a DTS file for faster loading next time.
          cachedPath.setExtension("cached.dts");
          // Cache the model to a DTS file for faster loading next time.
-         FileStream dtsStream;
+         FileStream dtsStream(FileStream::AsyncMode::Background);
          if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write))
          {
             Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str());

Engine/source/ts/collada/colladaShapeLoader.cpp

@@ -803,7 +803,7 @@ TSShape* loadColladaShape(const Torque::Path &path)
          {
             // Cache the Collada model to a DTS file for faster loading next time.
             cachedPath.setExtension("cached.dts");
-            FileStream dtsStream;
+            FileStream dtsStream(FileStream::AsyncMode::Background);
             if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write))
             {
                Torque::FS::FileSystemRef ref = Torque::FS::GetFileSystem(daePath);