Shader Gen to produce ShaderData

Shader gen now produces a shaderdata class - this should reduce full recompilation of shaders when a macro switch happens.

FileStream can now also be setup to be async write, so it will write out the file on a separate thread hopefully freeing up the main thread to continue working.

Engine/source/core/stream/fileStream.cpp

@@ -22,23 +22,75 @@
 
 #include "platform/platform.h"
 #include "core/stream/fileStream.h"
-
+#include "platform/threads/threadPool.h"
 
 //-----------------------------------------------------------------------------
 // FileStream methods...
 //-----------------------------------------------------------------------------
+static void writeBufferToFile(
+   Torque::FS::FileRef& file,
+   U32 buffHead,
+   U32 buffTail,
+   const U8* buffer)
+{
+   if (!file || buffHead == FileStream::BUFFER_INVALID)
+      return;
+
+   // match FileStream::flush logic
+   if (buffHead != file->getPosition())
+      file->setPosition(buffHead, Torque::FS::File::Begin);
+
+   U32 blockHead;
+   FileStream::calcBlockHead(buffHead, &blockHead);
+
+   file->write((char*)buffer + (buffHead - blockHead), buffTail - buffHead + 1);
+}
+
+struct FileCloseWorkItem : public ThreadPool::WorkItem
+{
+   Torque::FS::FileRef mFile;
+
+public:
+   FileCloseWorkItem(Torque::FS::FileRef file)
+      : mFile(file)
+   {
+   }
+
+protected:
+   void execute() override
+   {
+      if (!mFile)
+         return;
+
+      // When platforms free a file they
+      // remove the handle which causes the stall
+      // as they write metadata.
+      mFile->close();
+   }
+};
+
+void FileStream::dispatchAsyncClose()
+{
+   if (!mFile)
+      return;
+
+   FileCloseWorkItem* job = new FileCloseWorkItem(mFile);
+
+   ThreadPool::GLOBAL().queueWorkItem(job);
+}
 
 //-----------------------------------------------------------------------------
-FileStream::FileStream()
+FileStream::FileStream(AsyncMode flushMode)
 {
+   mAsyncMode = flushMode;
    dMemset(mBuffer, 0, sizeof(mBuffer));
    // initialize the file stream
    init();
 }
 
-FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode)
+FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode)
 {
-   FileStream  *newStream = new FileStream;
+   FileStream  *newStream = new FileStream(flushMode);
 
    bool success = newStream->open( inFileName, inMode );
 
@@ -193,7 +245,13 @@ void FileStream::close()
       // and close the file
       mFile->close();
 
-      AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
+      if (mAsyncMode == Background)
+         dispatchAsyncClose();
+      else
+      {
+         mFile->close();
+         AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
+      }
 
       mFile = NULL;
    }

Engine/source/core/stream/fileStream.h

@@ -39,15 +39,23 @@ public:
       BUFFER_INVALID = 0xffffffff   // file offsets must all be less than this
    };
 
+   enum AsyncMode
+   {
+      Blocking,      // current behavior
+      Background     // write-behind
+   };
+
    typedef char Ch;    //!< Character type. Only support char.
 public:
-   FileStream();                       // default constructor
+   AsyncMode mAsyncMode;
+   void dispatchAsyncClose();
+   FileStream(AsyncMode flushMode = Blocking);// default constructor
    virtual ~FileStream();              // destructor
 
    // This function will allocate a new FileStream and open it.
    //  If it fails to allocate or fails to open, it will return NULL.
    //  The caller is responsible for deleting the instance.
-   static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode);
+   static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode, AsyncMode flushMode = Blocking);
 
    // mandatory methods from Stream base class...
    bool hasCapability(const Capability i_cap) const override;
@@ -64,6 +72,7 @@ public:
    //rjson compatibility
    bool Flush() { return flush(); }
    FileStream* clone() const override;
+   static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
 
 protected:
    // more mandatory methods from Stream base class...
@@ -73,7 +82,6 @@ protected:
    void init();
    bool fillBuffer(const U32 i_startPosition);
    void clearBuffer();
-   static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
    static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail);
    void setStatus();
 

Engine/source/core/util/hashFunction.cpp

@@ -30,6 +30,7 @@
 #include "platform/platform.h"
 
 #include "core/util/hashFunction.h"
+#include "core/util/endian.h"
 
 namespace Torque
 {
@@ -268,4 +269,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval )
   return c;
 }
 
+// Generate a single 64bit hash from the input string.
+//
+// Don't get paranoid!  This has 1 in 18446744073709551616
+// chance for collision... it won't happen in this lifetime.
+//
+String getStringHash64(const String& in)
+{
+   String cacheKey = in;
+   cacheKey.replace("\n", " ");
+   U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0);
+   hash = convertHostToLEndian(hash);
+   U32 high = (U32)(hash >> 32);
+   U32 low = (U32)(hash & 0x00000000FFFFFFFF);
+   cacheKey = String::ToString("%x%x", high, low);
+   return cacheKey;
+}
+
 } // namespace

Engine/source/core/util/hashFunction.h

@@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval);
 
 extern U64 hash64(const U8 *k, U32 length, U64 initval);
 
+extern String getStringHash64(const String& in);
+
 }
 
 #endif // _HASHFUNCTION_H_