db-cache -- implementation of datablock caching system.

Engine/source/T3D/gameBase/gameConnection.cpp

@@ -20,6 +20,11 @@
 // IN THE SOFTWARE.
 //-----------------------------------------------------------------------------
 
+//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
+// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames
+// Copyright (C) 2015 Faust Logic, Inc.
+//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
+
 #include "platform/platform.h"
 #include "T3D/gameBase/gameConnection.h"
 
@@ -52,6 +57,10 @@
    #include "T3D/gameBase/std/stdMoveList.h"
 #endif
 
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+#include "core/stream/fileStream.h"
+#endif 
+
 //----------------------------------------------------------------------------
 #define MAX_MOVE_PACKET_SENDS 4
 
@@ -173,9 +182,20 @@ IMPLEMENT_CALLBACK( GameConnection, onFlash, void, (bool state), (state),
    "either is on or both are off.  Typically this is used to enable the flash postFx.\n\n"
    "@param state Set to true if either the damage flash or white out conditions are active.\n\n");
 
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+StringTableEntry GameConnection::server_cache_filename = "";
+StringTableEntry GameConnection::client_cache_filename = "";
+bool GameConnection::server_cache_on = false;
+bool GameConnection::client_cache_on = false;
+#endif 
 //----------------------------------------------------------------------------
 GameConnection::GameConnection()
 {
+   
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+   client_db_stream = new InfiniteBitStream;
+   server_cache_CRC = 0xffffffff;
+#endif 
    mLagging = false;
    mControlObject = NULL;
    mCameraObject = NULL;
@@ -246,6 +266,10 @@ GameConnection::~GameConnection()
       dFree(mConnectArgv[i]);
    dFree(mJoinPassword);
    delete mMoveList;
+
+#ifdef AFX_CAP_DATABLOCK_CACHE
+   delete client_db_stream;
+#endif 
 }
 
 //----------------------------------------------------------------------------
@@ -1608,6 +1632,14 @@ void GameConnection::preloadNextDataBlock(bool hadNewFiles)
          sendConnectionMessage(DataBlocksDownloadDone, mDataBlockSequence);
 
 //          gResourceManager->setMissingFileLogging(false);
+
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+         // This should be the last of the datablocks. An argument of false
+         // indicates that this is a client save.
+         if (clientCacheEnabled())
+            saveDatablockCache(false);
+#endif 
+
          return;
       }
       mFilesWereDownloaded = hadNewFiles;
@@ -1771,7 +1803,11 @@ DefineEngineMethod( GameConnection, transmitDataBlocks, void, (S32 sequence),,
     const U32 iCount = pGroup->size();
 
     // If this is the local client...
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+    if (GameConnection::getLocalClientConnection() == object && !GameConnection::serverCacheEnabled())
+#else
     if (GameConnection::getLocalClientConnection() == object)
+#endif 
     {
         // Set up a pointer to the datablock.
         SimDataBlock* pDataBlock = 0;
@@ -2166,6 +2202,13 @@ void GameConnection::consoleInit()
       "@ingroup Networking\n");
 
    // Con::addVariable("specialFog", TypeBool, &SceneGraph::useSpecial);
+
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+   Con::addVariable("$Pref::Server::DatablockCacheFilename",  TypeString,   &server_cache_filename);
+   Con::addVariable("$pref::Client::DatablockCacheFilename",  TypeString,   &client_cache_filename);
+   Con::addVariable("$Pref::Server::EnableDatablockCache",    TypeBool,     &server_cache_on);
+   Con::addVariable("$pref::Client::EnableDatablockCache",    TypeBool,     &client_cache_on);
+#endif 
 }
 
 DefineEngineMethod( GameConnection, startRecording, void, (const char* fileName),,
@@ -2360,4 +2403,393 @@ DefineEngineMethod( GameConnection, getVisibleGhostDistance, F32, (),,
    )
 {
    return object->getVisibleGhostDistance();
-}
+}
+
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+
+void GameConnection::tempDisableStringBuffering(BitStream* bs) const 
+{ 
+   bs->setStringBuffer(0); 
+}
+
+void GameConnection::restoreStringBuffering(BitStream* bs) const 
+{ 
+   bs->clearStringBuffer();
+}              
+
+// rewind to stream postion and then move raw bytes into client_db_stream
+// for caching purposes.
+void GameConnection::repackClientDatablock(BitStream* bstream, S32 start_pos)
+{
+   static U8 bit_buffer[Net::MaxPacketDataSize];
+
+   if (!clientCacheEnabled() || !client_db_stream)
+      return;
+
+   S32 cur_pos = bstream->getCurPos();
+   S32 n_bits = cur_pos - start_pos;
+   if (n_bits <= 0)
+      return;
+
+   bstream->setCurPos(start_pos);
+   bstream->readBits(n_bits, bit_buffer);
+   bstream->setCurPos(cur_pos);
+
+   //S32 start_pos2 = client_db_stream->getCurPos();
+   client_db_stream->writeBits(n_bits, bit_buffer);
+}
+
+#define CLIENT_CACHE_VERSION_CODE 47241113
+
+void GameConnection::saveDatablockCache(bool on_server)
+{
+   InfiniteBitStream bit_stream;
+   BitStream* bstream = 0;
+
+   if (on_server)
+   {
+      SimDataBlockGroup *g = Sim::getDataBlockGroup();
+
+      // find the first one we haven't sent:
+      U32 i, groupCount = g->size();
+      S32 key = this->getDataBlockModifiedKey();
+      for (i = 0; i < groupCount; i++)
+         if (((SimDataBlock*)(*g)[i])->getModifiedKey() > key)
+            break;
+
+      // nothing to save
+      if (i == groupCount) 
+         return;
+
+      bstream = &bit_stream;
+
+      for (;i < groupCount; i++) 
+      {
+         SimDataBlock* obj = (SimDataBlock*)(*g)[i];
+         GameConnection* gc = this;
+         NetConnection* conn = this;
+         SimObjectId id = obj->getId();
+
+         if (bstream->writeFlag(gc->getDataBlockModifiedKey() < obj->getModifiedKey()))        // A - flag
+         {
+            if (obj->getModifiedKey() > gc->getMaxDataBlockModifiedKey())
+               gc->setMaxDataBlockModifiedKey(obj->getModifiedKey());
+
+            bstream->writeInt(id - DataBlockObjectIdFirst,DataBlockObjectIdBitSize);            // B - int
+
+            S32 classId = obj->getClassId(conn->getNetClassGroup());
+            bstream->writeClassId(classId, NetClassTypeDataBlock, conn->getNetClassGroup());    // C - id
+            bstream->writeInt(i, DataBlockObjectIdBitSize);                                     // D - int
+            bstream->writeInt(groupCount, DataBlockObjectIdBitSize + 1);                        // E - int
+            obj->packData(bstream);
+         }
+      }
+   }
+   else
+   {
+      bstream = client_db_stream;
+   }
+
+   if (bstream->getPosition() <= 0)
+      return;
+
+   // zero out any leftover bits short of an even byte count
+   U32 n_leftover_bits = (bstream->getPosition()*8) - bstream->getCurPos();
+   if (n_leftover_bits >= 0 && n_leftover_bits <= 8)
+   {
+      // note - an unusual problem regarding setCurPos() results when there 
+      // are no leftover bytes. Adding a buffer byte in this case avoids the problem.
+      if (n_leftover_bits == 0)
+         n_leftover_bits = 8;
+      U8 bzero = 0;
+      bstream->writeBits(n_leftover_bits, &bzero);
+   }
+
+   // this is where we actually save the file
+   const char* filename = (on_server) ? server_cache_filename : client_cache_filename;
+   if (filename && filename[0] != '\0')
+   {
+      FileStream* f_stream;
+      if((f_stream = FileStream::createAndOpen(filename, Torque::FS::File::Write )) == NULL)
+      {
+         Con::printf("Failed to open file '%s'.", filename);
+         return;
+      }
+
+      U32 save_sz = bstream->getPosition();
+
+      if (!on_server)
+      {
+         f_stream->write((U32)CLIENT_CACHE_VERSION_CODE);
+         f_stream->write(save_sz);
+         f_stream->write(server_cache_CRC);
+         f_stream->write((U32)CLIENT_CACHE_VERSION_CODE);
+      }
+
+      f_stream->write(save_sz, bstream->getBuffer());
+
+      // zero out any leftover bytes short of a 4-byte multiple
+      while ((save_sz % 4) != 0)
+      {
+         f_stream->write((U8)0);
+         save_sz++;
+      }
+
+      delete f_stream;
+   }
+
+   if (!on_server)
+      client_db_stream->clear();
+}
+
+static bool afx_saved_db_cache = false;
+static U32 afx_saved_db_cache_CRC = 0xffffffff;
+
+void GameConnection::resetDatablockCache()
+{
+   afx_saved_db_cache = false;
+   afx_saved_db_cache_CRC = 0xffffffff;
+}
+
+ConsoleFunction(resetDatablockCache, void, 1, 1, "resetDatablockCache()")
+{
+   GameConnection::resetDatablockCache();
+}
+
+ConsoleFunction(isDatablockCacheSaved, bool, 1, 1, "resetDatablockCache()")
+{
+   return afx_saved_db_cache;
+}
+
+ConsoleFunction(getDatablockCacheCRC, S32, 1, 1, "getDatablockCacheCRC()")
+{
+   return (S32)afx_saved_db_cache_CRC;
+}
+
+ConsoleFunction(extractDatablockCacheCRC, S32, 2, 2, "extractDatablockCacheCRC(filename)")
+{
+   FileStream f_stream;
+   const char* fileName = argv[1];
+   if(!f_stream.open(fileName, Torque::FS::File::Read))
+   {
+      Con::errorf("Failed to open file '%s'.", fileName);
+      return -1;
+   }
+
+   U32 stream_sz = f_stream.getStreamSize();
+   if (stream_sz < 4*32)
+   {
+      Con::errorf("File '%s' is not a valid datablock cache.", fileName);
+      f_stream.close();
+      return -1;
+   }
+
+   U32 pre_code; f_stream.read(&pre_code);
+   U32 save_sz; f_stream.read(&save_sz);
+   U32 crc_code; f_stream.read(&crc_code);
+   U32 post_code; f_stream.read(&post_code);
+
+   f_stream.close();
+
+   if (pre_code != post_code)
+   {
+      Con::errorf("File '%s' is not a valid datablock cache.", fileName);
+      return -1;
+   }
+
+   if (pre_code != (U32)CLIENT_CACHE_VERSION_CODE)
+   {
+      Con::errorf("Version of datablock cache file '%s' does not match version of running software.", fileName);
+      return -1;
+   }
+
+   return (S32)crc_code;
+}
+
+ConsoleFunction(setDatablockCacheCRC, void, 2, 2, "setDatablockCacheCRC(crc)")
+{
+   GameConnection *conn = GameConnection::getConnectionToServer();
+   if(!conn)
+      return;
+
+   U32 crc_u = (U32)dAtoi(argv[1]);
+   conn->setServerCacheCRC(crc_u);
+}
+
+ConsoleMethod( GameConnection, saveDatablockCache, void, 2, 2, "saveDatablockCache()")
+{
+   if (GameConnection::serverCacheEnabled() && !afx_saved_db_cache)
+   {
+      // Save the datablocks to a cache file. An argument
+      // of true indicates that this is a server save.
+      object->saveDatablockCache(true);
+      afx_saved_db_cache = true;
+      afx_saved_db_cache_CRC = 0xffffffff;
+
+      static char filename_buffer[1024];
+      String filename(Torque::Path::CleanSeparators(object->serverCacheFilename()));
+      Con::expandScriptFilename(filename_buffer, sizeof(filename_buffer), filename.c_str());
+      Torque::Path givenPath(Torque::Path::CompressPath(filename_buffer));
+      Torque::FS::FileNodeRef fileRef = Torque::FS::GetFileNode(givenPath);
+      if ( fileRef == NULL )
+         Con::errorf("saveDatablockCache() failed to get CRC for file '%s'.", filename.c_str());
+      else
+         afx_saved_db_cache_CRC = (S32)fileRef->getChecksum();
+   }
+}
+
+ConsoleMethod( GameConnection, loadDatablockCache, void, 2, 2, "loadDatablockCache()")
+{
+   if (GameConnection::clientCacheEnabled())
+   {
+      object->loadDatablockCache();
+   }
+}
+
+ConsoleMethod( GameConnection, loadDatablockCache_Begin, bool, 2, 2, "loadDatablockCache_Begin()")
+{
+   if (GameConnection::clientCacheEnabled())
+   {
+      return object->loadDatablockCache_Begin();
+   }
+
+   return false;
+}
+
+ConsoleMethod( GameConnection, loadDatablockCache_Continue, bool, 2, 2, "loadDatablockCache_Continue()")
+{
+   if (GameConnection::clientCacheEnabled())
+   {
+      return object->loadDatablockCache_Continue();
+   }
+
+   return false;
+}
+
+static char*        afx_db_load_buf = 0;
+static U32          afx_db_load_buf_sz = 0;
+static BitStream*   afx_db_load_bstream = 0;
+
+void GameConnection::loadDatablockCache()
+{
+   if (!loadDatablockCache_Begin())
+      return;
+
+   while (loadDatablockCache_Continue())
+      ;
+}
+
+bool GameConnection::loadDatablockCache_Begin()
+{
+   if (!client_cache_filename || client_cache_filename[0] == '\0')
+   {
+      Con::errorf("No filename was specified for the client datablock cache.");
+      return false;
+   }
+
+   // open cache file
+   FileStream f_stream;
+   if(!f_stream.open(client_cache_filename, Torque::FS::File::Read))
+   {
+      Con::errorf("Failed to open file '%s'.", client_cache_filename);
+      return false;
+   }
+
+   // get file size
+   U32 stream_sz = f_stream.getStreamSize();
+   if (stream_sz <= 4*4)
+   {
+      Con::errorf("File '%s' is too small to be a valid datablock cache.", client_cache_filename);
+      f_stream.close();
+      return false;
+   }
+
+   // load header data
+   U32 pre_code; f_stream.read(&pre_code);
+   U32 save_sz; f_stream.read(&save_sz);
+   U32 crc_code; f_stream.read(&crc_code);
+   U32 post_code; f_stream.read(&post_code);
+
+   // validate header info 
+   if (pre_code != post_code)
+   {
+      Con::errorf("File '%s' is not a valid datablock cache.", client_cache_filename);
+      f_stream.close();
+      return false;
+   }
+   if (pre_code != (U32)CLIENT_CACHE_VERSION_CODE)
+   {
+      Con::errorf("Version of datablock cache file '%s' does not match version of running software.", client_cache_filename);
+      f_stream.close();
+      return false;
+   }
+
+   // allocated the in-memory buffer
+   afx_db_load_buf_sz = stream_sz - (4*4);
+   afx_db_load_buf = new char[afx_db_load_buf_sz];
+
+   // load data from file into memory
+   if (!f_stream.read(stream_sz, afx_db_load_buf))
+   {
+      Con::errorf("Failed to read data from file '%s'.", client_cache_filename);
+      f_stream.close();
+      delete [] afx_db_load_buf;
+      afx_db_load_buf = 0;
+      afx_db_load_buf_sz = 0;
+      return false;
+   }
+
+   // close file
+   f_stream.close();
+
+   // At this point we have the whole cache in memory
+
+   // create a bitstream from the in-memory buffer
+   afx_db_load_bstream = new BitStream(afx_db_load_buf, afx_db_load_buf_sz);
+
+   return true;
+}
+
+bool GameConnection::loadDatablockCache_Continue()
+{
+   if (!afx_db_load_bstream)
+      return false;
+
+   // prevent repacking of datablocks during load
+   BitStream* save_client_db_stream = client_db_stream;
+   client_db_stream = 0;
+
+   bool all_finished = false;
+
+   // loop through at most 16 datablocks
+   BitStream *bstream = afx_db_load_bstream;
+   for (S32 i = 0; i < 16; i++)
+   {
+      S32 save_pos = bstream->getCurPos();
+      if (!bstream->readFlag())
+      {
+         all_finished = true;
+         break;
+      }
+      bstream->setCurPos(save_pos);
+      SimDataBlockEvent evt;
+      evt.unpack(this, bstream);
+      evt.process(this);
+   }
+
+   client_db_stream = save_client_db_stream;
+
+   if (all_finished)
+   {
+      delete afx_db_load_bstream;
+      afx_db_load_bstream = 0;
+      delete [] afx_db_load_buf;
+      afx_db_load_buf = 0;
+      afx_db_load_buf_sz = 0;
+      return false;
+   }
+
+   return true;
+}
+
+#endif 

Engine/source/T3D/gameBase/gameConnection.h

@@ -20,6 +20,11 @@
 // IN THE SOFTWARE.
 //-----------------------------------------------------------------------------
 
+//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
+// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames
+// Copyright (C) 2015 Faust Logic, Inc.
+//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
+
 #ifndef _GAMECONNECTION_H_
 #define _GAMECONNECTION_H_
 
@@ -55,6 +60,14 @@ class MoveList;
 struct Move;
 struct AuthInfo;
 
+// To disable datablock caching, remove or comment out the AFX_CAP_DATABLOCK_CACHE define below.
+// Also, at a minimum, the following script preferences should be set to false:
+//   $pref::Client::EnableDatablockCache = false; (in arcane.fx/client/defaults.cs)
+//   $Pref::Server::EnableDatablockCache = false; (in arcane.fx/server/defaults.cs)
+// Alternatively, all script code marked with "DATABLOCK CACHE CODE" can be removed or
+// commented out.
+//
+#define AFX_CAP_DATABLOCK_CACHE
 const F32 MinCameraFov              = 1.f;      ///< min camera FOV
 const F32 MaxCameraFov              = 179.f;    ///< max camera FOV
 
@@ -372,6 +385,31 @@ protected:
    DECLARE_CALLBACK( void, setLagIcon, (bool state) );
    DECLARE_CALLBACK( void, onDataBlocksDone, (U32 sequence) );
    DECLARE_CALLBACK( void, onFlash, (bool state) );
+   
+#ifdef AFX_CAP_DATABLOCK_CACHE
+private:
+   static StringTableEntry  server_cache_filename;
+   static StringTableEntry  client_cache_filename;
+   static bool   server_cache_on;
+   static bool   client_cache_on;
+   BitStream*    client_db_stream;
+   U32           server_cache_CRC;
+public:
+   void          repackClientDatablock(BitStream*, S32 start_pos);
+   void          saveDatablockCache(bool on_server);
+   void          loadDatablockCache();
+   bool          loadDatablockCache_Begin();
+   bool          loadDatablockCache_Continue();
+   void          tempDisableStringBuffering(BitStream* bs) const;
+   void          restoreStringBuffering(BitStream* bs) const;
+   void          setServerCacheCRC(U32 crc) { server_cache_CRC = crc; }
+
+   static void   resetDatablockCache();
+   static bool   serverCacheEnabled() { return server_cache_on; }
+   static bool   clientCacheEnabled() { return client_cache_on; }
+   static const char* serverCacheFilename() { return server_cache_filename; }
+   static const char* clientCacheFilename() { return client_cache_filename; }
+#endif
 };
 
 #endif

Engine/source/T3D/gameBase/gameConnectionEvents.cpp

@@ -20,6 +20,11 @@
 // IN THE SOFTWARE.
 //-----------------------------------------------------------------------------
 
+//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
+// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames
+// Copyright (C) 2015 Faust Logic, Inc.
+//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
+
 #include "platform/platform.h"
 #include "core/dnet.h"
 #include "core/stream/bitStream.h"
@@ -136,6 +141,9 @@ void SimDataBlockEvent::notifyDelivered(NetConnection *conn, bool )
 
 void SimDataBlockEvent::pack(NetConnection *conn, BitStream *bstream)
 {
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+   ((GameConnection *)conn)->tempDisableStringBuffering(bstream);
+#endif 
    SimDataBlock* obj;
    Sim::findObject(id,obj);
    GameConnection *gc = (GameConnection *) conn;
@@ -157,10 +165,18 @@ void SimDataBlockEvent::pack(NetConnection *conn, BitStream *bstream)
       bstream->writeInt(classId ^ DebugChecksum, 32);
 #endif
    }
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+   ((GameConnection *)conn)->restoreStringBuffering(bstream);
+#endif 
 }
 
 void SimDataBlockEvent::unpack(NetConnection *cptr, BitStream *bstream)
 {
+#ifdef AFX_CAP_DATABLOCK_CACHE 
+   // stash the stream position prior to unpacking
+   S32 start_pos = bstream->getCurPos();
+   ((GameConnection *)cptr)->tempDisableStringBuffering(bstream);
+#endif
    if(bstream->readFlag())
    {
       mProcess = true;
@@ -215,6 +231,11 @@ void SimDataBlockEvent::unpack(NetConnection *cptr, BitStream *bstream)
 #endif
 
    }
+#ifdef AFX_CAP_DATABLOCK_CACHE
+   // rewind to stream position and then process raw bytes for caching
+   ((GameConnection *)cptr)->repackClientDatablock(bstream, start_pos);
+   ((GameConnection *)cptr)->restoreStringBuffering(bstream);
+#endif
 }
 
 void SimDataBlockEvent::write(NetConnection *cptr, BitStream *bstream)