From fec893cd8be1244fb7e89e9a92bb4e126ee72f65 Mon Sep 17 00:00:00 2001 From: Marc Chapman Date: Wed, 26 Jul 2017 23:41:57 +0100 Subject: [PATCH] db-cache -- implementation of datablock caching system. --- Engine/source/T3D/gameBase/gameConnection.cpp | 434 +++++++++++++++++- Engine/source/T3D/gameBase/gameConnection.h | 38 ++ .../T3D/gameBase/gameConnectionEvents.cpp | 21 + 3 files changed, 492 insertions(+), 1 deletion(-) diff --git a/Engine/source/T3D/gameBase/gameConnection.cpp b/Engine/source/T3D/gameBase/gameConnection.cpp index 08125c261..989c9fd56 100644 --- a/Engine/source/T3D/gameBase/gameConnection.cpp +++ b/Engine/source/T3D/gameBase/gameConnection.cpp @@ -20,6 +20,11 @@ // IN THE SOFTWARE. //----------------------------------------------------------------------------- +//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~// +// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames +// Copyright (C) 2015 Faust Logic, Inc. +//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~// + #include "platform/platform.h" #include "T3D/gameBase/gameConnection.h" @@ -52,6 +57,10 @@ #include "T3D/gameBase/std/stdMoveList.h" #endif +#ifdef AFX_CAP_DATABLOCK_CACHE +#include "core/stream/fileStream.h" +#endif + //---------------------------------------------------------------------------- #define MAX_MOVE_PACKET_SENDS 4 @@ -173,9 +182,20 @@ IMPLEMENT_CALLBACK( GameConnection, onFlash, void, (bool state), (state), "either is on or both are off. Typically this is used to enable the flash postFx.\n\n" "@param state Set to true if either the damage flash or white out conditions are active.\n\n"); +#ifdef AFX_CAP_DATABLOCK_CACHE +StringTableEntry GameConnection::server_cache_filename = ""; +StringTableEntry GameConnection::client_cache_filename = ""; +bool GameConnection::server_cache_on = false; +bool GameConnection::client_cache_on = false; +#endif //---------------------------------------------------------------------------- GameConnection::GameConnection() { + +#ifdef AFX_CAP_DATABLOCK_CACHE + client_db_stream = new InfiniteBitStream; + server_cache_CRC = 0xffffffff; +#endif mLagging = false; mControlObject = NULL; mCameraObject = NULL; @@ -246,6 +266,10 @@ GameConnection::~GameConnection() dFree(mConnectArgv[i]); dFree(mJoinPassword); delete mMoveList; + +#ifdef AFX_CAP_DATABLOCK_CACHE + delete client_db_stream; +#endif } //---------------------------------------------------------------------------- @@ -1608,6 +1632,14 @@ void GameConnection::preloadNextDataBlock(bool hadNewFiles) sendConnectionMessage(DataBlocksDownloadDone, mDataBlockSequence); // gResourceManager->setMissingFileLogging(false); + +#ifdef AFX_CAP_DATABLOCK_CACHE + // This should be the last of the datablocks. An argument of false + // indicates that this is a client save. + if (clientCacheEnabled()) + saveDatablockCache(false); +#endif + return; } mFilesWereDownloaded = hadNewFiles; @@ -1771,7 +1803,11 @@ DefineEngineMethod( GameConnection, transmitDataBlocks, void, (S32 sequence),, const U32 iCount = pGroup->size(); // If this is the local client... +#ifdef AFX_CAP_DATABLOCK_CACHE + if (GameConnection::getLocalClientConnection() == object && !GameConnection::serverCacheEnabled()) +#else if (GameConnection::getLocalClientConnection() == object) +#endif { // Set up a pointer to the datablock. SimDataBlock* pDataBlock = 0; @@ -2166,6 +2202,13 @@ void GameConnection::consoleInit() "@ingroup Networking\n"); // Con::addVariable("specialFog", TypeBool, &SceneGraph::useSpecial); + +#ifdef AFX_CAP_DATABLOCK_CACHE + Con::addVariable("$Pref::Server::DatablockCacheFilename", TypeString, &server_cache_filename); + Con::addVariable("$pref::Client::DatablockCacheFilename", TypeString, &client_cache_filename); + Con::addVariable("$Pref::Server::EnableDatablockCache", TypeBool, &server_cache_on); + Con::addVariable("$pref::Client::EnableDatablockCache", TypeBool, &client_cache_on); +#endif } DefineEngineMethod( GameConnection, startRecording, void, (const char* fileName),, @@ -2360,4 +2403,393 @@ DefineEngineMethod( GameConnection, getVisibleGhostDistance, F32, (),, ) { return object->getVisibleGhostDistance(); -} \ No newline at end of file +} + +#ifdef AFX_CAP_DATABLOCK_CACHE + +void GameConnection::tempDisableStringBuffering(BitStream* bs) const +{ + bs->setStringBuffer(0); +} + +void GameConnection::restoreStringBuffering(BitStream* bs) const +{ + bs->clearStringBuffer(); +} + +// rewind to stream postion and then move raw bytes into client_db_stream +// for caching purposes. +void GameConnection::repackClientDatablock(BitStream* bstream, S32 start_pos) +{ + static U8 bit_buffer[Net::MaxPacketDataSize]; + + if (!clientCacheEnabled() || !client_db_stream) + return; + + S32 cur_pos = bstream->getCurPos(); + S32 n_bits = cur_pos - start_pos; + if (n_bits <= 0) + return; + + bstream->setCurPos(start_pos); + bstream->readBits(n_bits, bit_buffer); + bstream->setCurPos(cur_pos); + + //S32 start_pos2 = client_db_stream->getCurPos(); + client_db_stream->writeBits(n_bits, bit_buffer); +} + +#define CLIENT_CACHE_VERSION_CODE 47241113 + +void GameConnection::saveDatablockCache(bool on_server) +{ + InfiniteBitStream bit_stream; + BitStream* bstream = 0; + + if (on_server) + { + SimDataBlockGroup *g = Sim::getDataBlockGroup(); + + // find the first one we haven't sent: + U32 i, groupCount = g->size(); + S32 key = this->getDataBlockModifiedKey(); + for (i = 0; i < groupCount; i++) + if (((SimDataBlock*)(*g)[i])->getModifiedKey() > key) + break; + + // nothing to save + if (i == groupCount) + return; + + bstream = &bit_stream; + + for (;i < groupCount; i++) + { + SimDataBlock* obj = (SimDataBlock*)(*g)[i]; + GameConnection* gc = this; + NetConnection* conn = this; + SimObjectId id = obj->getId(); + + if (bstream->writeFlag(gc->getDataBlockModifiedKey() < obj->getModifiedKey())) // A - flag + { + if (obj->getModifiedKey() > gc->getMaxDataBlockModifiedKey()) + gc->setMaxDataBlockModifiedKey(obj->getModifiedKey()); + + bstream->writeInt(id - DataBlockObjectIdFirst,DataBlockObjectIdBitSize); // B - int + + S32 classId = obj->getClassId(conn->getNetClassGroup()); + bstream->writeClassId(classId, NetClassTypeDataBlock, conn->getNetClassGroup()); // C - id + bstream->writeInt(i, DataBlockObjectIdBitSize); // D - int + bstream->writeInt(groupCount, DataBlockObjectIdBitSize + 1); // E - int + obj->packData(bstream); + } + } + } + else + { + bstream = client_db_stream; + } + + if (bstream->getPosition() <= 0) + return; + + // zero out any leftover bits short of an even byte count + U32 n_leftover_bits = (bstream->getPosition()*8) - bstream->getCurPos(); + if (n_leftover_bits >= 0 && n_leftover_bits <= 8) + { + // note - an unusual problem regarding setCurPos() results when there + // are no leftover bytes. Adding a buffer byte in this case avoids the problem. + if (n_leftover_bits == 0) + n_leftover_bits = 8; + U8 bzero = 0; + bstream->writeBits(n_leftover_bits, &bzero); + } + + // this is where we actually save the file + const char* filename = (on_server) ? server_cache_filename : client_cache_filename; + if (filename && filename[0] != '\0') + { + FileStream* f_stream; + if((f_stream = FileStream::createAndOpen(filename, Torque::FS::File::Write )) == NULL) + { + Con::printf("Failed to open file '%s'.", filename); + return; + } + + U32 save_sz = bstream->getPosition(); + + if (!on_server) + { + f_stream->write((U32)CLIENT_CACHE_VERSION_CODE); + f_stream->write(save_sz); + f_stream->write(server_cache_CRC); + f_stream->write((U32)CLIENT_CACHE_VERSION_CODE); + } + + f_stream->write(save_sz, bstream->getBuffer()); + + // zero out any leftover bytes short of a 4-byte multiple + while ((save_sz % 4) != 0) + { + f_stream->write((U8)0); + save_sz++; + } + + delete f_stream; + } + + if (!on_server) + client_db_stream->clear(); +} + +static bool afx_saved_db_cache = false; +static U32 afx_saved_db_cache_CRC = 0xffffffff; + +void GameConnection::resetDatablockCache() +{ + afx_saved_db_cache = false; + afx_saved_db_cache_CRC = 0xffffffff; +} + +ConsoleFunction(resetDatablockCache, void, 1, 1, "resetDatablockCache()") +{ + GameConnection::resetDatablockCache(); +} + +ConsoleFunction(isDatablockCacheSaved, bool, 1, 1, "resetDatablockCache()") +{ + return afx_saved_db_cache; +} + +ConsoleFunction(getDatablockCacheCRC, S32, 1, 1, "getDatablockCacheCRC()") +{ + return (S32)afx_saved_db_cache_CRC; +} + +ConsoleFunction(extractDatablockCacheCRC, S32, 2, 2, "extractDatablockCacheCRC(filename)") +{ + FileStream f_stream; + const char* fileName = argv[1]; + if(!f_stream.open(fileName, Torque::FS::File::Read)) + { + Con::errorf("Failed to open file '%s'.", fileName); + return -1; + } + + U32 stream_sz = f_stream.getStreamSize(); + if (stream_sz < 4*32) + { + Con::errorf("File '%s' is not a valid datablock cache.", fileName); + f_stream.close(); + return -1; + } + + U32 pre_code; f_stream.read(&pre_code); + U32 save_sz; f_stream.read(&save_sz); + U32 crc_code; f_stream.read(&crc_code); + U32 post_code; f_stream.read(&post_code); + + f_stream.close(); + + if (pre_code != post_code) + { + Con::errorf("File '%s' is not a valid datablock cache.", fileName); + return -1; + } + + if (pre_code != (U32)CLIENT_CACHE_VERSION_CODE) + { + Con::errorf("Version of datablock cache file '%s' does not match version of running software.", fileName); + return -1; + } + + return (S32)crc_code; +} + +ConsoleFunction(setDatablockCacheCRC, void, 2, 2, "setDatablockCacheCRC(crc)") +{ + GameConnection *conn = GameConnection::getConnectionToServer(); + if(!conn) + return; + + U32 crc_u = (U32)dAtoi(argv[1]); + conn->setServerCacheCRC(crc_u); +} + +ConsoleMethod( GameConnection, saveDatablockCache, void, 2, 2, "saveDatablockCache()") +{ + if (GameConnection::serverCacheEnabled() && !afx_saved_db_cache) + { + // Save the datablocks to a cache file. An argument + // of true indicates that this is a server save. + object->saveDatablockCache(true); + afx_saved_db_cache = true; + afx_saved_db_cache_CRC = 0xffffffff; + + static char filename_buffer[1024]; + String filename(Torque::Path::CleanSeparators(object->serverCacheFilename())); + Con::expandScriptFilename(filename_buffer, sizeof(filename_buffer), filename.c_str()); + Torque::Path givenPath(Torque::Path::CompressPath(filename_buffer)); + Torque::FS::FileNodeRef fileRef = Torque::FS::GetFileNode(givenPath); + if ( fileRef == NULL ) + Con::errorf("saveDatablockCache() failed to get CRC for file '%s'.", filename.c_str()); + else + afx_saved_db_cache_CRC = (S32)fileRef->getChecksum(); + } +} + +ConsoleMethod( GameConnection, loadDatablockCache, void, 2, 2, "loadDatablockCache()") +{ + if (GameConnection::clientCacheEnabled()) + { + object->loadDatablockCache(); + } +} + +ConsoleMethod( GameConnection, loadDatablockCache_Begin, bool, 2, 2, "loadDatablockCache_Begin()") +{ + if (GameConnection::clientCacheEnabled()) + { + return object->loadDatablockCache_Begin(); + } + + return false; +} + +ConsoleMethod( GameConnection, loadDatablockCache_Continue, bool, 2, 2, "loadDatablockCache_Continue()") +{ + if (GameConnection::clientCacheEnabled()) + { + return object->loadDatablockCache_Continue(); + } + + return false; +} + +static char* afx_db_load_buf = 0; +static U32 afx_db_load_buf_sz = 0; +static BitStream* afx_db_load_bstream = 0; + +void GameConnection::loadDatablockCache() +{ + if (!loadDatablockCache_Begin()) + return; + + while (loadDatablockCache_Continue()) + ; +} + +bool GameConnection::loadDatablockCache_Begin() +{ + if (!client_cache_filename || client_cache_filename[0] == '\0') + { + Con::errorf("No filename was specified for the client datablock cache."); + return false; + } + + // open cache file + FileStream f_stream; + if(!f_stream.open(client_cache_filename, Torque::FS::File::Read)) + { + Con::errorf("Failed to open file '%s'.", client_cache_filename); + return false; + } + + // get file size + U32 stream_sz = f_stream.getStreamSize(); + if (stream_sz <= 4*4) + { + Con::errorf("File '%s' is too small to be a valid datablock cache.", client_cache_filename); + f_stream.close(); + return false; + } + + // load header data + U32 pre_code; f_stream.read(&pre_code); + U32 save_sz; f_stream.read(&save_sz); + U32 crc_code; f_stream.read(&crc_code); + U32 post_code; f_stream.read(&post_code); + + // validate header info + if (pre_code != post_code) + { + Con::errorf("File '%s' is not a valid datablock cache.", client_cache_filename); + f_stream.close(); + return false; + } + if (pre_code != (U32)CLIENT_CACHE_VERSION_CODE) + { + Con::errorf("Version of datablock cache file '%s' does not match version of running software.", client_cache_filename); + f_stream.close(); + return false; + } + + // allocated the in-memory buffer + afx_db_load_buf_sz = stream_sz - (4*4); + afx_db_load_buf = new char[afx_db_load_buf_sz]; + + // load data from file into memory + if (!f_stream.read(stream_sz, afx_db_load_buf)) + { + Con::errorf("Failed to read data from file '%s'.", client_cache_filename); + f_stream.close(); + delete [] afx_db_load_buf; + afx_db_load_buf = 0; + afx_db_load_buf_sz = 0; + return false; + } + + // close file + f_stream.close(); + + // At this point we have the whole cache in memory + + // create a bitstream from the in-memory buffer + afx_db_load_bstream = new BitStream(afx_db_load_buf, afx_db_load_buf_sz); + + return true; +} + +bool GameConnection::loadDatablockCache_Continue() +{ + if (!afx_db_load_bstream) + return false; + + // prevent repacking of datablocks during load + BitStream* save_client_db_stream = client_db_stream; + client_db_stream = 0; + + bool all_finished = false; + + // loop through at most 16 datablocks + BitStream *bstream = afx_db_load_bstream; + for (S32 i = 0; i < 16; i++) + { + S32 save_pos = bstream->getCurPos(); + if (!bstream->readFlag()) + { + all_finished = true; + break; + } + bstream->setCurPos(save_pos); + SimDataBlockEvent evt; + evt.unpack(this, bstream); + evt.process(this); + } + + client_db_stream = save_client_db_stream; + + if (all_finished) + { + delete afx_db_load_bstream; + afx_db_load_bstream = 0; + delete [] afx_db_load_buf; + afx_db_load_buf = 0; + afx_db_load_buf_sz = 0; + return false; + } + + return true; +} + +#endif diff --git a/Engine/source/T3D/gameBase/gameConnection.h b/Engine/source/T3D/gameBase/gameConnection.h index 13084a637..a234960b0 100644 --- a/Engine/source/T3D/gameBase/gameConnection.h +++ b/Engine/source/T3D/gameBase/gameConnection.h @@ -20,6 +20,11 @@ // IN THE SOFTWARE. //----------------------------------------------------------------------------- +//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~// +// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames +// Copyright (C) 2015 Faust Logic, Inc. +//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~// + #ifndef _GAMECONNECTION_H_ #define _GAMECONNECTION_H_ @@ -55,6 +60,14 @@ class MoveList; struct Move; struct AuthInfo; +// To disable datablock caching, remove or comment out the AFX_CAP_DATABLOCK_CACHE define below. +// Also, at a minimum, the following script preferences should be set to false: +// $pref::Client::EnableDatablockCache = false; (in arcane.fx/client/defaults.cs) +// $Pref::Server::EnableDatablockCache = false; (in arcane.fx/server/defaults.cs) +// Alternatively, all script code marked with "DATABLOCK CACHE CODE" can be removed or +// commented out. +// +#define AFX_CAP_DATABLOCK_CACHE const F32 MinCameraFov = 1.f; ///< min camera FOV const F32 MaxCameraFov = 179.f; ///< max camera FOV @@ -372,6 +385,31 @@ protected: DECLARE_CALLBACK( void, setLagIcon, (bool state) ); DECLARE_CALLBACK( void, onDataBlocksDone, (U32 sequence) ); DECLARE_CALLBACK( void, onFlash, (bool state) ); + +#ifdef AFX_CAP_DATABLOCK_CACHE +private: + static StringTableEntry server_cache_filename; + static StringTableEntry client_cache_filename; + static bool server_cache_on; + static bool client_cache_on; + BitStream* client_db_stream; + U32 server_cache_CRC; +public: + void repackClientDatablock(BitStream*, S32 start_pos); + void saveDatablockCache(bool on_server); + void loadDatablockCache(); + bool loadDatablockCache_Begin(); + bool loadDatablockCache_Continue(); + void tempDisableStringBuffering(BitStream* bs) const; + void restoreStringBuffering(BitStream* bs) const; + void setServerCacheCRC(U32 crc) { server_cache_CRC = crc; } + + static void resetDatablockCache(); + static bool serverCacheEnabled() { return server_cache_on; } + static bool clientCacheEnabled() { return client_cache_on; } + static const char* serverCacheFilename() { return server_cache_filename; } + static const char* clientCacheFilename() { return client_cache_filename; } +#endif }; #endif diff --git a/Engine/source/T3D/gameBase/gameConnectionEvents.cpp b/Engine/source/T3D/gameBase/gameConnectionEvents.cpp index ddc7b1674..e79e55c45 100644 --- a/Engine/source/T3D/gameBase/gameConnectionEvents.cpp +++ b/Engine/source/T3D/gameBase/gameConnectionEvents.cpp @@ -20,6 +20,11 @@ // IN THE SOFTWARE. //----------------------------------------------------------------------------- +//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~// +// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames +// Copyright (C) 2015 Faust Logic, Inc. +//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~// + #include "platform/platform.h" #include "core/dnet.h" #include "core/stream/bitStream.h" @@ -136,6 +141,9 @@ void SimDataBlockEvent::notifyDelivered(NetConnection *conn, bool ) void SimDataBlockEvent::pack(NetConnection *conn, BitStream *bstream) { +#ifdef AFX_CAP_DATABLOCK_CACHE + ((GameConnection *)conn)->tempDisableStringBuffering(bstream); +#endif SimDataBlock* obj; Sim::findObject(id,obj); GameConnection *gc = (GameConnection *) conn; @@ -157,10 +165,18 @@ void SimDataBlockEvent::pack(NetConnection *conn, BitStream *bstream) bstream->writeInt(classId ^ DebugChecksum, 32); #endif } +#ifdef AFX_CAP_DATABLOCK_CACHE + ((GameConnection *)conn)->restoreStringBuffering(bstream); +#endif } void SimDataBlockEvent::unpack(NetConnection *cptr, BitStream *bstream) { +#ifdef AFX_CAP_DATABLOCK_CACHE + // stash the stream position prior to unpacking + S32 start_pos = bstream->getCurPos(); + ((GameConnection *)cptr)->tempDisableStringBuffering(bstream); +#endif if(bstream->readFlag()) { mProcess = true; @@ -215,6 +231,11 @@ void SimDataBlockEvent::unpack(NetConnection *cptr, BitStream *bstream) #endif } +#ifdef AFX_CAP_DATABLOCK_CACHE + // rewind to stream position and then process raw bytes for caching + ((GameConnection *)cptr)->repackClientDatablock(bstream, start_pos); + ((GameConnection *)cptr)->restoreStringBuffering(bstream); +#endif } void SimDataBlockEvent::write(NetConnection *cptr, BitStream *bstream)