db-cache -- implementation of datablock caching system.

This commit is contained in:
Marc Chapman 2017-07-26 23:41:57 +01:00
parent 169e539e47
commit fec893cd8b
3 changed files with 492 additions and 1 deletions

View file

@ -20,6 +20,11 @@
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames
// Copyright (C) 2015 Faust Logic, Inc.
//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
#include "platform/platform.h"
#include "T3D/gameBase/gameConnection.h"
@ -52,6 +57,10 @@
#include "T3D/gameBase/std/stdMoveList.h"
#endif
#ifdef AFX_CAP_DATABLOCK_CACHE
#include "core/stream/fileStream.h"
#endif
//----------------------------------------------------------------------------
#define MAX_MOVE_PACKET_SENDS 4
@ -173,9 +182,20 @@ IMPLEMENT_CALLBACK( GameConnection, onFlash, void, (bool state), (state),
"either is on or both are off. Typically this is used to enable the flash postFx.\n\n"
"@param state Set to true if either the damage flash or white out conditions are active.\n\n");
#ifdef AFX_CAP_DATABLOCK_CACHE
StringTableEntry GameConnection::server_cache_filename = "";
StringTableEntry GameConnection::client_cache_filename = "";
bool GameConnection::server_cache_on = false;
bool GameConnection::client_cache_on = false;
#endif
//----------------------------------------------------------------------------
GameConnection::GameConnection()
{
#ifdef AFX_CAP_DATABLOCK_CACHE
client_db_stream = new InfiniteBitStream;
server_cache_CRC = 0xffffffff;
#endif
mLagging = false;
mControlObject = NULL;
mCameraObject = NULL;
@ -246,6 +266,10 @@ GameConnection::~GameConnection()
dFree(mConnectArgv[i]);
dFree(mJoinPassword);
delete mMoveList;
#ifdef AFX_CAP_DATABLOCK_CACHE
delete client_db_stream;
#endif
}
//----------------------------------------------------------------------------
@ -1608,6 +1632,14 @@ void GameConnection::preloadNextDataBlock(bool hadNewFiles)
sendConnectionMessage(DataBlocksDownloadDone, mDataBlockSequence);
// gResourceManager->setMissingFileLogging(false);
#ifdef AFX_CAP_DATABLOCK_CACHE
// This should be the last of the datablocks. An argument of false
// indicates that this is a client save.
if (clientCacheEnabled())
saveDatablockCache(false);
#endif
return;
}
mFilesWereDownloaded = hadNewFiles;
@ -1771,7 +1803,11 @@ DefineEngineMethod( GameConnection, transmitDataBlocks, void, (S32 sequence),,
const U32 iCount = pGroup->size();
// If this is the local client...
#ifdef AFX_CAP_DATABLOCK_CACHE
if (GameConnection::getLocalClientConnection() == object && !GameConnection::serverCacheEnabled())
#else
if (GameConnection::getLocalClientConnection() == object)
#endif
{
// Set up a pointer to the datablock.
SimDataBlock* pDataBlock = 0;
@ -2166,6 +2202,13 @@ void GameConnection::consoleInit()
"@ingroup Networking\n");
// Con::addVariable("specialFog", TypeBool, &SceneGraph::useSpecial);
#ifdef AFX_CAP_DATABLOCK_CACHE
Con::addVariable("$Pref::Server::DatablockCacheFilename", TypeString, &server_cache_filename);
Con::addVariable("$pref::Client::DatablockCacheFilename", TypeString, &client_cache_filename);
Con::addVariable("$Pref::Server::EnableDatablockCache", TypeBool, &server_cache_on);
Con::addVariable("$pref::Client::EnableDatablockCache", TypeBool, &client_cache_on);
#endif
}
DefineEngineMethod( GameConnection, startRecording, void, (const char* fileName),,
@ -2360,4 +2403,393 @@ DefineEngineMethod( GameConnection, getVisibleGhostDistance, F32, (),,
)
{
return object->getVisibleGhostDistance();
}
}
#ifdef AFX_CAP_DATABLOCK_CACHE
void GameConnection::tempDisableStringBuffering(BitStream* bs) const
{
bs->setStringBuffer(0);
}
void GameConnection::restoreStringBuffering(BitStream* bs) const
{
bs->clearStringBuffer();
}
// rewind to stream postion and then move raw bytes into client_db_stream
// for caching purposes.
void GameConnection::repackClientDatablock(BitStream* bstream, S32 start_pos)
{
static U8 bit_buffer[Net::MaxPacketDataSize];
if (!clientCacheEnabled() || !client_db_stream)
return;
S32 cur_pos = bstream->getCurPos();
S32 n_bits = cur_pos - start_pos;
if (n_bits <= 0)
return;
bstream->setCurPos(start_pos);
bstream->readBits(n_bits, bit_buffer);
bstream->setCurPos(cur_pos);
//S32 start_pos2 = client_db_stream->getCurPos();
client_db_stream->writeBits(n_bits, bit_buffer);
}
#define CLIENT_CACHE_VERSION_CODE 47241113
void GameConnection::saveDatablockCache(bool on_server)
{
InfiniteBitStream bit_stream;
BitStream* bstream = 0;
if (on_server)
{
SimDataBlockGroup *g = Sim::getDataBlockGroup();
// find the first one we haven't sent:
U32 i, groupCount = g->size();
S32 key = this->getDataBlockModifiedKey();
for (i = 0; i < groupCount; i++)
if (((SimDataBlock*)(*g)[i])->getModifiedKey() > key)
break;
// nothing to save
if (i == groupCount)
return;
bstream = &bit_stream;
for (;i < groupCount; i++)
{
SimDataBlock* obj = (SimDataBlock*)(*g)[i];
GameConnection* gc = this;
NetConnection* conn = this;
SimObjectId id = obj->getId();
if (bstream->writeFlag(gc->getDataBlockModifiedKey() < obj->getModifiedKey())) // A - flag
{
if (obj->getModifiedKey() > gc->getMaxDataBlockModifiedKey())
gc->setMaxDataBlockModifiedKey(obj->getModifiedKey());
bstream->writeInt(id - DataBlockObjectIdFirst,DataBlockObjectIdBitSize); // B - int
S32 classId = obj->getClassId(conn->getNetClassGroup());
bstream->writeClassId(classId, NetClassTypeDataBlock, conn->getNetClassGroup()); // C - id
bstream->writeInt(i, DataBlockObjectIdBitSize); // D - int
bstream->writeInt(groupCount, DataBlockObjectIdBitSize + 1); // E - int
obj->packData(bstream);
}
}
}
else
{
bstream = client_db_stream;
}
if (bstream->getPosition() <= 0)
return;
// zero out any leftover bits short of an even byte count
U32 n_leftover_bits = (bstream->getPosition()*8) - bstream->getCurPos();
if (n_leftover_bits >= 0 && n_leftover_bits <= 8)
{
// note - an unusual problem regarding setCurPos() results when there
// are no leftover bytes. Adding a buffer byte in this case avoids the problem.
if (n_leftover_bits == 0)
n_leftover_bits = 8;
U8 bzero = 0;
bstream->writeBits(n_leftover_bits, &bzero);
}
// this is where we actually save the file
const char* filename = (on_server) ? server_cache_filename : client_cache_filename;
if (filename && filename[0] != '\0')
{
FileStream* f_stream;
if((f_stream = FileStream::createAndOpen(filename, Torque::FS::File::Write )) == NULL)
{
Con::printf("Failed to open file '%s'.", filename);
return;
}
U32 save_sz = bstream->getPosition();
if (!on_server)
{
f_stream->write((U32)CLIENT_CACHE_VERSION_CODE);
f_stream->write(save_sz);
f_stream->write(server_cache_CRC);
f_stream->write((U32)CLIENT_CACHE_VERSION_CODE);
}
f_stream->write(save_sz, bstream->getBuffer());
// zero out any leftover bytes short of a 4-byte multiple
while ((save_sz % 4) != 0)
{
f_stream->write((U8)0);
save_sz++;
}
delete f_stream;
}
if (!on_server)
client_db_stream->clear();
}
static bool afx_saved_db_cache = false;
static U32 afx_saved_db_cache_CRC = 0xffffffff;
void GameConnection::resetDatablockCache()
{
afx_saved_db_cache = false;
afx_saved_db_cache_CRC = 0xffffffff;
}
ConsoleFunction(resetDatablockCache, void, 1, 1, "resetDatablockCache()")
{
GameConnection::resetDatablockCache();
}
ConsoleFunction(isDatablockCacheSaved, bool, 1, 1, "resetDatablockCache()")
{
return afx_saved_db_cache;
}
ConsoleFunction(getDatablockCacheCRC, S32, 1, 1, "getDatablockCacheCRC()")
{
return (S32)afx_saved_db_cache_CRC;
}
ConsoleFunction(extractDatablockCacheCRC, S32, 2, 2, "extractDatablockCacheCRC(filename)")
{
FileStream f_stream;
const char* fileName = argv[1];
if(!f_stream.open(fileName, Torque::FS::File::Read))
{
Con::errorf("Failed to open file '%s'.", fileName);
return -1;
}
U32 stream_sz = f_stream.getStreamSize();
if (stream_sz < 4*32)
{
Con::errorf("File '%s' is not a valid datablock cache.", fileName);
f_stream.close();
return -1;
}
U32 pre_code; f_stream.read(&pre_code);
U32 save_sz; f_stream.read(&save_sz);
U32 crc_code; f_stream.read(&crc_code);
U32 post_code; f_stream.read(&post_code);
f_stream.close();
if (pre_code != post_code)
{
Con::errorf("File '%s' is not a valid datablock cache.", fileName);
return -1;
}
if (pre_code != (U32)CLIENT_CACHE_VERSION_CODE)
{
Con::errorf("Version of datablock cache file '%s' does not match version of running software.", fileName);
return -1;
}
return (S32)crc_code;
}
ConsoleFunction(setDatablockCacheCRC, void, 2, 2, "setDatablockCacheCRC(crc)")
{
GameConnection *conn = GameConnection::getConnectionToServer();
if(!conn)
return;
U32 crc_u = (U32)dAtoi(argv[1]);
conn->setServerCacheCRC(crc_u);
}
ConsoleMethod( GameConnection, saveDatablockCache, void, 2, 2, "saveDatablockCache()")
{
if (GameConnection::serverCacheEnabled() && !afx_saved_db_cache)
{
// Save the datablocks to a cache file. An argument
// of true indicates that this is a server save.
object->saveDatablockCache(true);
afx_saved_db_cache = true;
afx_saved_db_cache_CRC = 0xffffffff;
static char filename_buffer[1024];
String filename(Torque::Path::CleanSeparators(object->serverCacheFilename()));
Con::expandScriptFilename(filename_buffer, sizeof(filename_buffer), filename.c_str());
Torque::Path givenPath(Torque::Path::CompressPath(filename_buffer));
Torque::FS::FileNodeRef fileRef = Torque::FS::GetFileNode(givenPath);
if ( fileRef == NULL )
Con::errorf("saveDatablockCache() failed to get CRC for file '%s'.", filename.c_str());
else
afx_saved_db_cache_CRC = (S32)fileRef->getChecksum();
}
}
ConsoleMethod( GameConnection, loadDatablockCache, void, 2, 2, "loadDatablockCache()")
{
if (GameConnection::clientCacheEnabled())
{
object->loadDatablockCache();
}
}
ConsoleMethod( GameConnection, loadDatablockCache_Begin, bool, 2, 2, "loadDatablockCache_Begin()")
{
if (GameConnection::clientCacheEnabled())
{
return object->loadDatablockCache_Begin();
}
return false;
}
ConsoleMethod( GameConnection, loadDatablockCache_Continue, bool, 2, 2, "loadDatablockCache_Continue()")
{
if (GameConnection::clientCacheEnabled())
{
return object->loadDatablockCache_Continue();
}
return false;
}
static char* afx_db_load_buf = 0;
static U32 afx_db_load_buf_sz = 0;
static BitStream* afx_db_load_bstream = 0;
void GameConnection::loadDatablockCache()
{
if (!loadDatablockCache_Begin())
return;
while (loadDatablockCache_Continue())
;
}
bool GameConnection::loadDatablockCache_Begin()
{
if (!client_cache_filename || client_cache_filename[0] == '\0')
{
Con::errorf("No filename was specified for the client datablock cache.");
return false;
}
// open cache file
FileStream f_stream;
if(!f_stream.open(client_cache_filename, Torque::FS::File::Read))
{
Con::errorf("Failed to open file '%s'.", client_cache_filename);
return false;
}
// get file size
U32 stream_sz = f_stream.getStreamSize();
if (stream_sz <= 4*4)
{
Con::errorf("File '%s' is too small to be a valid datablock cache.", client_cache_filename);
f_stream.close();
return false;
}
// load header data
U32 pre_code; f_stream.read(&pre_code);
U32 save_sz; f_stream.read(&save_sz);
U32 crc_code; f_stream.read(&crc_code);
U32 post_code; f_stream.read(&post_code);
// validate header info
if (pre_code != post_code)
{
Con::errorf("File '%s' is not a valid datablock cache.", client_cache_filename);
f_stream.close();
return false;
}
if (pre_code != (U32)CLIENT_CACHE_VERSION_CODE)
{
Con::errorf("Version of datablock cache file '%s' does not match version of running software.", client_cache_filename);
f_stream.close();
return false;
}
// allocated the in-memory buffer
afx_db_load_buf_sz = stream_sz - (4*4);
afx_db_load_buf = new char[afx_db_load_buf_sz];
// load data from file into memory
if (!f_stream.read(stream_sz, afx_db_load_buf))
{
Con::errorf("Failed to read data from file '%s'.", client_cache_filename);
f_stream.close();
delete [] afx_db_load_buf;
afx_db_load_buf = 0;
afx_db_load_buf_sz = 0;
return false;
}
// close file
f_stream.close();
// At this point we have the whole cache in memory
// create a bitstream from the in-memory buffer
afx_db_load_bstream = new BitStream(afx_db_load_buf, afx_db_load_buf_sz);
return true;
}
bool GameConnection::loadDatablockCache_Continue()
{
if (!afx_db_load_bstream)
return false;
// prevent repacking of datablocks during load
BitStream* save_client_db_stream = client_db_stream;
client_db_stream = 0;
bool all_finished = false;
// loop through at most 16 datablocks
BitStream *bstream = afx_db_load_bstream;
for (S32 i = 0; i < 16; i++)
{
S32 save_pos = bstream->getCurPos();
if (!bstream->readFlag())
{
all_finished = true;
break;
}
bstream->setCurPos(save_pos);
SimDataBlockEvent evt;
evt.unpack(this, bstream);
evt.process(this);
}
client_db_stream = save_client_db_stream;
if (all_finished)
{
delete afx_db_load_bstream;
afx_db_load_bstream = 0;
delete [] afx_db_load_buf;
afx_db_load_buf = 0;
afx_db_load_buf_sz = 0;
return false;
}
return true;
}
#endif

View file

@ -20,6 +20,11 @@
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames
// Copyright (C) 2015 Faust Logic, Inc.
//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
#ifndef _GAMECONNECTION_H_
#define _GAMECONNECTION_H_
@ -55,6 +60,14 @@ class MoveList;
struct Move;
struct AuthInfo;
// To disable datablock caching, remove or comment out the AFX_CAP_DATABLOCK_CACHE define below.
// Also, at a minimum, the following script preferences should be set to false:
// $pref::Client::EnableDatablockCache = false; (in arcane.fx/client/defaults.cs)
// $Pref::Server::EnableDatablockCache = false; (in arcane.fx/server/defaults.cs)
// Alternatively, all script code marked with "DATABLOCK CACHE CODE" can be removed or
// commented out.
//
#define AFX_CAP_DATABLOCK_CACHE
const F32 MinCameraFov = 1.f; ///< min camera FOV
const F32 MaxCameraFov = 179.f; ///< max camera FOV
@ -372,6 +385,31 @@ protected:
DECLARE_CALLBACK( void, setLagIcon, (bool state) );
DECLARE_CALLBACK( void, onDataBlocksDone, (U32 sequence) );
DECLARE_CALLBACK( void, onFlash, (bool state) );
#ifdef AFX_CAP_DATABLOCK_CACHE
private:
static StringTableEntry server_cache_filename;
static StringTableEntry client_cache_filename;
static bool server_cache_on;
static bool client_cache_on;
BitStream* client_db_stream;
U32 server_cache_CRC;
public:
void repackClientDatablock(BitStream*, S32 start_pos);
void saveDatablockCache(bool on_server);
void loadDatablockCache();
bool loadDatablockCache_Begin();
bool loadDatablockCache_Continue();
void tempDisableStringBuffering(BitStream* bs) const;
void restoreStringBuffering(BitStream* bs) const;
void setServerCacheCRC(U32 crc) { server_cache_CRC = crc; }
static void resetDatablockCache();
static bool serverCacheEnabled() { return server_cache_on; }
static bool clientCacheEnabled() { return client_cache_on; }
static const char* serverCacheFilename() { return server_cache_filename; }
static const char* clientCacheFilename() { return client_cache_filename; }
#endif
};
#endif

View file

@ -20,6 +20,11 @@
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
// Arcane-FX for MIT Licensed Open Source version of Torque 3D from GarageGames
// Copyright (C) 2015 Faust Logic, Inc.
//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~//~~~~~~~~~~~~~~~~~~~~~//
#include "platform/platform.h"
#include "core/dnet.h"
#include "core/stream/bitStream.h"
@ -136,6 +141,9 @@ void SimDataBlockEvent::notifyDelivered(NetConnection *conn, bool )
void SimDataBlockEvent::pack(NetConnection *conn, BitStream *bstream)
{
#ifdef AFX_CAP_DATABLOCK_CACHE
((GameConnection *)conn)->tempDisableStringBuffering(bstream);
#endif
SimDataBlock* obj;
Sim::findObject(id,obj);
GameConnection *gc = (GameConnection *) conn;
@ -157,10 +165,18 @@ void SimDataBlockEvent::pack(NetConnection *conn, BitStream *bstream)
bstream->writeInt(classId ^ DebugChecksum, 32);
#endif
}
#ifdef AFX_CAP_DATABLOCK_CACHE
((GameConnection *)conn)->restoreStringBuffering(bstream);
#endif
}
void SimDataBlockEvent::unpack(NetConnection *cptr, BitStream *bstream)
{
#ifdef AFX_CAP_DATABLOCK_CACHE
// stash the stream position prior to unpacking
S32 start_pos = bstream->getCurPos();
((GameConnection *)cptr)->tempDisableStringBuffering(bstream);
#endif
if(bstream->readFlag())
{
mProcess = true;
@ -215,6 +231,11 @@ void SimDataBlockEvent::unpack(NetConnection *cptr, BitStream *bstream)
#endif
}
#ifdef AFX_CAP_DATABLOCK_CACHE
// rewind to stream position and then process raw bytes for caching
((GameConnection *)cptr)->repackClientDatablock(bstream, start_pos);
((GameConnection *)cptr)->restoreStringBuffering(bstream);
#endif
}
void SimDataBlockEvent::write(NetConnection *cptr, BitStream *bstream)