From 0b21072b101cffba54fdb396dddb06751bf7e956 Mon Sep 17 00:00:00 2001 From: AzaezelX Date: Mon, 21 Oct 2019 18:55:15 -0500 Subject: [PATCH] datablock caching, and dependency-object vs hooked up class method segregation 1) relies on https://github.com/Areloch/Torque3D/pull/99 for modular resolvers (re-included for ease of testing) 2) adds a new module::onServerScriptExec(%this) callback executed after datablock transmission 3) bypasses DB transmission in favor of a straight file read if the resulting network data would (via CRC compare) match what a client already has saved off under data\cache\ --- .../scripts/client/levelDownload.cs | 120 ++++++++++- .../clientServer/scripts/server/defaults.cs | 2 - .../scripts/server/levelDownload.cs | 57 ++++- .../clientServer/scripts/server/server.cs | 8 +- .../game/core/utility/scripts/module.cs | 194 +++++++++++++++++- 5 files changed, 362 insertions(+), 19 deletions(-) diff --git a/Templates/BaseGame/game/core/clientServer/scripts/client/levelDownload.cs b/Templates/BaseGame/game/core/clientServer/scripts/client/levelDownload.cs index 58f02c92f..fbd5bfccd 100644 --- a/Templates/BaseGame/game/core/clientServer/scripts/client/levelDownload.cs +++ b/Templates/BaseGame/game/core/clientServer/scripts/client/levelDownload.cs @@ -38,7 +38,73 @@ //---------------------------------------------------------------------------- // Phase 1 //---------------------------------------------------------------------------- -function clientCmdMissionStartPhase1(%seq, %missionName) +$pref::Client::EnableDatablockCache = true; +$pref::Client::DatablockCacheFilename = "data/cache/client/datablock_cache_c.dbc"; + +function clientCmdMissionStartPhase1_LoadCache(%seq, %missionName) +{ + if ($pref::Client::EnableDatablockCache && $loadFromDatablockCache) + { + if (!$pref::Video::disableVerticalSync) + { + warn("Disabling Vertical Sync during datablock cache load to avoid significant slowdown."); + $AFX_tempDisableVSync = true; + + $pref::Video::disableVerticalSync = true; + Canvas.resetVideoMode(); + } + + echo("<<<< Loading Datablocks From Cache >>>>"); + if (ServerConnection.loadDatablockCache_Begin()) + { + schedule(10, 0, "updateLoadDatablockCacheProgress", %seq, %missionName); + } + } +} + +function updateLoadDatablockCacheProgress(%seq, %missionName) +{ + if (ServerConnection.loadDatablockCache_Continue()) + { + $loadDatablockCacheProgressThread = schedule(10, 0, "updateLoadDatablockCacheProgress", %seq, %missionName); + return; + } + + if ($AFX_tempDisableVSync) + { + warn("Restoring Vertical Sync setting."); + $AFX_tempDisableVSync = false; + + $pref::Video::disableVerticalSync = false; + Canvas.resetVideoMode(); + } + + echo("<<<< Finished Loading Datablocks From Cache >>>>"); + clientCmdMissionStartPhase2(%seq,%missionName); +} + +function updateLoadDatablockCacheProgress(%seq, %missionName) +{ + if (ServerConnection.loadDatablockCache_Continue()) + { + $loadDatablockCacheProgressThread = schedule(10, 0, "updateLoadDatablockCacheProgress", %seq, %missionName); + return; + } + + if ($AFX_tempDisableVSync) + { + warn("Restoring Vertical Sync setting."); + $AFX_tempDisableVSync = false; + + $pref::Video::disableVerticalSync = false; + Canvas.resetVideoMode(); + } + + echo("<<<< Finished Loading Datablocks From Cache >>>>"); + clientCmdMissionStartPhase2(%seq,%missionName); +} + +function clientCmdMissionStartPhase1(%seq, %missionName, %cache_crc) { // These need to come after the cls. echo ("*** New Mission: " @ %missionName); @@ -61,6 +127,56 @@ function clientCmdMissionStartPhase1(%seq, %missionName) PostFXManager::settingsApplyDefaultPreset(); } + $loadFromDatablockCache = false; + if ($pref::Client::EnableDatablockCache) + { + %cache_filename = $pref::Client::DatablockCacheFilename; + + // if cache CRC is provided, check for validity + if (%cache_crc !$= "") + { + // check for existence of cache file + if (isFile(%cache_filename)) + { + // here we are not comparing the CRC of the cache itself, but the CRC of + // the server cache (stored in the header) when these datablocks were + // transmitted. + %my_cache_crc = extractDatablockCacheCRC(%cache_filename); + echo("<<<< client cache CRC:" SPC %my_cache_crc SPC ">>>>"); + echo("<<<< comparing CRC codes:" SPC "s:" @ %cache_crc SPC "c:" @ %my_cache_crc SPC ">>>>"); + if (%my_cache_crc == %cache_crc) + { + echo("<<<< cache CRC codes match, datablocks will be loaded from local cache. >>>>"); + $loadFromDatablockCache = true; + } + else + { + echo("<<<< cache CRC codes differ, datablocks will be transmitted and cached. >>>>" SPC %cache_crc); + setDatablockCacheCRC(%cache_crc); + } + } + else + { + echo("<<<< client datablock cache does not exist, datablocks will be transmitted and cached. >>>>"); + setDatablockCacheCRC(%cache_crc); + } + } + else + { + echo("<<<< server datablock caching is disabled, datablocks will be transmitted. >>>>"); + } + if ($loadFromDatablockCache) + { + // skip datablock transmission and initiate a cache load + commandToServer('MissionStartPhase1Ack_UseCache', %seq); + return; + } + } + else if (%cache_crc !$= "") + { + echo("<<<< client datablock caching is disabled, datablocks will be transmitted. >>>>"); + } + onMissionDownloadPhase("LOADING DATABLOCKS"); commandToServer('MissionStartPhase1Ack', %seq); @@ -164,7 +280,7 @@ function connect(%server) { %conn = new GameConnection(ServerConnection); RootGroup.add(ServerConnection); - %conn.setConnectArgs($pref::Player::Name); + %conn.setConnectArgs($pref::Player::Name, $ConncetInfoKey); %conn.setJoinPassword($Client::Password); %conn.connect(%server); } diff --git a/Templates/BaseGame/game/core/clientServer/scripts/server/defaults.cs b/Templates/BaseGame/game/core/clientServer/scripts/server/defaults.cs index b8e72e213..ed213e99d 100644 --- a/Templates/BaseGame/game/core/clientServer/scripts/server/defaults.cs +++ b/Templates/BaseGame/game/core/clientServer/scripts/server/defaults.cs @@ -46,8 +46,6 @@ $Pref::Server::ConnectionError = // overrides pref::net::port for dedicated servers $Pref::Server::Port = 28000; -$Pref::Server::EnableDatablockCache = true; -$Pref::Server::DatablockCacheFilename = "core/clientServer/scripts/server/afx/cache/afx_datablock_cache.dbc"; // If the password is set, clients must provide it in order // to connect to the server diff --git a/Templates/BaseGame/game/core/clientServer/scripts/server/levelDownload.cs b/Templates/BaseGame/game/core/clientServer/scripts/server/levelDownload.cs index de2f5e500..c5777f8fa 100644 --- a/Templates/BaseGame/game/core/clientServer/scripts/server/levelDownload.cs +++ b/Templates/BaseGame/game/core/clientServer/scripts/server/levelDownload.cs @@ -38,8 +38,26 @@ //---------------------------------------------------------------------------- // Phase 1 //---------------------------------------------------------------------------- +$Pref::Server::EnableDatablockCache = true; +$pref::Server::DatablockCacheFilename = "data/cache/server/datablock_cache_c.dbc"; function GameConnection::loadMission(%this) { + %cache_crc = ""; + + if ($Pref::Server::EnableDatablockCache) + { + if (!isDatablockCacheSaved()) + { + echo("<<<< saving server datablock cache >>>>"); + %this.saveDatablockCache(); + } + + if (isFile($Pref::Server::DatablockCacheFilename)) + { + %cache_crc = getDatablockCacheCRC(); + echo(" <<<< sending CRC to client:" SPC %cache_crc SPC ">>>>"); + } + } // Send over the information that will display the server info // when we learn it got there, we'll send the data blocks %this.currentPhase = 0; @@ -50,12 +68,41 @@ function GameConnection::loadMission(%this) } else { - commandToClient(%this, 'MissionStartPhase1', $missionSequence, $Server::MissionFile); + commandToClient(%this, 'MissionStartPhase1', $missionSequence, $Server::MissionFile, %cache_crc); echo("*** Sending mission load to client: " @ $Server::MissionFile); } } +function serverCmdMissionStartPhase1Ack_UseCache(%client, %seq) +{ + echo("<<<< client will load datablocks from a cache >>>>"); + echo(" <<<< skipping datablock transmission >>>>"); + + // Make sure to ignore calls from a previous mission load + if (%seq != $missionSequence || !$MissionRunning) + return; + if (%client.currentPhase != 0) + return; + %client.currentPhase = 1; + + // Start with the CRC + %client.setMissionCRC( $missionCRC ); + + %client.onBeginDatablockCacheLoad($missionSequence); +} + +function GameConnection::onBeginDatablockCacheLoad( %this, %missionSequence ) +{ + // Make sure to ignore calls from a previous mission load + if (%missionSequence != $missionSequence) + return; + if (%this.currentPhase != 1) + return; + %this.currentPhase = 1.5; + commandToClient(%this, 'MissionStartPhase1_LoadCache', $missionSequence, $Server::MissionFile); +} + function serverCmdMissionStartPhase1Ack(%client, %seq) { // Make sure to ignore calls from a previous mission load @@ -157,14 +204,6 @@ function serverCmdMissionStartPhase3Ack(%client, %seq) // Set the control object to the default camera if (!isObject(%client.camera)) { - if(!isObject(Observer)) - { - datablock CameraData(Observer) - { - mode = "Observer"; - }; - } - //if (isDefined("$Game::DefaultCameraClass")) %client.camera = spawnObject("Camera", Observer); } diff --git a/Templates/BaseGame/game/core/clientServer/scripts/server/server.cs b/Templates/BaseGame/game/core/clientServer/scripts/server/server.cs index 5663d8bc6..0c6c680af 100644 --- a/Templates/BaseGame/game/core/clientServer/scripts/server/server.cs +++ b/Templates/BaseGame/game/core/clientServer/scripts/server/server.cs @@ -204,10 +204,11 @@ function onServerCreated() loadDatablockFiles( DatablockFilesList, true ); + callOnModules("onServerScriptExec", "Core"); + callOnModules("onServerScriptExec", "Game"); + // Keep track of when the game started $Game::StartTime = $Sim::Time; - - onServerCreatedAFX(); } /// Shut down the server @@ -283,6 +284,9 @@ function onServerDestroyed() MissionCleanup.delete(); clearServerPaths(); + + if ($Pref::Server::EnableDatablockCache) + resetDatablockCache(); } /// Guid list maintenance functions diff --git a/Templates/BaseGame/game/core/utility/scripts/module.cs b/Templates/BaseGame/game/core/utility/scripts/module.cs index 2ff14c1e1..7d6cc5eed 100644 --- a/Templates/BaseGame/game/core/utility/scripts/module.cs +++ b/Templates/BaseGame/game/core/utility/scripts/module.cs @@ -1,5 +1,12 @@ +$traceModuleCalls=false; +$reportModuleFileConflicts=true; +if (!isObject(ExecFilesList)) + new ArrayObject(ExecFilesList); + function callOnModules(%functionName, %moduleGroup) { + //clear per module group file execution chain + ExecFilesList.empty(); //Get our modules so we can exec any specific client-side loading/handling %modulesList = ModuleDatabase.findModules(false); for(%i=0; %i < getWordCount(%modulesList); %i++) @@ -16,7 +23,14 @@ function callOnModules(%functionName, %moduleGroup) { eval(%module.scopeSet @ "." @ %functionName @ "();"); } - } + } + + %execFilecount = ExecFilesList.count(); + for (%i=0;%i<%execFilecount;%i++) + { + %filename = ExecFilesList.getKey(%i); + exec(%filename); + } } function loadModuleMaterials(%moduleGroup) @@ -69,10 +83,12 @@ function SimSet::getModulePath(%scopeSet) return ""; } -function SimSet::registerDatablock(%scopeSet, %datablockFilePath) +function SimSet::registerDatablock(%scopeSet, %datablockFilePath, %isExclusive) { + if ($traceModuleCalls) + warn("SimSet::registerDatablock"); %name = %scopeSet.getName(); - %moduleDef = ModuleDatabase.findModule(%name); + %moduleDef = ModuleDatabase.findModule(%name, 1); if(!isObject(%moduleDef)) { @@ -89,6 +105,176 @@ function SimSet::registerDatablock(%scopeSet, %datablockFilePath) %relativePath = makeRelativePath(%datablockFilePath); %fullPath = pathConcat(%moduleDef.ModulePath, %relativePath); + ///go through all entries + %locked = false; + %dbFilecount = DatablockFilesList.count(); + for (%i=0;%i<%dbFilecount;%i++) + { + %check = DatablockFilesList.getKey(%i); + //look for a substring match + %isMatch = strIsMatchExpr("*"@ %datablockFilePath,%check ); + if (%isMatch) + { + //check if we're already locked in + //and kill off any duplicates + //do note that doing it in this order means setting exclusive twice + //allows one to override exclusive with exclusive + %locked = DatablockFilesList.getValue(%i); + + if ((!%locked && !%isExclusive)&&($reportModuleFileConflicts)) + error("found" SPC %datablockFilePath SPC "duplicate file!"); + if (!%locked || (%locked && %isExclusive)) + { + DatablockFilesList.erase(%i); + } + } + } + //if we're not locked, or we are exclusive, go ahead and add it to the pile + //(ensures exclusives get re-added after that erasure) + if (!%locked || %isExclusive) + DatablockFilesList.add(%fullPath,%isExclusive); + if ($traceModuleCalls) + DatablockFilesList.echo(); +} + +function SimSet::unRegisterDatablock(%scopeSet, %datablockFilePath) +{ + if ($traceModuleCalls) + warn("SimSet::unRegisterDatablock"); + %name = %scopeSet.getName(); + %moduleDef = ModuleDatabase.findModule(%name, 1); + + if(!isObject(%moduleDef)) + { + error("Module::unRegisterDatablock() - unable to find a module with the moduleID of " @ %name); + return; + } - DatablockFilesList.add(%fullPath); + if(!isObject(DatablockFilesList)) + { + error("Module::unRegisterDatablock() - DatablockFilesList array object doesn't exist!"); + return; + } + + %relativePath = makeRelativePath(%datablockFilePath); + + %fullPath = pathConcat(%moduleDef.ModulePath, %relativePath); + ///go through all entries + %locked = false; + %dbFilecount = DatablockFilesList.count(); + for (%i=0;%i<%dbFilecount;%i++) + { + %check = DatablockFilesList.getKey(%i); + //look for a substring match + %isMatch = strIsMatchExpr("*"@ %datablockFilePath,%check ); + if (%isMatch) + { + //check if we're already locked in. if not, kill it. + %locked = DatablockFilesList.getValue(%i); + if (!%locked) + { + DatablockFilesList.erase(%i); + } + } + } + if ($traceModuleCalls) + DatablockFilesList.echo(); +} + +function SimSet::queueExec(%scopeSet, %execFilePath, %isExclusive) +{ + if ($traceModuleCalls) + warn("SimSet::queueExec"); + %name = %scopeSet.getName(); + %moduleDef = ModuleDatabase.findModule(%name, 1); + + if(!isObject(%moduleDef)) + { + error("Module::queueExec() - unable to find a module with the moduleID of " @ %name); + return; + } + + if(!isObject(ExecFilesList)) + { + error("Module::queueExec() - ExecFilesList array object doesn't exist!"); + return; + } + + if ($traceModuleCalls) + warn("module root path="@ makeRelativePath(%moduleDef.ModulePath)); + + %fullPath = makeRelativePath(%moduleDef.ModulePath) @ %execFilePath; + ///go through all entries + %locked = false; + %execFilecount = ExecFilesList.count(); + for (%i=0;%i<%execFilecount;%i++) + { + %check = ExecFilesList.getKey(%i); + //look for a substring match + %isMatch = strIsMatchExpr("*"@ %execFilePath,%check ); + if (%isMatch) + { + //check if we're already locked in + //and kill off any duplicates + //do note that doing it in this order means setting exclusive twice + //allows one to override exclusive with exclusive + %locked = ExecFilesList.getValue(%i); + if ((!%locked && !%isExclusive)&&($reportModuleFileConflicts)) + error("found" SPC %execFilePath SPC "duplicate file!"); + if (!%locked || (%locked && %isExclusive)) + { + ExecFilesList.erase(%i); + } + } + } + //if we're not locked, or we are exclusive, go ahead and add it to the pile + //(ensures exclusives get re-added after that erasure) + if (!%locked || %isExclusive) + ExecFilesList.add(%fullPath,%isExclusive); + if ($traceModuleCalls) + ExecFilesList.echo(); +} + +function SimSet::unQueueExec(%scopeSet, %execFilePath) +{ + if ($traceModuleCalls) + warn("SimSet::unRegisterDatablock"); + %name = %scopeSet.getName(); + %moduleDef = ModuleDatabase.findModule(%name, 1); + + if(!isObject(%moduleDef)) + { + error("Module::unRegisterDatablock() - unable to find a module with the moduleID of " @ %name); + return; + } + + if(!isObject(ExecFilesList)) + { + error("Module::unRegisterDatablock() - ExecFilesList array object doesn't exist!"); + return; + } + + %relativePath = makeRelativePath(%execFilePath); + + %fullPath = pathConcat(%moduleDef.ModulePath, %relativePath); + ///go through all entries + %locked = false; + %execFilecount = ExecFilesList.count(); + for (%i=0;%i<%execFilecount;%i++) + { + %check = ExecFilesList.getKey(%i); + //look for a substring match + %isMatch = strIsMatchExpr("*"@ %execFilePath,%check ); + if (%isMatch) + { + //check if we're already locked in. if not, kill it. + %locked = ExecFilesList.getValue(%i); + if (!%locked) + { + ExecFilesList.erase(%i); + } + } + } + if ($traceModuleCalls) + ExecFilesList.echo(); } \ No newline at end of file