Test PR of lazy load of ghosts and scene streaming

For now scene streaming only affects how subscenes are loaded. If you try it on everything in a scene expect issues as the namespaces are not linked up at that point.

This is a TEST! Do not merge! If you do you will open up a gate to an alternate reality where all that you believe will cease to exist in an endless void of darkness.

Engine/source/T3D/SubScene.cpp

@@ -25,6 +25,13 @@ IMPLEMENT_CALLBACK(SubScene, onLoaded, void, (), (),
 IMPLEMENT_CALLBACK(SubScene, onUnloaded, void, (), (),
    "@brief Called when a subScene has been unloaded and has game mode implications.\n\n");
 
+namespace Sim
+{
+   // Defined in simManager.cpp
+   extern SceneStreaming* sgStreamingInstance;
+}
+
+
 SubScene::SubScene() :
    mSubSceneAssetId(StringTable->EmptyString()),
    mGameModesNames(StringTable->EmptyString()),
@@ -360,10 +367,12 @@ void SubScene::_loadFile(bool addFileNotify)
 
    String evalCmd = String::ToString("exec(\"%s\");", mSubSceneAsset->getLevelPath());
 
+   Sim::sgStreamingInstance->smStreaming = true;
    String instantGroup = Con::getVariable("InstantGroup");
    Con::setIntVariable("InstantGroup", this->getId());
    Con::evaluate((const char*)evalCmd.c_str(), false, mSubSceneAsset->getLevelPath());
    Con::setVariable("InstantGroup", instantGroup.c_str());
+   Sim::sgStreamingInstance->smStreaming = false;
 
    if (addFileNotify)
       Torque::FS::AddChangeNotification(mSubSceneAsset->getLevelPath(), this, &SubScene::_onFileChanged);

Engine/source/T3D/gameBase/gameBase.cpp

@@ -575,6 +575,64 @@ void GameBase::readPacketData(GameConnection*, BitStream*)
 {
 }
 
+U32 GameBase::partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream)
+{
+   U32 retMask = Parent::partialPackUpdate(conn, mask, stream);
+
+   if (stream->writeFlag(mask & ScaleMask))
+   {
+      // Only write one bit if the scale is one.
+      if (stream->writeFlag(mObjScale != Point3F::One))
+         mathWrite(*stream, mObjScale);
+
+      retMask &= ~ScaleMask;
+   }
+
+   if (stream->writeFlag((mask & DataBlockMask) && mDataBlock != NULL))
+   {
+      stream->writeRangedU32(mDataBlock->getId(),
+         DataBlockObjectIdFirst,
+         DataBlockObjectIdLast);
+      if (stream->writeFlag(mNetFlags.test(NetOrdered)))
+         stream->writeInt(mOrderGUID, 16);
+
+      retMask &= ~DataBlockMask;
+   }
+
+   return retMask;
+}
+
+void GameBase::partialUnpackUpdate(NetConnection* conn, BitStream* stream)
+{
+   Parent::partialUnpackUpdate(conn, stream);
+
+   // ScaleMask
+   if (stream->readFlag())
+   {
+      if (stream->readFlag())
+      {
+         VectorF scale;
+         mathRead(*stream, &scale);
+         setScale(scale);
+      }
+      else
+         setScale(Point3F::One);
+   }
+
+   // DataBlockMask
+   if (stream->readFlag())
+   {
+      GameBaseData* dptr = 0;
+      SimObjectId id = stream->readRangedU32(DataBlockObjectIdFirst,
+         DataBlockObjectIdLast);
+      if (stream->readFlag())
+         mOrderGUID = stream->readInt(16);
+
+      if (!Sim::findObject(id, dptr) || !setDataBlock(dptr))
+         conn->setLastError("Invalid packet GameBase::unpackUpdate()");
+   }
+}
+
 U32 GameBase::packUpdate( NetConnection *connection, U32 mask, BitStream *stream )
 {
    U32 retMask = Parent::packUpdate( connection, mask, stream );

Engine/source/T3D/gameBase/gameBase.h

@@ -363,6 +363,8 @@ public:
    /// @{
    void interpolateTick(F32 dt) override;
    F32  getUpdatePriority( CameraScopeQuery *focusObject, U32 updateMask, S32 updateSkips ) override;
+   U32 partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream) override;
+   void partialUnpackUpdate(NetConnection* conn, BitStream* stream) override;
    U32  packUpdate  ( NetConnection *conn, U32 mask, BitStream *stream ) override;
    void unpackUpdate( NetConnection *conn,           BitStream *stream ) override;
 

Engine/source/T3D/tsStatic.cpp

@@ -942,6 +942,86 @@ void TSStatic::setTransform(const MatrixF& mat)
    setRenderTransform(mat);
 }
 
+U32 TSStatic::partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream)
+{
+   U32 retMask = Parent::partialPackUpdate(conn,mask,stream);
+
+   if (stream->writeFlag(mask & TransformMask))
+   {
+      mathWrite(*stream, getTransform());
+      retMask &= ~TransformMask;
+   }
+
+   if (stream->writeFlag(mask & AdvancedStaticOptionsMask))
+   {
+      PACK_ASSET_REFACTOR(conn, Shape);
+
+      stream->writeInt(mDecalType, 4);
+
+      stream->writeFlag(mAllowPlayerStep);
+      stream->writeFlag(mMeshCulling);
+      stream->writeFlag(mUseOriginSort);
+
+      stream->write(mRenderNormalScalar);
+
+      stream->write(mForceDetail);
+
+      if (stream->writeFlag(mPlayAmbient && hasAnim()))
+      {
+         if (stream->writeFlag(mAnimOffset != 0.0f))
+            stream->writeFloat(mAnimOffset, 7);
+
+         if (stream->writeFlag(mAnimSpeed != 1.0f))
+            stream->writeSignedFloat(mAnimSpeed / AnimSpeedMax, 7);
+      }
+
+      retMask &= ~AdvancedStaticOptionsMask;
+   }
+
+   return retMask;
+}
+
+void TSStatic::partialUnpackUpdate(NetConnection* conn, BitStream* stream)
+{
+   Parent::partialUnpackUpdate(conn, stream);
+
+   if (stream->readFlag()) // TransformMask
+   {
+      MatrixF mat;
+      mathRead(*stream, &mat);
+      setTransform(mat);
+      setRenderTransform(mat);
+   }
+
+   if (stream->readFlag()) // AdvancedStaticOptionsMask
+   {
+      UNPACK_ASSET_REFACTOR(conn, Shape);
+
+      mDecalType = (MeshType)stream->readInt(4);
+
+      mAllowPlayerStep = stream->readFlag();
+      mMeshCulling = stream->readFlag();
+      mUseOriginSort = stream->readFlag();
+
+      stream->read(&mRenderNormalScalar);
+
+      stream->read(&mForceDetail);
+
+      mPlayAmbient = stream->readFlag();
+      if (mPlayAmbient)
+      {
+         if (stream->readFlag())
+            mAnimOffset = stream->readFloat(7);
+
+         if (stream->readFlag())
+            mAnimSpeed = stream->readSignedFloat(7) * AnimSpeedMax;
+      }
+
+      //update our shape, figuring that it likely changed
+      _createShape();
+   }
+}
+
 U32 TSStatic::packUpdate(NetConnection* con, U32 mask, BitStream* stream)
 {
    U32 retMask = Parent::packUpdate(con, mask, stream);

Engine/source/T3D/tsStatic.h

@@ -254,6 +254,8 @@ public:
    void reSkin();
 
    // NetObject
+   U32 partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream) override;
+   void partialUnpackUpdate(NetConnection* conn, BitStream* stream) override;
    U32 packUpdate(NetConnection* conn, U32 mask, BitStream* stream) override;
    void unpackUpdate(NetConnection* conn, BitStream* stream) override;
 

Engine/source/console/sim.cpp

@@ -266,3 +266,18 @@ DefineEngineFunction( isValidObjectName, bool, (const char * name), , "( string
 }
 
 ConsoleFunctionGroupEnd( SimFunctions );
+
+void SceneStreaming::processTick()
+{
+   if (smStreaming)
+   {
+      for (U32 i = 0; i < mMaxObjects && !smPendingRegister.empty(); i++)
+      {
+         SimObject* obj = smPendingRegister.first();
+         smPendingRegister.pop_front();
+
+         if (!obj->onAdd())
+            obj->unregisterObject();
+      }
+   }
+}

Engine/source/console/sim.h

@@ -35,6 +35,9 @@
 #ifndef _CONSOLE_H_
 #include "console/console.h"
 #endif
+#ifndef _ITICKABLE_H_
+#include "core/iTickable.h"
+#endif
 
 // Forward Refs
 class SimSet;
@@ -44,6 +47,21 @@ class SimObject;
 class SimEvent;
 class Stream;
 
+class SceneStreaming : public virtual ITickable
+{
+protected:
+   U32 mMaxObjects = 20;
+public:
+   SceneStreaming(){}
+   ~SceneStreaming() {}
+    bool smStreaming;
+   Vector<SimObject*> smPendingRegister;
+
+   void interpolateTick(F32 delta) override {return;}
+   void processTick() override;
+   void advanceTime(F32 timeDelta) override { return; }
+};
+
 // Sim Types
 typedef U32 SimTime;
 typedef U32 SimObjectId;

Engine/source/console/simManager.cpp

@@ -280,6 +280,7 @@ U32 getTargetTime()
 
 SimGroup *gRootGroup = NULL;
 SimManagerNameDictionary *gNameDictionary;
+SceneStreaming* sgStreamingInstance = NULL;
 SimIdDictionary *gIdDictionary;
 U32 gNextObjectId;
 
@@ -287,7 +288,8 @@ static void initRoot()
 {
    gIdDictionary = new SimIdDictionary;
    gNameDictionary = new SimManagerNameDictionary;
-
+   sgStreamingInstance = new SceneStreaming;
+   sgStreamingInstance->smStreaming = false;
    gRootGroup = new SimGroup();
    gRootGroup->incRefCount();
  
@@ -305,6 +307,7 @@ static void shutdownRoot()
       gRootGroup->deleteObject();
    gRootGroup = NULL;
 
+   SAFE_DELETE(sgStreamingInstance);
    SAFE_DELETE(gNameDictionary);
    SAFE_DELETE(gIdDictionary);
 }

Engine/source/console/simObject.cpp

@@ -43,6 +43,7 @@
 #include "console/script.h"
 
 #include "sim/netObject.h"
+#include "scene/sceneObject.h"
 
 ImplementBitfieldType(GameTypeMasksType,
    "The type of animation effect to apply to this material.\n"
@@ -109,6 +110,7 @@ namespace Sim
    // Defined in simManager.cpp
    extern SimGroup *gRootGroup;
    extern SimManagerNameDictionary *gNameDictionary;
+   extern SceneStreaming* sgStreamingInstance;
    extern SimIdDictionary *gIdDictionary;
    extern U32 gNextObjectId;
 }
@@ -725,6 +727,12 @@ bool SimObject::registerObject()
 
    Sim::gNameDictionary->insert(this);
 
+   if (Sim::sgStreamingInstance->smStreaming && dynamic_cast<SceneObject*>(this))
+   {
+      Sim::sgStreamingInstance->smPendingRegister.push_back(this);
+      return true; // pretend success
+   }
+
    // Notify object
    bool ret = onAdd();
 

Engine/source/console/simObject.h

@@ -40,7 +40,10 @@
 #ifndef _TAML_CALLBACKS_H_
 #include "persistence/taml/tamlCallbacks.h"
 #endif
+
+#ifndef _OBJECTTYPES_H_
 #include "T3D/objectTypes.h"
+#endif
 
 class Stream;
 class LightManager;

Engine/source/scene/sceneObject.cpp

@@ -898,6 +898,28 @@ bool SceneObject::_setSelectionEnabled( void *object, const char *index, const c
 
 //--------------------------------------------------------------------------
 
+U32 SceneObject::partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream)
+{
+   U32 retMask = Parent::partialPackUpdate(conn, mask, stream);
+
+   if (stream->writeFlag(mask & FlagMask))
+   {
+      stream->writeRangedU32((U32)mObjectFlags, 0, getObjectFlagMax());
+      retMask &= ~FlagMask;
+   }
+
+   return retMask;
+}
+
+void SceneObject::partialUnpackUpdate(NetConnection* conn, BitStream* stream)
+{
+   Parent::unpackUpdate(conn, stream);
+
+   // FlagMask
+   if (stream->readFlag())
+      mObjectFlags = stream->readRangedU32(0, getObjectFlagMax());
+}
+
 U32 SceneObject::packUpdate( NetConnection* conn, U32 mask, BitStream* stream )
 {
    U32 retMask = Parent::packUpdate( conn, mask, stream );

Engine/source/scene/sceneObject.h

@@ -738,6 +738,8 @@ class SceneObject : public NetObject, public ProcessObject
       void setProcessTick( bool t ) override;
 
       // NetObject.
+      U32 partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream) override;
+      void partialUnpackUpdate(NetConnection* conn, BitStream* stream) override;
       U32 packUpdate( NetConnection* conn, U32 mask, BitStream* stream ) override;
       void unpackUpdate( NetConnection* conn, BitStream* stream ) override;
       void onCameraScopeQuery( NetConnection* connection, CameraScopeQuery* query ) override;

Engine/source/sim/netConnection.cpp

@@ -215,7 +215,7 @@ U32 NetConnection::getSequence()
 static U32 gPacketRateToServer = 32;
 static U32 gPacketUpdateDelayToServer = 32;
 static U32 gPacketRateToClient = 10;
-static U32 gPacketSize = 508;
+static U32 gPacketSize = 1500;
 
 void NetConnection::consoleInit()
 {
@@ -319,7 +319,7 @@ void NetConnection::checkMaxRate()
       // These changes introduced in T3D 1.1 Preview reduce the packet headroom which leads
       // to some spells and effects running out of room when dynamic variables are used
       // to send launch-time parameters to clients.
-      packetSize = 512;
+      packetSize = 1500;
    }
 
    gPacketUpdateDelayToServer = 1024 / packetRateToServer;
@@ -440,6 +440,9 @@ NetConnection::NetConnection()
 
    // Ensure NetAddress is cleared
    dMemset(&mNetAddress, '\0', sizeof(NetAddress));
+
+   mGhostByteBudget = 800; // test payload.
+   mGhostByteRemainder = mGhostByteBudget;
 }
 
 NetConnection::~NetConnection()

Engine/source/sim/netConnection.h

@@ -556,6 +556,8 @@ private:
    NetConnection *mNextTableHash;
    static NetConnection *mHashTable[HashTableSize];
 
+   U32 mGhostByteBudget;
+   U32 mGhostByteRemainder;
    /// @}
 
 protected:

Engine/source/sim/netGhost.cpp

@@ -424,16 +424,43 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 
    bstream->writeInt(sendSize - 3, GhostIndexBitSize);
 
-   U32 count = 0;
+   S32 bytesThisPacket = 0;
+   U32 maxBytesPerPacket = mGhostByteBudget;
    //
    for(i = mGhostZeroUpdateIndex - 1; i >= 0 && !bstream->isFull(); i--)
    {
 	   walk = mGhostArray[i];
 		if(walk->flags & (GhostInfo::KillingGhost | GhostInfo::Ghosting))
 		   continue;
-		
-      bstream->writeFlag(true);
 
+      // Optional: skip low-priority objects if needed
+      if (walk->updateSkipCount == 0 && walk->updateMask == 0)
+         continue;
+		
+      U32 ghostSize = 0;
+
+      // If not-yet-ghosted, include the object's class ID and update mask overhead
+      if (walk->flags & GhostInfo::NotYetGhosted)
+         ghostSize += sizeof(U32) * 2; // rough estimate for class ID + flags
+
+      // Add actual object size
+      if (walk->obj)
+      {
+         U32 estimatedNetSize = walk->obj->getClassRep()->getSizeof() / 32; // rough estimate for object data
+         ghostSize += estimatedNetSize;
+      }
+
+      // Check against byte budget
+      if (bytesThisPacket + ghostSize > maxBytesPerPacket)
+      {
+         // stop here — packet is full
+         break;
+      }
+
+      bytesThisPacket += ghostSize;
+
+      // Existing code to write this ghost:
+      bstream->writeFlag(true);
       bstream->writeInt(walk->index, sendSize);
       U32 updateMask = walk->updateMask;
 
@@ -463,6 +490,7 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 #ifdef TORQUE_DEBUG_NET
          U32 startPos = bstream->getCurPos();
 #endif
+         U32 retMask = 0;
          if(walk->flags & GhostInfo::NotYetGhosted)
          {
             S32 classId = walk->obj->getClassId(getNetClassGroup());
@@ -470,26 +498,14 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 #ifdef TORQUE_DEBUG_NET
             bstream->writeInt(classId ^ DebugChecksum, 32);
 #endif
-
+            retMask = walk->obj->partialPackUpdate(this, updateMask, bstream);
             walk->flags &= ~GhostInfo::NotYetGhosted;
             walk->flags |= GhostInfo::Ghosting;
             upd->ghostInfoFlags = GhostInfo::Ghosting;
          }
-#ifdef TORQUE_DEBUG_NET
+         else{
-         else {
+            retMask = walk->obj->packUpdate(this, updateMask, bstream);
-            S32 classId = walk->obj->getClassId(getNetClassGroup());
-            bstream->writeClassId(classId, NetClassTypeObject, getNetClassGroup());
-            bstream->writeInt(classId ^ DebugChecksum, 32);
          }
-#endif
-         // update the object
-#ifdef TORQUE_NET_STATS
-         U32 beginSize = bstream->getBitPosition();
-#endif
-         U32 retMask = walk->obj->packUpdate(this, updateMask, bstream);
-#ifdef TORQUE_NET_STATS
-         walk->obj->getClassRep()->updateNetStatPack(updateMask, bstream->getBitPosition() - beginSize);
-#endif
          DEBUG_LOG(("PKLOG %d GHOST %d: %s", getId(), bstream->getBitPosition() - 16 - startPos, walk->obj->getClassName()));
 
          AssertFatal((retMask & (~updateMask)) == 0, "Cannot set new bits in packUpdate return");
@@ -508,7 +524,6 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 #endif
       }
       walk->updateSkipCount = 0;
-      count++;
    }
    //Con::printf("Ghosts updated: %d (%d remain)", count, mGhostZeroUpdateIndex);
    // no more objects...
@@ -545,7 +560,7 @@ void NetConnection::ghostReadPacket(BitStream *bstream)
       index = (U32) bstream->readInt(idSize);
       if(bstream->readFlag()) // is this ghost being deleted?
       {
-		 mGhostsActive--;
+		   mGhostsActive--;
          AssertFatal(mLocalGhosts[index] != NULL, "Error, NULL ghost encountered.");
          mLocalGhosts[index]->deleteObject();
          mLocalGhosts[index] = NULL;
@@ -589,13 +604,8 @@ void NetConnection::ghostReadPacket(BitStream *bstream)
             // give derived classes a chance to prepare ghost for reading
             ghostPreRead(mLocalGhosts[index],true);
 
-#ifdef TORQUE_NET_STATS
+            mLocalGhosts[index]->partialUnpackUpdate(this, bstream);
-            U32 beginSize = bstream->getBitPosition();
+
-#endif
-            mLocalGhosts[index]->unpackUpdate(this, bstream);
-#ifdef TORQUE_NET_STATS
-            mLocalGhosts[index]->getClassRep()->updateNetStatUnpack(bstream->getBitPosition() - beginSize);
-#endif
             // Setup the remote object pointers before
             // we register so that it can be used from onAdd.
             if( mRemoteConnection )
@@ -637,14 +647,7 @@ void NetConnection::ghostReadPacket(BitStream *bstream)
 #endif
             // give derived classes a chance to prepare ghost for reading
             ghostPreRead(mLocalGhosts[index],false);
-
-#ifdef TORQUE_NET_STATS
-            U32 beginSize = bstream->getBitPosition();
-#endif
             mLocalGhosts[index]->unpackUpdate(this, bstream);
-#ifdef TORQUE_NET_STATS
-            mLocalGhosts[index]->getClassRep()->updateNetStatUnpack(bstream->getBitPosition() - beginSize);
-#endif
             ghostReadExtra(mLocalGhosts[index],bstream,false);
          }
          //PacketStream::getStats()->addBits(PacketStats::Receive, bstream->getCurPos() - startPos, ghostRefs[index].localGhost->getPersistTag());

Engine/source/sim/netObject.cpp

@@ -328,6 +328,15 @@ F32 NetObject::getUpdatePriority(CameraScopeQuery*, U32, S32 updateSkips)
    return F32(updateSkips) * 0.1;
 }
 
+U32 NetObject::partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream)
+{
+    return mask;
+}
+
+void NetObject::partialUnpackUpdate(NetConnection*, BitStream*)
+{
+}
+
 U32 NetObject::packUpdate(NetConnection* conn, U32 mask, BitStream* stream)
 {
    return 0;

Engine/source/sim/netObject.h

@@ -304,6 +304,12 @@ public:
    /// @param   orMask   Bit(s) to set
    virtual void setMaskBits(U32 orMask);
 
+   /// <summary>
+   /// Used for ghosting to figure out what we can send.
+   /// </summary>
+   /// <returns>The network size of the data being sent for a full update.</returns>
+   virtual U32 getNetSize() const { return 900; }
+
    /// Clear the specified bits from the dirty mask.
    ///
    /// @param   orMask   Bits to clear
@@ -337,6 +343,22 @@ public:
    /// @returns A floating point value indicating priority. These are typically < 5.0.
    virtual F32 getUpdatePriority(CameraScopeQuery *focusObject, U32 updateMask, S32 updateSkips);
 
+   /// Instructs this object to pack its state for transfer over the network.
+   ///
+   /// @param   conn    Net connection being used
+   /// @param   mask    Mask indicating fields to transmit.
+   /// @param   stream  Bitstream to pack data to
+   ///
+   /// @returns Any bits which were not dealt with. The value is stored by the networking
+   ///          system. Don't set bits you weren't passed.
+   virtual U32  partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream);
+
+   /// Instructs this object to read state data previously packed with packUpdate.
+   ///
+   /// @param   conn    Net connection being used
+   /// @param   stream  stream to read from
+   virtual void partialUnpackUpdate(NetConnection* conn, BitStream* stream);
+
    /// Instructs this object to pack its state for transfer over the network.
    ///
    /// @param   conn    Net connection being used