Test PR of lazy load of ghosts and scene streaming

For now scene streaming only affects how subscenes are loaded. If you try it on everything in a scene expect issues as the namespaces are not linked up at that point.

This is a TEST! Do not merge! If you do you will open up a gate to an alternate reality where all that you believe will cease to exist in an endless void of darkness.

Engine/source/sim/netConnection.cpp

@@ -215,7 +215,7 @@ U32 NetConnection::getSequence()
 static U32 gPacketRateToServer = 32;
 static U32 gPacketUpdateDelayToServer = 32;
 static U32 gPacketRateToClient = 10;
-static U32 gPacketSize = 508;
+static U32 gPacketSize = 1500;
 
 void NetConnection::consoleInit()
 {
@@ -319,7 +319,7 @@ void NetConnection::checkMaxRate()
       // These changes introduced in T3D 1.1 Preview reduce the packet headroom which leads
       // to some spells and effects running out of room when dynamic variables are used
       // to send launch-time parameters to clients.
-      packetSize = 512;
+      packetSize = 1500;
    }
 
    gPacketUpdateDelayToServer = 1024 / packetRateToServer;
@@ -440,6 +440,9 @@ NetConnection::NetConnection()
 
    // Ensure NetAddress is cleared
    dMemset(&mNetAddress, '\0', sizeof(NetAddress));
+
+   mGhostByteBudget = 800; // test payload.
+   mGhostByteRemainder = mGhostByteBudget;
 }
 
 NetConnection::~NetConnection()

Engine/source/sim/netConnection.h

@@ -556,6 +556,8 @@ private:
    NetConnection *mNextTableHash;
    static NetConnection *mHashTable[HashTableSize];
 
+   U32 mGhostByteBudget;
+   U32 mGhostByteRemainder;
    /// @}
 
 protected:

Engine/source/sim/netGhost.cpp

@@ -424,16 +424,43 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 
    bstream->writeInt(sendSize - 3, GhostIndexBitSize);
 
-   U32 count = 0;
+   S32 bytesThisPacket = 0;
+   U32 maxBytesPerPacket = mGhostByteBudget;
    //
    for(i = mGhostZeroUpdateIndex - 1; i >= 0 && !bstream->isFull(); i--)
    {
 	   walk = mGhostArray[i];
 		if(walk->flags & (GhostInfo::KillingGhost | GhostInfo::Ghosting))
 		   continue;
-		
-      bstream->writeFlag(true);
 
+      // Optional: skip low-priority objects if needed
+      if (walk->updateSkipCount == 0 && walk->updateMask == 0)
+         continue;
+		
+      U32 ghostSize = 0;
+
+      // If not-yet-ghosted, include the object's class ID and update mask overhead
+      if (walk->flags & GhostInfo::NotYetGhosted)
+         ghostSize += sizeof(U32) * 2; // rough estimate for class ID + flags
+
+      // Add actual object size
+      if (walk->obj)
+      {
+         U32 estimatedNetSize = walk->obj->getClassRep()->getSizeof() / 32; // rough estimate for object data
+         ghostSize += estimatedNetSize;
+      }
+
+      // Check against byte budget
+      if (bytesThisPacket + ghostSize > maxBytesPerPacket)
+      {
+         // stop here — packet is full
+         break;
+      }
+
+      bytesThisPacket += ghostSize;
+
+      // Existing code to write this ghost:
+      bstream->writeFlag(true);
       bstream->writeInt(walk->index, sendSize);
       U32 updateMask = walk->updateMask;
 
@@ -463,6 +490,7 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 #ifdef TORQUE_DEBUG_NET
          U32 startPos = bstream->getCurPos();
 #endif
+         U32 retMask = 0;
          if(walk->flags & GhostInfo::NotYetGhosted)
          {
             S32 classId = walk->obj->getClassId(getNetClassGroup());
@@ -470,26 +498,14 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 #ifdef TORQUE_DEBUG_NET
             bstream->writeInt(classId ^ DebugChecksum, 32);
 #endif
-
+            retMask = walk->obj->partialPackUpdate(this, updateMask, bstream);
             walk->flags &= ~GhostInfo::NotYetGhosted;
             walk->flags |= GhostInfo::Ghosting;
             upd->ghostInfoFlags = GhostInfo::Ghosting;
          }
-#ifdef TORQUE_DEBUG_NET
-         else {
-            S32 classId = walk->obj->getClassId(getNetClassGroup());
-            bstream->writeClassId(classId, NetClassTypeObject, getNetClassGroup());
-            bstream->writeInt(classId ^ DebugChecksum, 32);
+         else{
+            retMask = walk->obj->packUpdate(this, updateMask, bstream);
          }
-#endif
-         // update the object
-#ifdef TORQUE_NET_STATS
-         U32 beginSize = bstream->getBitPosition();
-#endif
-         U32 retMask = walk->obj->packUpdate(this, updateMask, bstream);
-#ifdef TORQUE_NET_STATS
-         walk->obj->getClassRep()->updateNetStatPack(updateMask, bstream->getBitPosition() - beginSize);
-#endif
          DEBUG_LOG(("PKLOG %d GHOST %d: %s", getId(), bstream->getBitPosition() - 16 - startPos, walk->obj->getClassName()));
 
          AssertFatal((retMask & (~updateMask)) == 0, "Cannot set new bits in packUpdate return");
@@ -508,7 +524,6 @@ void NetConnection::ghostWritePacket(BitStream *bstream, PacketNotify *notify)
 #endif
       }
       walk->updateSkipCount = 0;
-      count++;
    }
    //Con::printf("Ghosts updated: %d (%d remain)", count, mGhostZeroUpdateIndex);
    // no more objects...
@@ -545,7 +560,7 @@ void NetConnection::ghostReadPacket(BitStream *bstream)
       index = (U32) bstream->readInt(idSize);
       if(bstream->readFlag()) // is this ghost being deleted?
       {
-		 mGhostsActive--;
+		   mGhostsActive--;
          AssertFatal(mLocalGhosts[index] != NULL, "Error, NULL ghost encountered.");
          mLocalGhosts[index]->deleteObject();
          mLocalGhosts[index] = NULL;
@@ -589,13 +604,8 @@ void NetConnection::ghostReadPacket(BitStream *bstream)
             // give derived classes a chance to prepare ghost for reading
             ghostPreRead(mLocalGhosts[index],true);
 
-#ifdef TORQUE_NET_STATS
-            U32 beginSize = bstream->getBitPosition();
-#endif
-            mLocalGhosts[index]->unpackUpdate(this, bstream);
-#ifdef TORQUE_NET_STATS
-            mLocalGhosts[index]->getClassRep()->updateNetStatUnpack(bstream->getBitPosition() - beginSize);
-#endif
+            mLocalGhosts[index]->partialUnpackUpdate(this, bstream);
+
             // Setup the remote object pointers before
             // we register so that it can be used from onAdd.
             if( mRemoteConnection )
@@ -637,14 +647,7 @@ void NetConnection::ghostReadPacket(BitStream *bstream)
 #endif
             // give derived classes a chance to prepare ghost for reading
             ghostPreRead(mLocalGhosts[index],false);
-
-#ifdef TORQUE_NET_STATS
-            U32 beginSize = bstream->getBitPosition();
-#endif
             mLocalGhosts[index]->unpackUpdate(this, bstream);
-#ifdef TORQUE_NET_STATS
-            mLocalGhosts[index]->getClassRep()->updateNetStatUnpack(bstream->getBitPosition() - beginSize);
-#endif
             ghostReadExtra(mLocalGhosts[index],bstream,false);
          }
          //PacketStream::getStats()->addBits(PacketStats::Receive, bstream->getCurPos() - startPos, ghostRefs[index].localGhost->getPersistTag());

Engine/source/sim/netObject.cpp

@@ -328,6 +328,15 @@ F32 NetObject::getUpdatePriority(CameraScopeQuery*, U32, S32 updateSkips)
    return F32(updateSkips) * 0.1;
 }
 
+U32 NetObject::partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream)
+{
+    return mask;
+}
+
+void NetObject::partialUnpackUpdate(NetConnection*, BitStream*)
+{
+}
+
 U32 NetObject::packUpdate(NetConnection* conn, U32 mask, BitStream* stream)
 {
    return 0;

Engine/source/sim/netObject.h

@@ -304,6 +304,12 @@ public:
    /// @param   orMask   Bit(s) to set
    virtual void setMaskBits(U32 orMask);
 
+   /// <summary>
+   /// Used for ghosting to figure out what we can send.
+   /// </summary>
+   /// <returns>The network size of the data being sent for a full update.</returns>
+   virtual U32 getNetSize() const { return 900; }
+
    /// Clear the specified bits from the dirty mask.
    ///
    /// @param   orMask   Bits to clear
@@ -337,6 +343,22 @@ public:
    /// @returns A floating point value indicating priority. These are typically < 5.0.
    virtual F32 getUpdatePriority(CameraScopeQuery *focusObject, U32 updateMask, S32 updateSkips);
 
+   /// Instructs this object to pack its state for transfer over the network.
+   ///
+   /// @param   conn    Net connection being used
+   /// @param   mask    Mask indicating fields to transmit.
+   /// @param   stream  Bitstream to pack data to
+   ///
+   /// @returns Any bits which were not dealt with. The value is stored by the networking
+   ///          system. Don't set bits you weren't passed.
+   virtual U32  partialPackUpdate(NetConnection* conn, U32 mask, BitStream* stream);
+
+   /// Instructs this object to read state data previously packed with packUpdate.
+   ///
+   /// @param   conn    Net connection being used
+   /// @param   stream  stream to read from
+   virtual void partialUnpackUpdate(NetConnection* conn, BitStream* stream);
+
    /// Instructs this object to pack its state for transfer over the network.
    ///
    /// @param   conn    Net connection being used