From 74638c0f2ce4c60d0dbddf9d6c997683949094d6 Mon Sep 17 00:00:00 2001 From: marauder2k7 Date: Tue, 9 Dec 2025 10:04:33 +0000 Subject: [PATCH] Asset data management Added functions to AssetManager to pack/unpack assets using an id hash instead of their id string this cuts down how much data is sent across the network for assets All asset macros updated to use this new functionality. --- Engine/source/T3D/assets/SoundAsset.h | 46 +----- Engine/source/T3D/assets/assetMacroHelpers.h | 142 ++----------------- Engine/source/assets/assetManager.cpp | 78 ++++++++++ Engine/source/assets/assetManager.h | 22 +++ 4 files changed, 115 insertions(+), 173 deletions(-) diff --git a/Engine/source/T3D/assets/SoundAsset.h b/Engine/source/T3D/assets/SoundAsset.h index 2e767de88..1def01066 100644 --- a/Engine/source/T3D/assets/SoundAsset.h +++ b/Engine/source/T3D/assets/SoundAsset.h @@ -536,52 +536,12 @@ if (m##name##AssetId[index] != StringTable->EmptyString())\ } #define PACKDATA_SOUNDASSET_ARRAY(name, index)\ - if (stream->writeFlag(AssetDatabase.isDeclaredAsset(m##name##AssetId[index])))\ - {\ - stream->writeString(m##name##AssetId[index]);\ - }\ - else\ - {\ - if(stream->writeFlag(Sim::findObject(m##name##Name[index])))\ - {\ - SFXTrack* sndTrack = get##name##Profile(index);\ - if(stream->writeFlag(sndTrack != nullptr))\ - {\ - stream->writeRangedU32(SimObjectId(sndTrack->getId()), DataBlockObjectIdFirst, DataBlockObjectIdLast);\ - sfxWrite(stream, sndTrack);\ - }\ - }\ - else\ - {\ - stream->writeString(m##name##Name[index]);\ - }\ - } - + AssetDatabase.packDataAsset(stream, m##name##AssetId[index]) //network recieve - datablock #define UNPACKDATA_SOUNDASSET_ARRAY(name, index)\ - if (stream->readFlag())\ - {\ - m##name##AssetId[index] = stream->readSTString();\ - _set##name(m##name##AssetId[index], index);\ - }\ - else\ - {\ - if(stream->readFlag())\ - {\ - String errorStr;\ - if(stream->readFlag())\ - {\ - m##name##SFXId[index] = stream->readRangedU32( DataBlockObjectIdFirst, DataBlockObjectIdLast );\ - sfxReadAndResolve(stream, &m##name##Profile[index], errorStr);\ - }\ - }\ - else\ - {\ - m##name##Name[index] = stream->readSTString(); \ - _set##name(m##name##Name[index], index); \ - }\ - } + m##name##AssetId[index] = AssetDatabase.unpackDataAsset(stream) + #pragma endregion #endif // _ASSET_BASE_H_ diff --git a/Engine/source/T3D/assets/assetMacroHelpers.h b/Engine/source/T3D/assets/assetMacroHelpers.h index a6d2816e7..ce4bb37d9 100644 --- a/Engine/source/T3D/assets/assetMacroHelpers.h +++ b/Engine/source/T3D/assets/assetMacroHelpers.h @@ -59,122 +59,46 @@ if (m##name##AssetId != StringTable->EmptyString())\ //network send - datablock refactor #define PACKDATA_ASSET_REFACTOR(name)\ - if (stream->writeFlag(m##name##Asset.notNull()))\ - {\ - stream->writeString(m##name##Asset.getAssetId());\ - }\ - else if (stream->writeFlag(m##name##File != StringTable->EmptyString()))\ - {\ - stream->writeString(m##name##File);\ - }\ +AssetDatabase.packDataAsset(stream, m##name##Asset.getAssetId()) //network recieve - datablock #define UNPACKDATA_ASSET_REFACTOR(name)\ - if (stream->readFlag())\ - {\ - _set##name(stream->readSTString());\ - }\ - else if (stream->readFlag())\ - {\ - _set##name(stream->readSTString());\ - }\ - else\ - {\ - _set##name(StringTable->EmptyString());\ - }\ +m##name##Asset = AssetDatabase.unpackDataAsset(stream) //network send - object-instance #define PACK_ASSET_REFACTOR(netconn, name)\ - if (stream->writeFlag(m##name##Asset.notNull()))\ - {\ - NetStringHandle assetIdStr = m##name##Asset.getAssetId();\ - netconn->packNetStringHandleU(stream, assetIdStr);\ - }\ - else if (stream->writeFlag(m##name##File != StringTable->EmptyString()))\ - {\ - NetStringHandle fileStr = m##name##File;\ - netconn->packNetStringHandleU(stream, fileStr);\ - }\ +AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset.getAssetId()) //network recieve - object-instance #define UNPACK_ASSET_REFACTOR(netconn, name)\ - if (stream->readFlag())\ - {\ - _set##name(netconn->unpackNetStringHandleU(stream).getString());\ - }\ - else if (stream->readFlag())\ - {\ - _set##name(netconn->unpackNetStringHandleU(stream).getString());\ - }\ - else\ - {\ - _set##name(StringTable->EmptyString());\ - }\ +_set##name(AssetDatabase.unpackUpdateAsset(netconn, stream)) //network send - datablock #define PACKDATA_ASSET_ARRAY_REFACTOR(name, max)\ for (U32 i = 0; i < max; i++)\ {\ - if (stream->writeFlag(m##name##Asset[i].notNull()))\ - {\ - stream->writeString(m##name##Asset[i].getAssetId()); \ - }\ - else if (stream->writeFlag(m##name##File[i] != StringTable->EmptyString()))\ - {\ - stream->writeString(m##name##File[i]);\ - }\ + AssetDatabase.packDataAsset(stream, m##name##Asset[i].getAssetId());\ } //network recieve - datablock #define UNPACKDATA_ASSET_ARRAY_REFACTOR(name, max)\ for (U32 i = 0; i < max; i++)\ {\ - if (stream->readFlag())\ - {\ - m##name##Asset[i] = stream->readSTString();\ - }\ - else if (stream->readFlag())\ - {\ - _set##name(stream->readSTString(), i);\ - }\ - else\ - {\ - _set##name(StringTable->EmptyString(), i);\ - }\ + m##name##Asset[i] = AssetDatabase.unpackDataAsset(stream);\ } //network send - object-instance #define PACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\ for (U32 i = 0; i < max; i++)\ {\ - if (stream->writeFlag(m##name##Asset[i].notNull()))\ - {\ - NetStringHandle assetIdStr = m##name##Asset[i].getAssetId();\ - netconn->packNetStringHandleU(stream, assetIdStr);\ - }\ - else if (stream->writeFlag(m##name##File[i] != StringTable->EmptyString()))\ - {\ - NetStringHandle fileStr = m##name##File[i];\ - netconn->packNetStringHandleU(stream, fileStr);\ - }\ + AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset[i].getAssetId());\ } //network recieve - object-instance #define UNPACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\ for (U32 i = 0; i < max; i++)\ {\ - if (stream->readFlag())\ - {\ - m##name##Asset[i] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\ - }\ - else if (stream->readFlag())\ - {\ - _set##name(StringTable->insert(netconn->unpackNetStringHandleU(stream).getString()), i);\ - }\ - else\ - {\ - _set##name(StringTable->EmptyString(), i);\ - }\ + _set##name(AssetDatabase.unpackUpdateAsset(netconn, stream), i);\ } #define DEF_ASSET_BINDS_REFACTOR(className,name)\ @@ -348,61 +272,19 @@ if (m##name##AssetId[index] != StringTable->EmptyString())\ } //network send - datablock #define PACKDATA_ASSET_ARRAY(name, index)\ - if (stream->writeFlag(m##name##Asset[index].notNull()))\ - {\ - stream->writeString(m##name##Asset[index].getAssetId());\ - }\ - else if (stream->writeFlag(m##name##Name[index] != StringTable->EmptyString()))\ - {\ - stream->writeString(m##name##Name[index].getAssetId());\ - }\ + AssetDatabase.packDataAsset(stream, m##name##Asset[index].getAssetId()) //network recieve - datablock #define UNPACKDATA_ASSET_ARRAY(name, index)\ - if (stream->readFlag())\ - {\ - m##name##AssetId[index] = stream->readSTString();\ - _set##name(m##name##AssetId[index], index);\ - }\ - else if (stream->readFlag())\ - {\ - m##name##Name[index] = stream->readSTString();\ - _set##name(m##name##Name[index], index);\ - }\ - else\ - {\ - _set##name(StringTable->EmptyString());\ - }\ + m##name##Asset[index] = AssetDatabase.unpackDataAsset(stream) //network send - object-instance #define PACK_ASSET_ARRAY(netconn, name, index)\ - if (stream->writeFlag(m##name##Asset[index].notNull()))\ - {\ - NetStringHandle assetIdStr = m##name##Asset[index].getAssetId();\ - netconn->packNetStringHandleU(stream, assetIdStr);\ - }\ - else if (stream->writeFlag(m##name##Name[index] != StringTable->EmptyString()))\ - {\ - NetStringHandle fileStr = m##name##Name[index].getAssetId();\ - netconn->packNetStringHandleU(stream, fileStr);\ - }\ +AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset[index].getAssetId()); //network recieve - object-instance #define UNPACK_ASSET_ARRAY(netconn, name, index)\ - if (stream->readFlag())\ - {\ - m##name##AssetId[index] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\ - _set##name(m##name##AssetId[index], index);\ - }\ - else if (stream->readFlag())\ - {\ - m##name##Name[index] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\ - _set##name(m##name##Name[index], index);\ - }\ - else\ - {\ - _set##name(StringTable->EmptyString(), index);\ - } +_set##name(AssetDatabase.unpackUpdateAsset(netconn, stream), index); //script methods for class.asset acces //declare general get, getAsset and set methods diff --git a/Engine/source/assets/assetManager.cpp b/Engine/source/assets/assetManager.cpp index 5d471d03f..9f1f603ee 100644 --- a/Engine/source/assets/assetManager.cpp +++ b/Engine/source/assets/assetManager.cpp @@ -964,6 +964,55 @@ bool AssetManager::renameReferencedAsset( const char* pAssetIdFrom, const char* return true; } +void AssetManager::packDataAsset(BitStream* stream, const char* pAssetId) +{ + if(stream->writeFlag(isDeclaredAsset(pAssetId))) + { + stream->write(mAssetToNetId.find(pAssetId)->value); + } +} + +const char* AssetManager::unpackDataAsset(BitStream* stream) +{ + if (stream->readFlag()) + { + typeAssetNetId netId; + stream->read(&netId); + + typeNetIdToAssetMap::iterator netChar = mNetIdToAsset.find(netId); + if (netChar != mNetIdToAsset.end()) + { + return netChar->value; + } + } + + return StringTable->EmptyString(); +} + +void AssetManager::packUpdateAsset(NetConnection* con, U32 mask, BitStream* stream, const char* pAssetId) +{ + if (stream->writeFlag(isDeclaredAsset(pAssetId))) + { + stream->write(mAssetToNetId.find(pAssetId)->value); + } +} + +const char* AssetManager::unpackUpdateAsset(NetConnection* con, BitStream* stream) +{ + if (stream->readFlag()) + { + typeAssetNetId netId; + stream->read(&netId); + + typeNetIdToAssetMap::iterator netChar = mNetIdToAsset.find(netId); + if (netChar != mNetIdToAsset.end()) + { + return netChar->value; + } + } + return StringTable->EmptyString(); +} + bool AssetManager::compileAllAssets(const bool compressed, const bool includeUnloaded) { // Debug Profiling. @@ -2642,6 +2691,17 @@ const char* AssetManager::getAssetLooseFile(const char* pAssetId, const S32& ind //----------------------------------------------------------------------------- +static U32 HashAssetId(const char* str) +{ + U32 hash = 2166136261u; + while (*str) + { + hash ^= (U8)*str++; + hash *= 16777619u; + } + return hash; +} + bool AssetManager::scanDeclaredAssets( const char* pPath, const char* pExtension, const bool recurse, ModuleDefinition* pModuleDefinition ) { // Debug Profiling. @@ -2763,6 +2823,24 @@ bool AssetManager::scanDeclaredAssets( const char* pPath, const char* pExtension continue; } + U32 netId = HashAssetId(assetIdBuffer); + + // Collision detection + typeNetIdToAssetMap::iterator netIterator = mNetIdToAsset.find(netId); + if (netIterator != mNetIdToAsset.end()) + { + Con::errorf( + "AssetManager: Hash collision for '%s' and '%s'", + assetIdBuffer, + mNetIdToAsset.find(netId)->value + ); + + AssertFatal(false, "Asset hash collision detected."); + } + + mNetIdToAsset.insert(netId, foundAssetDefinition.mAssetId); + mAssetToNetId.insert(foundAssetDefinition.mAssetId, netId); + // Create new asset definition. AssetDefinition* pAssetDefinition = new AssetDefinition( foundAssetDefinition ); diff --git a/Engine/source/assets/assetManager.h b/Engine/source/assets/assetManager.h index 60afe08c9..ed915ccbb 100644 --- a/Engine/source/assets/assetManager.h +++ b/Engine/source/assets/assetManager.h @@ -59,6 +59,10 @@ #include "assets/assetFieldTypes.h" #endif +#ifndef _NETCONNECTION_H_ +#include "sim/netConnection.h" +#endif + // Debug Profiling. #include "platform/profiler.h" @@ -84,6 +88,11 @@ public: typedef HashTable typeAssetIsDependedOnHash; typedef HashMap typeAssetPtrRefreshHash; + // ASSET NETWORK PACK + typedef U32 typeAssetNetId; + typedef HashMap typeNetIdToAssetMap; + typedef HashMap typeAssetToNetIdMap; + // ASSET NETWORK PACK END private: /// Declared assets. typeDeclaredAssetsHash mDeclaredAssets; @@ -91,6 +100,11 @@ private: /// Referenced assets. typeReferencedAssetsHash mReferencedAssets; + // ASSET NETWORK PACK + typeNetIdToAssetMap mNetIdToAsset; + typeAssetToNetIdMap mAssetToNetId; + // ASSET NETWORK PACK END + /// Asset dependencies. typeAssetDependsOnHash mAssetDependsOn; typeAssetIsDependedOnHash mAssetIsDependedOn; @@ -151,6 +165,14 @@ public: bool isReferencedAsset( const char* pAssetId ); bool renameReferencedAsset( const char* pAssetIdFrom, const char* pAssetIdTo ); + // ASSET NETWORK PACK + void packDataAsset(BitStream* stream, const char* pAssetId); + const char* unpackDataAsset(BitStream* stream); + + void packUpdateAsset(NetConnection* con, U32 mask, BitStream* stream, const char* pAssetId); + const char* unpackUpdateAsset(NetConnection* con, BitStream* stream); + // ASSET NETWORK PACK END + /// /// Compile all assets. ///