TorqueGameEngines/Torque3D
1
0
Fork 0
mirror of https://github.com/TorqueGameEngines/Torque3D.git synced 2026-07-15 16:44:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Asset data management

Browse source 
Added functions to AssetManager to pack/unpack assets using an id hash instead of their id string this cuts down how much data is sent across the network for assets

All asset macros updated to use this new functionality.
This commit is contained in:
marauder2k7 2025-12-09 10:04:33 +00:00
parent 563c0266e4
commit 74638c0f2c
4 changed files with 115 additions and 173 deletions
Download patch file Download diff file Expand all files Collapse all files

46
Engine/source/T3D/assets/SoundAsset.h
View file

@ -536,52 +536,12 @@ if (m##name##AssetId[index] != StringTable->EmptyString())\
} }
#define PACKDATA_SOUNDASSET_ARRAY(name, index)\ #define PACKDATA_SOUNDASSET_ARRAY(name, index)\
if (stream->writeFlag(AssetDatabase.isDeclaredAsset(m##name##AssetId[index])))\ AssetDatabase.packDataAsset(stream, m##name##AssetId[index])
{\
stream->writeString(m##name##AssetId[index]);\
}\
else\
{\
if(stream->writeFlag(Sim::findObject(m##name##Name[index])))\
{\
SFXTrack* sndTrack = get##name##Profile(index);\
if(stream->writeFlag(sndTrack != nullptr))\
{\
stream->writeRangedU32(SimObjectId(sndTrack->getId()), DataBlockObjectIdFirst, DataBlockObjectIdLast);\
sfxWrite(stream, sndTrack);\
}\
}\
else\
{\
stream->writeString(m##name##Name[index]);\
}\
}
//network recieve - datablock //network recieve - datablock
#define UNPACKDATA_SOUNDASSET_ARRAY(name, index)\ #define UNPACKDATA_SOUNDASSET_ARRAY(name, index)\
if (stream->readFlag())\ m##name##AssetId[index] = AssetDatabase.unpackDataAsset(stream)
{\
m##name##AssetId[index] = stream->readSTString();\
_set##name(m##name##AssetId[index], index);\
}\
else\
{\
if(stream->readFlag())\
{\
String errorStr;\
if(stream->readFlag())\
{\
m##name##SFXId[index] = stream->readRangedU32( DataBlockObjectIdFirst, DataBlockObjectIdLast );\
sfxReadAndResolve(stream, &m##name##Profile[index], errorStr);\
}\
}\
else\
{\
m##name##Name[index] = stream->readSTString(); \
_set##name(m##name##Name[index], index); \
}\
}
#pragma endregion #pragma endregion
#endif // _ASSET_BASE_H_ #endif // _ASSET_BASE_H_

142
Engine/source/T3D/assets/assetMacroHelpers.h
View file

@ -59,122 +59,46 @@ if (m##name##AssetId != StringTable->EmptyString())\
//network send - datablock refactor //network send - datablock refactor
#define PACKDATA_ASSET_REFACTOR(name)\ #define PACKDATA_ASSET_REFACTOR(name)\
if (stream->writeFlag(m##name##Asset.notNull()))\ AssetDatabase.packDataAsset(stream, m##name##Asset.getAssetId())
{\
stream->writeString(m##name##Asset.getAssetId());\
}\
else if (stream->writeFlag(m##name##File != StringTable->EmptyString()))\
{\
stream->writeString(m##name##File);\
}\
//network recieve - datablock //network recieve - datablock
#define UNPACKDATA_ASSET_REFACTOR(name)\ #define UNPACKDATA_ASSET_REFACTOR(name)\
if (stream->readFlag())\ m##name##Asset = AssetDatabase.unpackDataAsset(stream)
{\
_set##name(stream->readSTString());\
}\
else if (stream->readFlag())\
{\
_set##name(stream->readSTString());\
}\
else\
{\
_set##name(StringTable->EmptyString());\
}\
//network send - object-instance //network send - object-instance
#define PACK_ASSET_REFACTOR(netconn, name)\ #define PACK_ASSET_REFACTOR(netconn, name)\
if (stream->writeFlag(m##name##Asset.notNull()))\ AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset.getAssetId())
{\
NetStringHandle assetIdStr = m##name##Asset.getAssetId();\
netconn->packNetStringHandleU(stream, assetIdStr);\
}\
else if (stream->writeFlag(m##name##File != StringTable->EmptyString()))\
{\
NetStringHandle fileStr = m##name##File;\
netconn->packNetStringHandleU(stream, fileStr);\
}\
//network recieve - object-instance //network recieve - object-instance
#define UNPACK_ASSET_REFACTOR(netconn, name)\ #define UNPACK_ASSET_REFACTOR(netconn, name)\
if (stream->readFlag())\ _set##name(AssetDatabase.unpackUpdateAsset(netconn, stream))
{\
_set##name(netconn->unpackNetStringHandleU(stream).getString());\
}\
else if (stream->readFlag())\
{\
_set##name(netconn->unpackNetStringHandleU(stream).getString());\
}\
else\
{\
_set##name(StringTable->EmptyString());\
}\
//network send - datablock //network send - datablock
#define PACKDATA_ASSET_ARRAY_REFACTOR(name, max)\ #define PACKDATA_ASSET_ARRAY_REFACTOR(name, max)\
for (U32 i = 0; i < max; i++)\ for (U32 i = 0; i < max; i++)\
{\ {\
if (stream->writeFlag(m##name##Asset[i].notNull()))\ AssetDatabase.packDataAsset(stream, m##name##Asset[i].getAssetId());\
{\
stream->writeString(m##name##Asset[i].getAssetId()); \
}\
else if (stream->writeFlag(m##name##File[i] != StringTable->EmptyString()))\
{\
stream->writeString(m##name##File[i]);\
}\
} }
//network recieve - datablock //network recieve - datablock
#define UNPACKDATA_ASSET_ARRAY_REFACTOR(name, max)\ #define UNPACKDATA_ASSET_ARRAY_REFACTOR(name, max)\
for (U32 i = 0; i < max; i++)\ for (U32 i = 0; i < max; i++)\
{\ {\
if (stream->readFlag())\ m##name##Asset[i] = AssetDatabase.unpackDataAsset(stream);\
{\
m##name##Asset[i] = stream->readSTString();\
}\
else if (stream->readFlag())\
{\
_set##name(stream->readSTString(), i);\
}\
else\
{\
_set##name(StringTable->EmptyString(), i);\
}\
} }
//network send - object-instance //network send - object-instance
#define PACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\ #define PACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\
for (U32 i = 0; i < max; i++)\ for (U32 i = 0; i < max; i++)\
{\ {\
if (stream->writeFlag(m##name##Asset[i].notNull()))\ AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset[i].getAssetId());\
{\
NetStringHandle assetIdStr = m##name##Asset[i].getAssetId();\
netconn->packNetStringHandleU(stream, assetIdStr);\
}\
else if (stream->writeFlag(m##name##File[i] != StringTable->EmptyString()))\
{\
NetStringHandle fileStr = m##name##File[i];\
netconn->packNetStringHandleU(stream, fileStr);\
}\
} }
//network recieve - object-instance //network recieve - object-instance
#define UNPACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\ #define UNPACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\
for (U32 i = 0; i < max; i++)\ for (U32 i = 0; i < max; i++)\
{\ {\
if (stream->readFlag())\ _set##name(AssetDatabase.unpackUpdateAsset(netconn, stream), i);\
{\
m##name##Asset[i] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\
}\
else if (stream->readFlag())\
{\
_set##name(StringTable->insert(netconn->unpackNetStringHandleU(stream).getString()), i);\
}\
else\
{\
_set##name(StringTable->EmptyString(), i);\
}\
} }
#define DEF_ASSET_BINDS_REFACTOR(className,name)\ #define DEF_ASSET_BINDS_REFACTOR(className,name)\
@ -348,61 +272,19 @@ if (m##name##AssetId[index] != StringTable->EmptyString())\
} }
//network send - datablock //network send - datablock
#define PACKDATA_ASSET_ARRAY(name, index)\ #define PACKDATA_ASSET_ARRAY(name, index)\
if (stream->writeFlag(m##name##Asset[index].notNull()))\ AssetDatabase.packDataAsset(stream, m##name##Asset[index].getAssetId())
{\
stream->writeString(m##name##Asset[index].getAssetId());\
}\
else if (stream->writeFlag(m##name##Name[index] != StringTable->EmptyString()))\
{\
stream->writeString(m##name##Name[index].getAssetId());\
}\
//network recieve - datablock //network recieve - datablock
#define UNPACKDATA_ASSET_ARRAY(name, index)\ #define UNPACKDATA_ASSET_ARRAY(name, index)\
if (stream->readFlag())\ m##name##Asset[index] = AssetDatabase.unpackDataAsset(stream)
{\
m##name##AssetId[index] = stream->readSTString();\
_set##name(m##name##AssetId[index], index);\
}\
else if (stream->readFlag())\
{\
m##name##Name[index] = stream->readSTString();\
_set##name(m##name##Name[index], index);\
}\
else\
{\
_set##name(StringTable->EmptyString());\
}\
//network send - object-instance //network send - object-instance
#define PACK_ASSET_ARRAY(netconn, name, index)\ #define PACK_ASSET_ARRAY(netconn, name, index)\
if (stream->writeFlag(m##name##Asset[index].notNull()))\ AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset[index].getAssetId());
{\
NetStringHandle assetIdStr = m##name##Asset[index].getAssetId();\
netconn->packNetStringHandleU(stream, assetIdStr);\
}\
else if (stream->writeFlag(m##name##Name[index] != StringTable->EmptyString()))\
{\
NetStringHandle fileStr = m##name##Name[index].getAssetId();\
netconn->packNetStringHandleU(stream, fileStr);\
}\
//network recieve - object-instance //network recieve - object-instance
#define UNPACK_ASSET_ARRAY(netconn, name, index)\ #define UNPACK_ASSET_ARRAY(netconn, name, index)\
if (stream->readFlag())\ _set##name(AssetDatabase.unpackUpdateAsset(netconn, stream), index);
{\
m##name##AssetId[index] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\
_set##name(m##name##AssetId[index], index);\
}\
else if (stream->readFlag())\
{\
m##name##Name[index] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\
_set##name(m##name##Name[index], index);\
}\
else\
{\
_set##name(StringTable->EmptyString(), index);\
}
//script methods for class.asset acces //script methods for class.asset acces
//declare general get<entry>, get<entry>Asset and set<entry> methods //declare general get<entry>, get<entry>Asset and set<entry> methods

78
Engine/source/assets/assetManager.cpp
View file

@ -964,6 +964,55 @@ bool AssetManager::renameReferencedAsset( const char* pAssetIdFrom, const char*
return true; return true;
} }
void AssetManager::packDataAsset(BitStream* stream, const char* pAssetId)
{
if(stream->writeFlag(isDeclaredAsset(pAssetId)))
{
stream->write(mAssetToNetId.find(pAssetId)->value);
}
}
const char* AssetManager::unpackDataAsset(BitStream* stream)
{
if (stream->readFlag())
{
typeAssetNetId netId;
stream->read(&netId);
typeNetIdToAssetMap::iterator netChar = mNetIdToAsset.find(netId);
if (netChar != mNetIdToAsset.end())
{
return netChar->value;
}
}
return StringTable->EmptyString();
}
void AssetManager::packUpdateAsset(NetConnection* con, U32 mask, BitStream* stream, const char* pAssetId)
{
if (stream->writeFlag(isDeclaredAsset(pAssetId)))
{
stream->write(mAssetToNetId.find(pAssetId)->value);
}
}
const char* AssetManager::unpackUpdateAsset(NetConnection* con, BitStream* stream)
{
if (stream->readFlag())
{
typeAssetNetId netId;
stream->read(&netId);
typeNetIdToAssetMap::iterator netChar = mNetIdToAsset.find(netId);
if (netChar != mNetIdToAsset.end())
{
return netChar->value;
}
}
return StringTable->EmptyString();
}
bool AssetManager::compileAllAssets(const bool compressed, const bool includeUnloaded) bool AssetManager::compileAllAssets(const bool compressed, const bool includeUnloaded)
{ {
// Debug Profiling. // Debug Profiling.
@ -2642,6 +2691,17 @@ const char* AssetManager::getAssetLooseFile(const char* pAssetId, const S32& ind
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
static U32 HashAssetId(const char* str)
{
U32 hash = 2166136261u;
while (*str)
{
hash ^= (U8)*str++;
hash *= 16777619u;
}
return hash;
}
bool AssetManager::scanDeclaredAssets( const char* pPath, const char* pExtension, const bool recurse, ModuleDefinition* pModuleDefinition ) bool AssetManager::scanDeclaredAssets( const char* pPath, const char* pExtension, const bool recurse, ModuleDefinition* pModuleDefinition )
{ {
// Debug Profiling. // Debug Profiling.
@ -2763,6 +2823,24 @@ bool AssetManager::scanDeclaredAssets( const char* pPath, const char* pExtension
continue; continue;
} }
U32 netId = HashAssetId(assetIdBuffer);
// Collision detection
typeNetIdToAssetMap::iterator netIterator = mNetIdToAsset.find(netId);
if (netIterator != mNetIdToAsset.end())
{
Con::errorf(
"AssetManager: Hash collision for '%s' and '%s'",
assetIdBuffer,
mNetIdToAsset.find(netId)->value
);
AssertFatal(false, "Asset hash collision detected.");
}
mNetIdToAsset.insert(netId, foundAssetDefinition.mAssetId);
mAssetToNetId.insert(foundAssetDefinition.mAssetId, netId);
// Create new asset definition. // Create new asset definition.
AssetDefinition* pAssetDefinition = new AssetDefinition( foundAssetDefinition ); AssetDefinition* pAssetDefinition = new AssetDefinition( foundAssetDefinition );

22
Engine/source/assets/assetManager.h
View file

@ -59,6 +59,10 @@
#include "assets/assetFieldTypes.h" #include "assets/assetFieldTypes.h"
#endif #endif
#ifndef _NETCONNECTION_H_
#include "sim/netConnection.h"
#endif
// Debug Profiling. // Debug Profiling.
#include "platform/profiler.h" #include "platform/profiler.h"
@ -84,6 +88,11 @@ public:
typedef HashTable<typeAssetId, typeAssetId> typeAssetIsDependedOnHash; typedef HashTable<typeAssetId, typeAssetId> typeAssetIsDependedOnHash;
typedef HashMap<AssetPtrBase*, AssetPtrCallback*> typeAssetPtrRefreshHash; typedef HashMap<AssetPtrBase*, AssetPtrCallback*> typeAssetPtrRefreshHash;
// ASSET NETWORK PACK
typedef U32 typeAssetNetId;
typedef HashMap<typeAssetNetId, typeAssetId> typeNetIdToAssetMap;
typedef HashMap<typeAssetId, typeAssetNetId> typeAssetToNetIdMap;
// ASSET NETWORK PACK END
private: private:
/// Declared assets. /// Declared assets.
typeDeclaredAssetsHash mDeclaredAssets; typeDeclaredAssetsHash mDeclaredAssets;
@ -91,6 +100,11 @@ private:
/// Referenced assets. /// Referenced assets.
typeReferencedAssetsHash mReferencedAssets; typeReferencedAssetsHash mReferencedAssets;
// ASSET NETWORK PACK
typeNetIdToAssetMap mNetIdToAsset;
typeAssetToNetIdMap mAssetToNetId;
// ASSET NETWORK PACK END
/// Asset dependencies. /// Asset dependencies.
typeAssetDependsOnHash mAssetDependsOn; typeAssetDependsOnHash mAssetDependsOn;
typeAssetIsDependedOnHash mAssetIsDependedOn; typeAssetIsDependedOnHash mAssetIsDependedOn;
@ -151,6 +165,14 @@ public:
bool isReferencedAsset( const char* pAssetId ); bool isReferencedAsset( const char* pAssetId );
bool renameReferencedAsset( const char* pAssetIdFrom, const char* pAssetIdTo ); bool renameReferencedAsset( const char* pAssetIdFrom, const char* pAssetIdTo );
// ASSET NETWORK PACK
void packDataAsset(BitStream* stream, const char* pAssetId);
const char* unpackDataAsset(BitStream* stream);
void packUpdateAsset(NetConnection* con, U32 mask, BitStream* stream, const char* pAssetId);
const char* unpackUpdateAsset(NetConnection* con, BitStream* stream);
// ASSET NETWORK PACK END
/// <summary> /// <summary>
/// Compile all assets. /// Compile all assets.
/// </summary> /// </summary>