TorqueGameEngines/Torque3D
1
0
Fork 0
mirror of https://github.com/TorqueGameEngines/Torque3D.git synced 2026-01-19 20:24:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Asset data management

Browse source 
Added functions to AssetManager to pack/unpack assets using an id hash instead of their id string this cuts down how much data is sent across the network for assets

All asset macros updated to use this new functionality.
This commit is contained in:
marauder2k7 2025-12-09 10:04:33 +00:00
parent 563c0266e4
commit 74638c0f2c
4 changed files with 115 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
Engine/source/T3D/assets/SoundAsset.h
View file

@ -536,52 +536,12 @@ if (m##name##AssetId[index] != StringTable->EmptyString())\
}
#define PACKDATA_SOUNDASSET_ARRAY(name, index)\
if (stream->writeFlag(AssetDatabase.isDeclaredAsset(m##name##AssetId[index])))\
{\
stream->writeString(m##name##AssetId[index]);\
}\
else\
{\
if(stream->writeFlag(Sim::findObject(m##name##Name[index])))\
{\
SFXTrack* sndTrack = get##name##Profile(index);\
if(stream->writeFlag(sndTrack != nullptr))\
{\
stream->writeRangedU32(SimObjectId(sndTrack->getId()), DataBlockObjectIdFirst, DataBlockObjectIdLast);\
sfxWrite(stream, sndTrack);\
}\
}\
else\
{\
stream->writeString(m##name##Name[index]);\
}\
}
AssetDatabase.packDataAsset(stream, m##name##AssetId[index])
//network recieve - datablock
#define UNPACKDATA_SOUNDASSET_ARRAY(name, index)\
if (stream->readFlag())\
{\
m##name##AssetId[index] = stream->readSTString();\
_set##name(m##name##AssetId[index], index);\
}\
else\
{\
if(stream->readFlag())\
{\
String errorStr;\
if(stream->readFlag())\
{\
m##name##SFXId[index] = stream->readRangedU32( DataBlockObjectIdFirst, DataBlockObjectIdLast );\
sfxReadAndResolve(stream, &m##name##Profile[index], errorStr);\
}\
}\
else\
{\
m##name##Name[index] = stream->readSTString(); \
_set##name(m##name##Name[index], index); \
}\
}
m##name##AssetId[index] = AssetDatabase.unpackDataAsset(stream)
#pragma endregion
#endif // _ASSET_BASE_H_

142
Engine/source/T3D/assets/assetMacroHelpers.h
View file

@ -59,122 +59,46 @@ if (m##name##AssetId != StringTable->EmptyString())\
//network send - datablock refactor
#define PACKDATA_ASSET_REFACTOR(name)\
if (stream->writeFlag(m##name##Asset.notNull()))\
{\
stream->writeString(m##name##Asset.getAssetId());\
}\
else if (stream->writeFlag(m##name##File != StringTable->EmptyString()))\
{\
stream->writeString(m##name##File);\
}\
AssetDatabase.packDataAsset(stream, m##name##Asset.getAssetId())
//network recieve - datablock
#define UNPACKDATA_ASSET_REFACTOR(name)\
if (stream->readFlag())\
{\
_set##name(stream->readSTString());\
}\
else if (stream->readFlag())\
{\
_set##name(stream->readSTString());\
}\
else\
{\
_set##name(StringTable->EmptyString());\
}\
m##name##Asset = AssetDatabase.unpackDataAsset(stream)
//network send - object-instance
#define PACK_ASSET_REFACTOR(netconn, name)\
if (stream->writeFlag(m##name##Asset.notNull()))\
{\
NetStringHandle assetIdStr = m##name##Asset.getAssetId();\
netconn->packNetStringHandleU(stream, assetIdStr);\
}\
else if (stream->writeFlag(m##name##File != StringTable->EmptyString()))\
{\
NetStringHandle fileStr = m##name##File;\
netconn->packNetStringHandleU(stream, fileStr);\
}\
AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset.getAssetId())
//network recieve - object-instance
#define UNPACK_ASSET_REFACTOR(netconn, name)\
if (stream->readFlag())\
{\
_set##name(netconn->unpackNetStringHandleU(stream).getString());\
}\
else if (stream->readFlag())\
{\
_set##name(netconn->unpackNetStringHandleU(stream).getString());\
}\
else\
{\
_set##name(StringTable->EmptyString());\
}\
_set##name(AssetDatabase.unpackUpdateAsset(netconn, stream))
//network send - datablock
#define PACKDATA_ASSET_ARRAY_REFACTOR(name, max)\
for (U32 i = 0; i < max; i++)\
{\
if (stream->writeFlag(m##name##Asset[i].notNull()))\
{\
stream->writeString(m##name##Asset[i].getAssetId()); \
}\
else if (stream->writeFlag(m##name##File[i] != StringTable->EmptyString()))\
{\
stream->writeString(m##name##File[i]);\
}\
AssetDatabase.packDataAsset(stream, m##name##Asset[i].getAssetId());\
}
//network recieve - datablock
#define UNPACKDATA_ASSET_ARRAY_REFACTOR(name, max)\
for (U32 i = 0; i < max; i++)\
{\
if (stream->readFlag())\
{\
m##name##Asset[i] = stream->readSTString();\
}\
else if (stream->readFlag())\
{\
_set##name(stream->readSTString(), i);\
}\
else\
{\
_set##name(StringTable->EmptyString(), i);\
}\
m##name##Asset[i] = AssetDatabase.unpackDataAsset(stream);\
}
//network send - object-instance
#define PACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\
for (U32 i = 0; i < max; i++)\
{\
if (stream->writeFlag(m##name##Asset[i].notNull()))\
{\
NetStringHandle assetIdStr = m##name##Asset[i].getAssetId();\
netconn->packNetStringHandleU(stream, assetIdStr);\
}\
else if (stream->writeFlag(m##name##File[i] != StringTable->EmptyString()))\
{\
NetStringHandle fileStr = m##name##File[i];\
netconn->packNetStringHandleU(stream, fileStr);\
}\
AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset[i].getAssetId());\
}
//network recieve - object-instance
#define UNPACK_ASSET_ARRAY_REFACTOR(netconn, name, max)\
for (U32 i = 0; i < max; i++)\
{\
if (stream->readFlag())\
{\
m##name##Asset[i] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\
}\
else if (stream->readFlag())\
{\
_set##name(StringTable->insert(netconn->unpackNetStringHandleU(stream).getString()), i);\
}\
else\
{\
_set##name(StringTable->EmptyString(), i);\
}\
_set##name(AssetDatabase.unpackUpdateAsset(netconn, stream), i);\
}
#define DEF_ASSET_BINDS_REFACTOR(className,name)\
@ -348,61 +272,19 @@ if (m##name##AssetId[index] != StringTable->EmptyString())\
}
//network send - datablock
#define PACKDATA_ASSET_ARRAY(name, index)\
if (stream->writeFlag(m##name##Asset[index].notNull()))\
{\
stream->writeString(m##name##Asset[index].getAssetId());\
}\
else if (stream->writeFlag(m##name##Name[index] != StringTable->EmptyString()))\
{\
stream->writeString(m##name##Name[index].getAssetId());\
}\
AssetDatabase.packDataAsset(stream, m##name##Asset[index].getAssetId())
//network recieve - datablock
#define UNPACKDATA_ASSET_ARRAY(name, index)\
if (stream->readFlag())\
{\
m##name##AssetId[index] = stream->readSTString();\
_set##name(m##name##AssetId[index], index);\
}\
else if (stream->readFlag())\
{\
m##name##Name[index] = stream->readSTString();\
_set##name(m##name##Name[index], index);\
}\
else\
{\
_set##name(StringTable->EmptyString());\
}\
m##name##Asset[index] = AssetDatabase.unpackDataAsset(stream)
//network send - object-instance
#define PACK_ASSET_ARRAY(netconn, name, index)\
if (stream->writeFlag(m##name##Asset[index].notNull()))\
{\
NetStringHandle assetIdStr = m##name##Asset[index].getAssetId();\
netconn->packNetStringHandleU(stream, assetIdStr);\
}\
else if (stream->writeFlag(m##name##Name[index] != StringTable->EmptyString()))\
{\
NetStringHandle fileStr = m##name##Name[index].getAssetId();\
netconn->packNetStringHandleU(stream, fileStr);\
}\
AssetDatabase.packUpdateAsset(netconn, mask, stream, m##name##Asset[index].getAssetId());
//network recieve - object-instance
#define UNPACK_ASSET_ARRAY(netconn, name, index)\
if (stream->readFlag())\
{\
m##name##AssetId[index] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\
_set##name(m##name##AssetId[index], index);\
}\
else if (stream->readFlag())\
{\
m##name##Name[index] = StringTable->insert(netconn->unpackNetStringHandleU(stream).getString());\
_set##name(m##name##Name[index], index);\
}\
else\
{\
_set##name(StringTable->EmptyString(), index);\
}
_set##name(AssetDatabase.unpackUpdateAsset(netconn, stream), index);
//script methods for class.asset acces
//declare general get<entry>, get<entry>Asset and set<entry> methods

78
Engine/source/assets/assetManager.cpp
View file

@ -964,6 +964,55 @@ bool AssetManager::renameReferencedAsset( const char* pAssetIdFrom, const char*
return true;
}
void AssetManager::packDataAsset(BitStream* stream, const char* pAssetId)
{
if(stream->writeFlag(isDeclaredAsset(pAssetId)))
{
stream->write(mAssetToNetId.find(pAssetId)->value);
}
}
const char* AssetManager::unpackDataAsset(BitStream* stream)
{
if (stream->readFlag())
{
typeAssetNetId netId;
stream->read(&netId);
typeNetIdToAssetMap::iterator netChar = mNetIdToAsset.find(netId);
if (netChar != mNetIdToAsset.end())
{
return netChar->value;
}
}
return StringTable->EmptyString();
}
void AssetManager::packUpdateAsset(NetConnection* con, U32 mask, BitStream* stream, const char* pAssetId)
{
if (stream->writeFlag(isDeclaredAsset(pAssetId)))
{
stream->write(mAssetToNetId.find(pAssetId)->value);
}
}
const char* AssetManager::unpackUpdateAsset(NetConnection* con, BitStream* stream)
{
if (stream->readFlag())
{
typeAssetNetId netId;
stream->read(&netId);
typeNetIdToAssetMap::iterator netChar = mNetIdToAsset.find(netId);
if (netChar != mNetIdToAsset.end())
{
return netChar->value;
}
}
return StringTable->EmptyString();
}
bool AssetManager::compileAllAssets(const bool compressed, const bool includeUnloaded)
{
// Debug Profiling.
@ -2642,6 +2691,17 @@ const char* AssetManager::getAssetLooseFile(const char* pAssetId, const S32& ind
//-----------------------------------------------------------------------------
static U32 HashAssetId(const char* str)
{
U32 hash = 2166136261u;
while (*str)
{
hash ^= (U8)*str++;
hash *= 16777619u;
}
return hash;
}
bool AssetManager::scanDeclaredAssets( const char* pPath, const char* pExtension, const bool recurse, ModuleDefinition* pModuleDefinition )
{
// Debug Profiling.
@ -2763,6 +2823,24 @@ bool AssetManager::scanDeclaredAssets( const char* pPath, const char* pExtension
continue;
}
U32 netId = HashAssetId(assetIdBuffer);
// Collision detection
typeNetIdToAssetMap::iterator netIterator = mNetIdToAsset.find(netId);
if (netIterator != mNetIdToAsset.end())
{
Con::errorf(
"AssetManager: Hash collision for '%s' and '%s'",
assetIdBuffer,
mNetIdToAsset.find(netId)->value
);
AssertFatal(false, "Asset hash collision detected.");
}
mNetIdToAsset.insert(netId, foundAssetDefinition.mAssetId);
mAssetToNetId.insert(foundAssetDefinition.mAssetId, netId);
// Create new asset definition.
AssetDefinition* pAssetDefinition = new AssetDefinition( foundAssetDefinition );

22
Engine/source/assets/assetManager.h
View file

@ -59,6 +59,10 @@
#include "assets/assetFieldTypes.h"
#endif
#ifndef _NETCONNECTION_H_
#include "sim/netConnection.h"
#endif
// Debug Profiling.
#include "platform/profiler.h"
@ -84,6 +88,11 @@ public:
typedef HashTable<typeAssetId, typeAssetId> typeAssetIsDependedOnHash;
typedef HashMap<AssetPtrBase*, AssetPtrCallback*> typeAssetPtrRefreshHash;
// ASSET NETWORK PACK
typedef U32 typeAssetNetId;
typedef HashMap<typeAssetNetId, typeAssetId> typeNetIdToAssetMap;
typedef HashMap<typeAssetId, typeAssetNetId> typeAssetToNetIdMap;
// ASSET NETWORK PACK END
private:
/// Declared assets.
typeDeclaredAssetsHash mDeclaredAssets;
@ -91,6 +100,11 @@ private:
/// Referenced assets.
typeReferencedAssetsHash mReferencedAssets;
// ASSET NETWORK PACK
typeNetIdToAssetMap mNetIdToAsset;
typeAssetToNetIdMap mAssetToNetId;
// ASSET NETWORK PACK END
/// Asset dependencies.
typeAssetDependsOnHash mAssetDependsOn;
typeAssetIsDependedOnHash mAssetIsDependedOn;
@ -151,6 +165,14 @@ public:
bool isReferencedAsset( const char* pAssetId );
bool renameReferencedAsset( const char* pAssetIdFrom, const char* pAssetIdTo );
// ASSET NETWORK PACK
void packDataAsset(BitStream* stream, const char* pAssetId);
const char* unpackDataAsset(BitStream* stream);
void packUpdateAsset(NetConnection* con, U32 mask, BitStream* stream, const char* pAssetId);
const char* unpackUpdateAsset(NetConnection* con, BitStream* stream);
// ASSET NETWORK PACK END
/// <summary>
/// Compile all assets.
/// </summary>